diff options
1161 files changed, 5406 insertions, 31560 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index c0f68a1c0b7..440ecb18024 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -69,6 +69,7 @@ add_subdirectory(container-messagebus) add_subdirectory(container-search) add_subdirectory(container-search-gui) add_subdirectory(container-search-and-docproc) +add_subdirectory(container-spifly) add_subdirectory(cloud-tenant-cd) add_subdirectory(clustercontroller-apps) add_subdirectory(clustercontroller-core) @@ -92,7 +93,6 @@ add_subdirectory(http-client) add_subdirectory(jdisc-cloud-aws) add_subdirectory(jdisc_core) add_subdirectory(jdisc-security-filters) -add_subdirectory(jdisc_jetty) add_subdirectory(jrt_test) add_subdirectory(linguistics) add_subdirectory(linguistics-components) @@ -121,7 +121,6 @@ add_subdirectory(slobrok) add_subdirectory(standalone-container) add_subdirectory(storage) add_subdirectory(storageserver) -add_subdirectory(statistics) add_subdirectory(streamingvisitors) add_subdirectory(tenant-cd-api) add_subdirectory(vbench) @@ -130,11 +129,10 @@ add_subdirectory(vdstestlib) add_subdirectory(vespa-athenz) add_subdirectory(vespa-feed-client) add_subdirectory(vespa-feed-client-cli) -add_subdirectory(vespa-http-client) add_subdirectory(vespa-osgi-testrunner) add_subdirectory(vespa-testrunner-components) add_subdirectory(vespa_feed_perf) -add_subdirectory(vespa_jersey2) +add_subdirectory(vespa-3party-bundles) add_subdirectory(vespabase) add_subdirectory(vespaclient) add_subdirectory(vespaclient-core) diff --git a/Code-map.md b/Code-map.md index 40307ef7e30..dfd11fffdab 100644 --- a/Code-map.md +++ b/Code-map.md @@ -61,7 +61,7 @@ Document operation modules: - [indexinglanguage](https://github.com/vespa-engine/vespa/tree/master/indexinglanguage) - implementation of the "indexing" language which is used to express the statements prefixed by "indexing:" in the search definition. - [docprocs](https://github.com/vespa-engine/vespa/tree/master/docprocs) - document processor components bundled with Vespa. Notably the Indexingprocessor - a document processor invoking the indexing language statements configured for the document type in question on document operations. - [vespaclient-container-plugin](https://github.com/vespa-engine/vespa/tree/master/vespaclient-container-plugin) - implements the document/v1 API and internal API used by the Java HTTP client on top of the jDisc container, forwarding to the Document API. -- [vespa-http-client](https://github.com/vespa-engine/vespa/tree/master/vespa-http-client) - client for fast writing to the internal API implemented by vespaclient-container-plugin. +- [vespa-feed-client](https://github.com/vespa-engine/vespa/tree/master/vespa-feed-client) - client for fast writing to the internal API implemented by vespaclient-container-plugin. ## Content nodes @@ -1 +1 @@ -7.164.0 +8.0.0 diff --git a/abi-check-plugin/pom.xml b/abi-check-plugin/pom.xml index 8d782b1d426..d24765a5f88 100644 --- a/abi-check-plugin/pom.xml +++ b/abi-check-plugin/pom.xml @@ -6,11 +6,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>abi-check-plugin</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>maven-plugin</packaging> <name>${project.artifactId}</name> <description>Maven Plugin for ensuring ABI stability.</description> diff --git a/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/AnnotationCollector.java b/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/AnnotationCollector.java index d6a06d1e9c1..a68fe955cc0 100644 --- a/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/AnnotationCollector.java +++ b/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/AnnotationCollector.java @@ -13,7 +13,7 @@ public class AnnotationCollector extends ClassVisitor { private final Set<String> annotations = new HashSet<>(); public AnnotationCollector() { - super(Opcodes.ASM7); + super(Opcodes.ASM9); } @Override diff --git a/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/PublicSignatureCollector.java b/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/PublicSignatureCollector.java index 29121d66ef9..737a2724fc8 100644 --- a/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/PublicSignatureCollector.java +++ b/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/PublicSignatureCollector.java @@ -27,7 +27,7 @@ public class PublicSignatureCollector extends ClassVisitor { private Set<String> currentFields; public PublicSignatureCollector() { - super(Opcodes.ASM7); + super(Opcodes.ASM9); } private static boolean testBit(long access, long mask) { diff --git a/annotations/pom.xml b/annotations/pom.xml index 358188f71c2..e3f5455c88b 100644 --- a/annotations/pom.xml +++ b/annotations/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>annotations</artifactId> <packaging>bundle</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>annotations</name> <description>Public API annotations</description> <repositories> diff --git a/application-model/pom.xml b/application-model/pom.xml index 7eac247e249..2143f3a5ffd 100644 --- a/application-model/pom.xml +++ b/application-model/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>application-model</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>${project.artifactId}</name> <description>Model classes for Vespa application entities such as host name, application id etc.</description> <dependencies> diff --git a/application-preprocessor/pom.xml b/application-preprocessor/pom.xml index 644903e79dd..f8b521140f7 100644 --- a/application-preprocessor/pom.xml +++ b/application-preprocessor/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>application-preprocessor</artifactId> <packaging>jar</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/application/abi-spec.json b/application/abi-spec.json index 21eacb152c0..2053eabd9ad 100644 --- a/application/abi-spec.json +++ b/application/abi-spec.json @@ -239,7 +239,6 @@ "public static java.lang.String getDefMd5()", "public static java.lang.String getDefName()", "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", "public void <init>(com.yahoo.application.MockApplicationConfig$Builder)", "public com.yahoo.application.MockApplicationConfig$Mystruct mystruct()", "public java.util.List mystructlist()", @@ -255,7 +254,6 @@ "public static final java.lang.String CONFIG_DEF_MD5", "public static final java.lang.String CONFIG_DEF_NAME", "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", "public static final java.lang.String[] CONFIG_DEF_SCHEMA" ] }, diff --git a/application/pom.xml b/application/pom.xml index 7e1b54a7904..51448502d1d 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>application</artifactId> <packaging>jar</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <description>Runs an application directly from services.xml</description> <dependencies> <dependency> @@ -60,19 +60,6 @@ <version>${project.version}</version> <scope>compile</scope> </dependency> - <!-- Because these are provided in container-core and just preinstalled: --> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>jdisc_jetty</artifactId> - <version>${project.version}</version> - <scope>compile</scope> - <exclusions> - <exclusion> - <groupId>org.apache.aries.spifly</groupId> - <artifactId>org.apache.aries.spifly.dynamic.bundle</artifactId> - </exclusion> - </exclusions> - </dependency> <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> @@ -100,6 +87,10 @@ <artifactId>antlr4-runtime</artifactId> </dependency> <dependency> + <groupId>org.apache.felix</groupId> + <artifactId>org.apache.felix.framework</artifactId> + </dependency> + <dependency> <groupId>org.apache.opennlp</groupId> <artifactId>opennlp-tools</artifactId> </dependency> @@ -138,6 +129,51 @@ <scope>test</scope> </dependency> + <!-- START JETTY embedded jars --> + <dependency> + <groupId>org.eclipse.jetty.alpn</groupId> + <artifactId>alpn-api</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty.http2</groupId> + <artifactId>http2-server</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-alpn-java-server</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-client</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-continuation</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-jmx</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlet</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlets</artifactId> + <scope>test</scope> + </dependency> + <!-- END JETTY embedded jars --> + </dependencies> <build> diff --git a/application/src/main/java/com/yahoo/application/container/DocumentProcessing.java b/application/src/main/java/com/yahoo/application/container/DocumentProcessing.java index f0f85fb78dd..0c581357281 100644 --- a/application/src/main/java/com/yahoo/application/container/DocumentProcessing.java +++ b/application/src/main/java/com/yahoo/application/container/DocumentProcessing.java @@ -3,8 +3,8 @@ package com.yahoo.application.container; import com.yahoo.api.annotations.Beta; import com.yahoo.component.ComponentSpecification; -import com.yahoo.docproc.DocprocExecutor; -import com.yahoo.docproc.DocprocService; +import com.yahoo.docproc.impl.DocprocExecutor; +import com.yahoo.docproc.impl.DocprocService; import com.yahoo.docproc.DocumentProcessor; import com.yahoo.docproc.jdisc.DocumentProcessingHandler; import com.yahoo.document.DocumentType; @@ -59,13 +59,8 @@ public final class DocumentProcessing { * @return Progress.DONE or Progress.FAILED * @throws RuntimeException if one of the document processors in the chain throws, or if the calling thread is interrupted */ - @SuppressWarnings("removal") // TODO Vespa 8: remove public DocumentProcessor.Progress process(ComponentSpecification chain, com.yahoo.docproc.Processing processing) { DocprocExecutor executor = getExecutor(chain); - - // TODO Vespa 8: Remove statement (registry will be removed from Processing) - processing.setDocprocServiceRegistry(handler.getDocprocServiceRegistry()); - return executor.processUntilDone(processing); } @@ -82,17 +77,11 @@ public final class DocumentProcessing { * @return any Progress * @throws RuntimeException if one of the document processors in the chain throws */ - @SuppressWarnings("removal") // TODO Vespa 8: remove public DocumentProcessor.Progress processOnce(ComponentSpecification chain, com.yahoo.docproc.Processing processing) { DocprocExecutor executor = getExecutor(chain); - - // TODO Vespa 8: Remove statement (registry will be removed from Processing) - processing.setDocprocServiceRegistry(handler.getDocprocServiceRegistry()); - return executor.process(processing); } - @SuppressWarnings("removal") // TODO Vespa 8: remove private DocprocExecutor getExecutor(ComponentSpecification chain) { DocprocService service = handler.getDocprocServiceRegistry().getComponent(chain); if (service == null) { diff --git a/application/src/main/java/com/yahoo/application/container/SynchronousRequestResponseHandler.java b/application/src/main/java/com/yahoo/application/container/SynchronousRequestResponseHandler.java index 87d1eff05ab..d21b7af73a0 100644 --- a/application/src/main/java/com/yahoo/application/container/SynchronousRequestResponseHandler.java +++ b/application/src/main/java/com/yahoo/application/container/SynchronousRequestResponseHandler.java @@ -11,7 +11,6 @@ import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.service.CurrentContainer; import com.yahoo.jdisc.test.TestDriver; -import javax.annotation.concurrent.ThreadSafe; import java.net.URI; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -22,7 +21,6 @@ import java.util.concurrent.CountDownLatch; /** * @author Einar M R Rosenvinge */ -@ThreadSafe @Beta final class SynchronousRequestResponseHandler { @@ -121,7 +119,7 @@ final class SynchronousRequestResponseHandler { } } - @ThreadSafe + private static class BlockingResponseHandler implements ResponseHandler, ContentChannel { private volatile com.yahoo.jdisc.Response discResponse = null; private CountDownLatch closedLatch = new CountDownLatch(1); diff --git a/application/src/main/java/com/yahoo/application/container/handler/Headers.java b/application/src/main/java/com/yahoo/application/container/handler/Headers.java index 3e0c148bcc7..03b31e38659 100644 --- a/application/src/main/java/com/yahoo/application/container/handler/Headers.java +++ b/application/src/main/java/com/yahoo/application/container/handler/Headers.java @@ -4,7 +4,6 @@ package com.yahoo.application.container.handler; import com.yahoo.api.annotations.Beta; import com.yahoo.jdisc.HeaderFields; -import javax.annotation.concurrent.NotThreadSafe; import java.util.Collection; import java.util.List; import java.util.Map; @@ -18,7 +17,6 @@ import java.util.Set; * @author Einar M R Rosenvinge * @author Simon Thoresen Hult */ -@NotThreadSafe @Beta public class Headers implements Map<String, List<String>> { private final HeaderFields h = new HeaderFields(); diff --git a/application/src/main/java/com/yahoo/application/container/handler/Request.java b/application/src/main/java/com/yahoo/application/container/handler/Request.java index 1606af498fa..117737b8fee 100644 --- a/application/src/main/java/com/yahoo/application/container/handler/Request.java +++ b/application/src/main/java/com/yahoo/application/container/handler/Request.java @@ -2,7 +2,6 @@ package com.yahoo.application.container.handler; import com.yahoo.api.annotations.Beta; -import net.jcip.annotations.Immutable; import java.nio.charset.StandardCharsets; import java.security.Principal; @@ -16,7 +15,6 @@ import java.util.concurrent.ConcurrentHashMap; * @author Einar M R Rosenvinge * @see Response */ -@Immutable @Beta public class Request { diff --git a/application/src/main/java/com/yahoo/application/container/handler/Response.java b/application/src/main/java/com/yahoo/application/container/handler/Response.java index 91632068328..88f42a429ee 100644 --- a/application/src/main/java/com/yahoo/application/container/handler/Response.java +++ b/application/src/main/java/com/yahoo/application/container/handler/Response.java @@ -4,7 +4,6 @@ package com.yahoo.application.container.handler; import com.yahoo.api.annotations.Beta; import com.yahoo.jdisc.http.HttpHeaders; import com.yahoo.text.Utf8; -import net.jcip.annotations.Immutable; import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; @@ -20,7 +19,6 @@ import java.util.regex.Pattern; * @author Einar M R Rosenvinge * @see Request */ -@Immutable @Beta public class Response { diff --git a/application/src/test/app-packages/searcher-app/components/com.yahoo.vespatest.ExtraHitSearcher.jar b/application/src/test/app-packages/searcher-app/components/com.yahoo.vespatest.ExtraHitSearcher.jar Binary files differindex 90845ab51f9..77f06e869ca 100644 --- a/application/src/test/app-packages/searcher-app/components/com.yahoo.vespatest.ExtraHitSearcher.jar +++ b/application/src/test/app-packages/searcher-app/components/com.yahoo.vespatest.ExtraHitSearcher.jar diff --git a/application/src/test/java/com/yahoo/application/ApplicationTest.java b/application/src/test/java/com/yahoo/application/ApplicationTest.java index 86873033b8a..3d3476cd99b 100644 --- a/application/src/test/java/com/yahoo/application/ApplicationTest.java +++ b/application/src/test/java/com/yahoo/application/ApplicationTest.java @@ -263,36 +263,6 @@ public class ApplicationTest { } @Test - public void client() throws Exception { - try (ApplicationFacade app = new ApplicationFacade(Application.fromBuilder(new Application.Builder() - .documentType("test", new String(this.getClass().getResourceAsStream("/test.sd").readAllBytes(), StandardCharsets.UTF_8)) - .container("default", new Application.Builder.Container() - .client("mbus://*/*", MockClient.class) - .documentProcessor(MockDispatchDocproc.class) - )) - )) { - - Map<String, DocumentType> typeMap = app.application().getJDisc("jdisc").documentProcessing().getDocumentTypes(); - assertNotNull(typeMap); - - DocumentType docType = typeMap.get("test"); - Document doc = new Document(docType, "id:foo:test::bar"); - doc.setFieldValue("title", "hello"); - - assertEquals(DocumentProcessor.Progress.DONE, app.process(new DocumentPut(doc))); - - MockClient client = (MockClient) app.getClientById(MockClient.class.getName()); - assertNotNull(client); - assertEquals(1, client.getCounter()); - - MockDispatchDocproc docproc = (MockDispatchDocproc) app.getComponentById(MockDispatchDocproc.class.getName() + "@default"); - assertNotNull(docproc); - assertEquals(1, docproc.getResponses().size()); - assertEquals(200, docproc.getResponses().get(0).getStatus()); - } - } - - @Test public void file_distribution() { try (Application application = Application.fromApplicationPackage(new File("src/test/app-packages/filedistribution/"), Networking.disable)) { // Deployment succeeded diff --git a/application/src/test/java/com/yahoo/application/container/ContainerTest.java b/application/src/test/java/com/yahoo/application/container/ContainerTest.java index f36237066b5..fda8e0c597a 100644 --- a/application/src/test/java/com/yahoo/application/container/ContainerTest.java +++ b/application/src/test/java/com/yahoo/application/container/ContainerTest.java @@ -64,11 +64,10 @@ public class ContainerTest { try (JDisc container = fromServicesXml("<services>" + // "<container version=\"1.0\" id=\"id1\" />" + // "<container version=\"1.0\" />" + // - "<jdisc version=\"1.0\"/>" + // "</services>", Networking.disable)) { fail("expected exception"); } catch (Exception e) { - assertTrue(e.getMessage().contains("container id='id1', container id='', jdisc id=''")); + assertTrue(e.getMessage().contains("container id='id1', container id=''")); } } diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml index 08521216736..d5e6b85f571 100644 --- a/athenz-identity-provider-service/pom.xml +++ b/athenz-identity-provider-service/pom.xml @@ -10,7 +10,7 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <dependencies> diff --git a/bundle-plugin-test/integration-test/pom.xml b/bundle-plugin-test/integration-test/pom.xml index 5f9192b064b..a97d30fbaec 100644 --- a/bundle-plugin-test/integration-test/pom.xml +++ b/bundle-plugin-test/integration-test/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa.bundle-plugin</groupId> <artifactId>bundle-plugin-test</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <artifactId>integration-test</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>jar</packaging> <dependencies> diff --git a/bundle-plugin-test/pom.xml b/bundle-plugin-test/pom.xml index 0500e69acb3..16dd1238f53 100644 --- a/bundle-plugin-test/pom.xml +++ b/bundle-plugin-test/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <groupId>com.yahoo.vespa.bundle-plugin</groupId> <artifactId>bundle-plugin-test</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>pom</packaging> <modules> <module>integration-test</module> diff --git a/bundle-plugin-test/test-bundles/artifact-version-for-exports-dep/pom.xml b/bundle-plugin-test/test-bundles/artifact-version-for-exports-dep/pom.xml index e4ef18d179d..6d8095c2153 100644 --- a/bundle-plugin-test/test-bundles/artifact-version-for-exports-dep/pom.xml +++ b/bundle-plugin-test/test-bundles/artifact-version-for-exports-dep/pom.xml @@ -8,7 +8,7 @@ <parent> <groupId>com.yahoo.vespa.bundle-plugin</groupId> <artifactId>test-bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <artifactId>artifact-version-for-exports-dep</artifactId> @@ -16,7 +16,7 @@ But version is set to the release version by build scripts before building. Then, the dependent bundle will not find the artifact. Skipping this step for a sub-module seems impossible with the maven-versions-plugin, and cumbersome with factorylib. --> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> </dependencies> diff --git a/bundle-plugin-test/test-bundles/artifact-version-for-exports/pom.xml b/bundle-plugin-test/test-bundles/artifact-version-for-exports/pom.xml index ede831adfbc..289d2f67eb0 100644 --- a/bundle-plugin-test/test-bundles/artifact-version-for-exports/pom.xml +++ b/bundle-plugin-test/test-bundles/artifact-version-for-exports/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa.bundle-plugin</groupId> <artifactId>test-bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <artifactId>artifact-version-for-exports</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> diff --git a/bundle-plugin-test/test-bundles/main/pom.xml b/bundle-plugin-test/test-bundles/main/pom.xml index 8153d46bcb7..b5f8f7b9a6a 100644 --- a/bundle-plugin-test/test-bundles/main/pom.xml +++ b/bundle-plugin-test/test-bundles/main/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa.bundle-plugin</groupId> <artifactId>test-bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <artifactId>main</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> diff --git a/bundle-plugin-test/test-bundles/pom.xml b/bundle-plugin-test/test-bundles/pom.xml index faddeafc76d..3af10826adc 100644 --- a/bundle-plugin-test/test-bundles/pom.xml +++ b/bundle-plugin-test/test-bundles/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa.bundle-plugin</groupId> <artifactId>bundle-plugin-test</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <artifactId>test-bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>pom</packaging> <build> <plugins> diff --git a/bundle-plugin/pom.xml b/bundle-plugin/pom.xml index 7993dfecafd..be5d2d59de2 100644 --- a/bundle-plugin/pom.xml +++ b/bundle-plugin/pom.xml @@ -6,11 +6,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>bundle-plugin</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>maven-plugin</packaging> <name>${project.artifactId}</name> <description>Maven Plugin for creating OSGi bundles for the JDisc Container.</description> diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Analyze.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Analyze.java index 7c34539921b..a93f8cb87d5 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Analyze.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Analyze.java @@ -65,7 +65,7 @@ public class Analyze { } static AnnotationVisitor visitAnnotationDefault(ImportCollector collector) { - return new AnnotationVisitor(Opcodes.ASM7) { + return new AnnotationVisitor(Opcodes.ASM9) { @Override public void visit(String name, Object value) { } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java index c5522c4c96a..46a35b07ea7 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java @@ -32,7 +32,7 @@ class AnalyzeClassVisitor extends ClassVisitor implements ImportCollector { private final Optional<ArtifactVersion> defaultExportPackageVersion; AnalyzeClassVisitor(ArtifactVersion defaultExportPackageVersion) { - super(Opcodes.ASM7); + super(Opcodes.ASM9); this.defaultExportPackageVersion = Optional.ofNullable(defaultExportPackageVersion); } @@ -103,7 +103,7 @@ class AnalyzeClassVisitor extends ClassVisitor implements ImportCollector { } private AnnotationVisitor visitExportPackage() { - return new AnnotationVisitor(Opcodes.ASM7) { + return new AnnotationVisitor(Opcodes.ASM9) { private int major = defaultExportPackageVersion.map(ArtifactVersion::getMajorVersion) .orElse(defaultVersionValue("major")); private int minor = defaultExportPackageVersion.map(ArtifactVersion::getMinorVersion) diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeFieldVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeFieldVisitor.java index 051df41d62b..87a2b2eb941 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeFieldVisitor.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeFieldVisitor.java @@ -19,7 +19,7 @@ public class AnalyzeFieldVisitor extends FieldVisitor implements ImportCollector private final Set<String> imports = new HashSet<>(); public AnalyzeFieldVisitor(AnalyzeClassVisitor analyzeClassVisitor) { - super(Opcodes.ASM7); + super(Opcodes.ASM9); this.analyzeClassVisitor = analyzeClassVisitor; } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java index 7913f315cdd..cdf6e93abda 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java @@ -26,7 +26,7 @@ class AnalyzeMethodVisitor extends MethodVisitor implements ImportCollector { private final AnalyzeClassVisitor analyzeClassVisitor; AnalyzeMethodVisitor(AnalyzeClassVisitor analyzeClassVisitor) { - super(Opcodes.ASM7); + super(Opcodes.ASM9); this.analyzeClassVisitor = analyzeClassVisitor; } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeSignatureVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeSignatureVisitor.java index 61c37e99edf..deb08ee21fa 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeSignatureVisitor.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeSignatureVisitor.java @@ -18,7 +18,7 @@ class AnalyzeSignatureVisitor extends SignatureVisitor implements ImportCollecto private final Set<String> imports = new HashSet<>(); AnalyzeSignatureVisitor(AnalyzeClassVisitor analyzeClassVisitor) { - super(Opcodes.ASM7); + super(Opcodes.ASM9); this.analyzeClassVisitor = analyzeClassVisitor; } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AbstractGenerateOsgiManifestMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AbstractGenerateOsgiManifestMojo.java index 8816aa9f1c5..e058e0dfa6f 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AbstractGenerateOsgiManifestMojo.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AbstractGenerateOsgiManifestMojo.java @@ -62,6 +62,12 @@ abstract class AbstractGenerateOsgiManifestMojo extends AbstractMojo { @Parameter(alias = "Import-Package") String importPackage; + // Spifly specific headers for java.util.ServiceLoader support in OSGi context. For internal use only. + @Parameter(alias = "SPI-Provider") + String spiProvider; + @Parameter(alias = "SPI-Consumer") + String spiConsumer; + Map<String, String> generateManifestContent( Collection<Artifact> jarArtifactsToInclude, Map<String, ImportPackages.Import> calculatedImports, @@ -90,7 +96,8 @@ abstract class AbstractGenerateOsgiManifestMojo extends AbstractMojo { addIfNotEmpty(ret, "Bundle-ClassPath", bundleClassPath(jarArtifactsToInclude)); addIfNotEmpty(ret, "Import-Package", importPackage); addIfNotEmpty(ret, "Export-Package", exportPackage); - + addIfNotEmpty(ret, "SPI-Provider", spiProvider); + addIfNotEmpty(ret, "SPI-Consumer", spiConsumer); return ret; } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java index 1c061ca49a2..f9f56a31b20 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.plugin.classanalysis; +import com.google.common.collect.ImmutableList; import com.yahoo.container.plugin.classanalysis.sampleclasses.Base; import com.yahoo.container.plugin.classanalysis.sampleclasses.ClassAnnotation; import com.yahoo.container.plugin.classanalysis.sampleclasses.InvisibleAnnotation; @@ -10,6 +11,8 @@ import com.yahoo.container.plugin.classanalysis.sampleclasses.InvisibleDummyAnno import com.yahoo.container.plugin.classanalysis.sampleclasses.Fields; import com.yahoo.container.plugin.classanalysis.sampleclasses.Interface1; import com.yahoo.container.plugin.classanalysis.sampleclasses.Interface2; +import com.yahoo.container.plugin.classanalysis.sampleclasses.RecordWithOverride; +import com.yahoo.container.plugin.classanalysis.sampleclasses.SwitchStatement; import com.yahoo.container.plugin.classanalysis.sampleclasses.MethodAnnotation; import com.yahoo.container.plugin.classanalysis.sampleclasses.MethodInvocation; import com.yahoo.osgi.annotation.ExportPackage; @@ -169,4 +172,23 @@ public class AnalyzeClassTest { assertTrue(Analyze.analyzeClass(classFile("class/Utf8")).getReferencedClasses() .contains("org.aspectj.weaver.MethodDeclarationLineNumber")); } + + @Test + public void switch_statements_are_analyzed() { + var referencedClasses = analyzeClass(SwitchStatement.class).getReferencedClasses(); + assertTrue(referencedClasses.contains(name(ImmutableList.class))); + assertTrue(referencedClasses.contains(name(IllegalArgumentException.class))); + } + + @Test + public void records_are_analyzed() { + var referencedClasses = analyzeClass(RecordWithOverride.class).getReferencedClasses(); + assertTrue(referencedClasses.containsAll(List.of( + name(java.util.List.class), + name(Byte.class), + name(ImmutableList.class), + name(IllegalArgumentException.class) + ))); + + } } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/RecordWithOverride.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/RecordWithOverride.java new file mode 100644 index 00000000000..299a9930215 --- /dev/null +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/RecordWithOverride.java @@ -0,0 +1,20 @@ +package com.yahoo.container.plugin.classanalysis.sampleclasses; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * Input for class analysis test verifying Java 15 records, + * plus Java 16 pattern matching for instanceof. + * + * @author gjoranv + */ +public record RecordWithOverride(List<Byte> list) { + + public RecordWithOverride { + if (list instanceof ImmutableList<Byte> l) { + throw new IllegalArgumentException(l.toString()); + } + } +} diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/SwitchStatement.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/SwitchStatement.java new file mode 100644 index 00000000000..5dbe90e2b58 --- /dev/null +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/SwitchStatement.java @@ -0,0 +1,23 @@ +package com.yahoo.container.plugin.classanalysis.sampleclasses; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.Collection; + +/** + * Input for class analysis test verifying Java 14 switch statement. + * + * @author gjoranv + */ +public class SwitchStatement { + + void switchStatement() throws Exception{ + String foo = ""; + Collection<?> c = switch (foo) { + case "list" -> ImmutableList.of(); + case "set" -> ImmutableSet.of(); + default -> throw new IllegalArgumentException(); + }; + } +} diff --git a/client/go/cmd/query.go b/client/go/cmd/query.go index 7abf17d5358..82ff5185cbd 100644 --- a/client/go/cmd/query.go +++ b/client/go/cmd/query.go @@ -105,10 +105,13 @@ func query(cli *CLI, arguments []string, timeoutSecs int, curl bool) error { } func splitArg(argument string) (string, string) { - equalsIndex := strings.Index(argument, "=") - if equalsIndex < 1 { + parts := strings.SplitN(argument, "=", 2) + if len(parts) < 2 { + return "yql", parts[0] + } + if strings.HasPrefix(strings.ToLower(parts[0]), "select ") { + // A query containing '=' return "yql", argument - } else { - return argument[0:equalsIndex], argument[equalsIndex+1:] } + return parts[0], parts[1] } diff --git a/client/go/cmd/query_test.go b/client/go/cmd/query_test.go index ab03058d079..d024ca5189e 100644 --- a/client/go/cmd/query_test.go +++ b/client/go/cmd/query_test.go @@ -39,8 +39,20 @@ func TestQueryNonJsonResult(t *testing.T) { func TestQueryWithMultipleParameters(t *testing.T) { assertQuery(t, - "?hits=5&timeout=20s&yql=select+from+sources+%2A+where+title+contains+%27foo%27", - "select from sources * where title contains 'foo'", "hits=5", "timeout=20s") + "?hits=5&timeout=20s&yql=select+from+sources+%2A+where+title+contains+%27foo%27+and+year+%3D+2000", + "select from sources * where title contains 'foo' and year = 2000", "hits=5", "timeout=20s") +} + +func TestQueryWithEquals(t *testing.T) { + assertQuery(t, + "?timeout=10s&yql=SELECT+from+sources+%2A+where+title+contains+%27foo%27+and+year+%3D+2000", + "SELECT from sources * where title contains 'foo' and year = 2000") +} + +func TestQuerySelect(t *testing.T) { + assertQuery(t, + "?hits=5&select=%7B%22select%22%3A%7B%22where%22%3A%7B%22contains%22%3A%5B%22title%22%2C%22a%22%5D%7D%7D%7D&timeout=10s", + `select={"select":{"where":{"contains":["title","a"]}}}`, "hits=5") } func TestQueryWithExplicitYqlParameter(t *testing.T) { diff --git a/client/pom.xml b/client/pom.xml index 190b0a4a73f..065cb2c4317 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -8,13 +8,13 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>client</artifactId> <packaging>jar</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <dependency> diff --git a/client/src/main/java/ai/vespa/client/dsl/Q.java b/client/src/main/java/ai/vespa/client/dsl/Q.java index 50e009b131d..cfc0121c3f3 100644 --- a/client/src/main/java/ai/vespa/client/dsl/Q.java +++ b/client/src/main/java/ai/vespa/client/dsl/Q.java @@ -169,12 +169,6 @@ public final class Q { return new WeakAnd(query); } - /** @deprecated use weakand(query) */ - @Deprecated // Remove on Vespa 8 - public static WeakAnd weakand(String ignored, Query query) { - return new WeakAnd(query); - } - /** * GeoLocation geo location * https://docs.vespa.ai/en/reference/query-language-reference.html#geolocation diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 99bef1e22e6..84ebba80584 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>cloud-tenant-base</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../cloud-tenant-base/pom.xml</relativePath> </parent> <artifactId>cloud-tenant-base-dependencies-enforcer</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>pom</packaging> <properties> @@ -21,6 +21,8 @@ <!-- MUST BE KEPT IN SYNC WITH parent/pom.xml --> <athenz.version>1.10.54</athenz.version> + <bouncycastle.version>1.68</bouncycastle.version> + <felix.version>7.0.1</felix.version> <httpclient5.version>5.1.2</httpclient5.version> <httpclient.version>4.5.13</httpclient.version> <httpcore.version>4.4.13</httpcore.version> @@ -32,14 +34,13 @@ <!-- ALL BELOW MUST BE KEPT IN SYNC WITH container-dependency-versions pom Copied here because vz-tenant-base does not have a parent. --> <aopalliance.version>1.0</aopalliance.version> - <bouncycastle.version>1.68</bouncycastle.version> - <felix.version>7.0.1</felix.version> - <felix.log.version>1.0.1</felix.log.version> - <findbugs.version>1.3.9</findbugs.version> - <guava.version>20.0</guava.version> - <guice.version>3.0</guice.version> + <guava.version>27.1-jre</guava.version> + <guice.version>4.2.3</guice.version> + <jackson2.version>2.13.3</jackson2.version> + <jackson-databind.version>${jackson2.version}</jackson-databind.version> <javax.inject.version>1</javax.inject.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> + <javax.ws.rs-api.version>2.0.1</javax.ws.rs-api.version> <jaxb.version>2.3.0</jaxb.version> <jetty.version>9.4.46.v20220331</jetty.version> <jetty-alpn.version>1.1.3.v20160715</jetty-alpn.version> @@ -47,16 +48,6 @@ <org.json.version>20090211</org.json.version> <slf4j.version>1.7.32</slf4j.version> <!-- WARNING: when updated, also update c.y.v.tenant:base pom --> <xml-apis.version>1.4.01</xml-apis.version> - - <hk2.version>2.5.0-b32</hk2.version> - <hk2.osgi-resource-locator.version>1.0.1</hk2.osgi-resource-locator.version> - <jackson2.version>2.13.3</jackson2.version> - <jackson-databind.version>${jackson2.version}</jackson-databind.version> - <javassist.version>3.20.0-GA</javassist.version> - <javax.annotation-api.version>1.2</javax.annotation-api.version> - <javax.validation-api.version>1.1.0.Final</javax.validation-api.version> - <javax.ws.rs-api.version>2.0.1</javax.ws.rs-api.version> - <jersey2.version>2.25</jersey2.version> </properties> <build> @@ -88,50 +79,21 @@ <include>com.fasterxml.jackson.datatype:jackson-datatype-jdk8:[${jackson2.version}]:jar:provided</include> <include>com.fasterxml.jackson.datatype:jackson-datatype-jsr310:[${jackson2.version}]:jar:provided</include> - <!-- Use version range for jax deps, because jersey and junit affect the versions. --> - <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:[2.5.4, ${jackson2.version}]:jar:provided</include> - <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:[2.5.4, ${jackson2.version}]:jar:provided</include> - <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations:[2.5.4, ${jackson2.version}]:jar:provided</include> - - <!-- guava with dependencies --> - <include>com.google.code.findbugs:jsr305:[${findbugs.version}]:jar:provided</include> + <!-- Guava with its internal dependencies --> <include>com.google.guava:guava:[${guava.version}]:jar:provided</include> + <include>com.google.errorprone:error_prone_annotations:[2.2.0]:jar:provided</include> + <include>com.google.guava:failureaccess:[1.0.1]:jar:provided</include> + <include>com.google.j2objc:j2objc-annotations:[1.1]:jar:provided</include> - <include>com.google.inject.extensions:guice-assistedinject:[${guice.version}]:jar:provided</include> - <include>com.google.inject.extensions:guice-multibindings:[${guice.version}]:jar:provided</include> <include>com.google.inject:guice:[${guice.version}]:jar:provided:no_aop</include> <include>com.sun.activation:javax.activation:[1.2.0]:jar:provided</include> <include>com.sun.xml.bind:jaxb-core:[${jaxb.version}]:jar:provided</include> <include>com.sun.xml.bind:jaxb-impl:[${jaxb.version}]:jar:provided</include> <include>commons-logging:commons-logging:[1.2]:jar:provided</include> - <include>javax.annotation:javax.annotation-api:[${javax.annotation-api.version}]:jar:provided</include> <include>javax.inject:javax.inject:[${javax.inject.version}]:jar:provided</include> <include>javax.servlet:javax.servlet-api:[${javax.servlet-api.version}]:jar:provided</include> - <include>javax.validation:validation-api:[${javax.validation-api.version}]:jar:provided</include> <include>javax.ws.rs:javax.ws.rs-api:[${javax.ws.rs-api.version}]:jar:provided</include> <include>javax.xml.bind:jaxb-api:[${jaxb.version}]:jar:provided</include> - <include>net.jcip:jcip-annotations:[1.0]:jar:provided</include> - <include>org.lz4:lz4-java:[${org.lz4.version}]:jar:provided</include> - <include>org.apache.felix:org.apache.felix.framework:[${felix.version}]:jar:provided</include> - <include>org.apache.felix:org.apache.felix.log:[${felix.log.version}]:jar:provided</include> - <include>org.apache.felix:org.apache.felix.main:[${felix.version}]:jar:provided</include> - <include>org.bouncycastle:bcpkix-jdk15on:[${bouncycastle.version}]:jar:provided</include> - <include>org.bouncycastle:bcprov-jdk15on:[${bouncycastle.version}]:jar:provided</include> - <include>org.eclipse.jetty:jetty-http:[${jetty.version}]:jar:provided</include> - <include>org.eclipse.jetty:jetty-io:[${jetty.version}]:jar:provided</include> - <include>org.eclipse.jetty:jetty-util:[${jetty.version}]:jar:provided</include> - <include>org.glassfish.hk2.external:aopalliance-repackaged:[${hk2.version}]:jar:provided</include> - <include>org.glassfish.hk2.external:javax.inject:[${hk2.version}]:jar:provided</include> - <include>org.glassfish.hk2:hk2-api:[${hk2.version}]:jar:provided</include> - <include>org.glassfish.hk2:hk2-locator:[${hk2.version}]:jar:provided</include> - <include>org.glassfish.hk2:hk2-utils:[${hk2.version}]:jar:provided</include> - <include>org.glassfish.hk2:osgi-resource-locator:[${hk2.osgi-resource-locator.version}]:jar:provided</include> - <include>org.glassfish.jersey.bundles.repackaged:jersey-guava:[${jersey2.version}]:jar:provided</include> - <include>org.glassfish.jersey.core:jersey-client:[${jersey2.version}]:jar:provided</include> - <include>org.glassfish.jersey.core:jersey-common:[${jersey2.version}]:jar:provided</include> - <include>org.glassfish.jersey.core:jersey-server:[${jersey2.version}]:jar:provided</include> - <include>org.javassist:javassist:[${javassist.version}]:jar:provided</include> - <include>org.json:json:[${org.json.version}]:jar:provided</include> <include>org.slf4j:jcl-over-slf4j:[${slf4j.version}]:jar:provided</include> <include>org.slf4j:log4j-over-slf4j:[${slf4j.version}]:jar:provided</include> <include>org.slf4j:slf4j-api:[${slf4j.version}]:jar:provided</include> @@ -174,10 +136,8 @@ <include>com.yahoo.vespa:searchcore:*:jar:provided</include> <include>com.yahoo.vespa:searchlib:*:jar:provided</include> <include>com.yahoo.vespa:security-utils:*:jar:provided</include> - <include>com.yahoo.vespa:statistics:*:jar:provided</include> <include>com.yahoo.vespa:vdslib:*:jar:provided</include> - <include>com.yahoo.vespa:vespa-http-client:*:jar:provided</include> - <include>com.yahoo.vespa:vespa_jersey2:*:pom:provided</include> + <include>com.yahoo.vespa:vespa-3party-bundles:*:pom:provided</include> <include>com.yahoo.vespa:vespaclient-container-plugin:*:jar:provided</include> <include>com.yahoo.vespa:vespajlib:*:jar:provided</include> <include>com.yahoo.vespa:vespalog:*:jar:provided</include> @@ -195,7 +155,6 @@ <include>com.yahoo.vespa:container-test:*:jar:test</include> <include>com.yahoo.vespa:hosted-api:*:jar:test</include> <include>com.yahoo.vespa:indexinglanguage:*:jar:test</include> - <include>com.yahoo.vespa:jdisc_jetty:*:jar:test</include> <include>com.yahoo.vespa:logd:*:jar:test</include> <include>com.yahoo.vespa:metrics-proxy:*:jar:test</include> <include>com.yahoo.vespa:metrics:*:jar:test</include> @@ -208,7 +167,7 @@ <include>com.yahoo.vespa:vespa-athenz:*:jar:test</include> <include>com.yahoo.vespa:vespa-feed-client:*:jar:test</include> <include>com.yahoo.vespa:vespa-feed-client-api:*:jar:test</include> - <include>com.yahoo.vespa:vespa_jersey2:*:pom:test</include> + <include>com.yahoo.vespa:vespa-3party-bundles:*:pom:test</include> <include>com.yahoo.vespa:vespaclient-core:*:jar:test</include> <include>com.yahoo.vespa:vsm:*:jar:test</include> @@ -247,6 +206,7 @@ <include>org.apache.commons:commons-exec:1.3:jar:test</include> <include>org.apache.commons:commons-compress:1.21:jar:test</include> <include>org.apache.commons:commons-math3:3.6.1:jar:test</include> + <include>org.apache.felix:org.apache.felix.framework:[${felix.version}]:jar:test</include> <include>org.apache.httpcomponents.client5:httpclient5:${httpclient5.version}:jar:test</include> <include>org.apache.httpcomponents.core5:httpcore5:${httpclient5.version}:jar:test</include> <include>org.apache.httpcomponents.core5:httpcore5-h2:${httpclient5.version}:jar:test</include> @@ -255,29 +215,35 @@ <include>org.apache.httpcomponents:httpmime:${httpclient.version}:jar:test</include> <include>org.apache.opennlp:opennlp-tools:1.9.3:jar:test</include> <include>org.apiguardian:apiguardian-api:1.1.0:jar:test</include> - <include>org.codehaus.woodstox:stax2-api:3.1.4:jar:test</include> + <include>org.bouncycastle:bcpkix-jdk15on:[${bouncycastle.version}]:jar:test</include> + <include>org.bouncycastle:bcprov-jdk15on:[${bouncycastle.version}]:jar:test</include> <include>org.codehaus.woodstox:stax2-api:3.1.4:jar:test</include> <include>org.eclipse.jetty.alpn:alpn-api:[${jetty-alpn.version}]:jar:test</include> <include>org.eclipse.jetty.http2:http2-common:[${jetty.version}]:jar:test</include> <include>org.eclipse.jetty.http2:http2-hpack:[${jetty.version}]:jar:test</include> <include>org.eclipse.jetty.http2:http2-server:[${jetty.version}]:jar:test</include> - <include>org.eclipse.jetty:jetty-alpn-server:[${jetty.version}]:jar:test</include> <include>org.eclipse.jetty:jetty-alpn-java-server:[${jetty.version}]:jar:test</include> + <include>org.eclipse.jetty:jetty-alpn-server:[${jetty.version}]:jar:test</include> <include>org.eclipse.jetty:jetty-client:[${jetty.version}]:jar:test</include> <include>org.eclipse.jetty:jetty-continuation:[${jetty.version}]:jar:test</include> + <include>org.eclipse.jetty:jetty-http:[${jetty.version}]:jar:test</include> + <include>org.eclipse.jetty:jetty-io:[${jetty.version}]:jar:test</include> <include>org.eclipse.jetty:jetty-jmx:[${jetty.version}]:jar:test</include> <include>org.eclipse.jetty:jetty-security:[${jetty.version}]:jar:test</include> <include>org.eclipse.jetty:jetty-server:[${jetty.version}]:jar:test</include> <include>org.eclipse.jetty:jetty-servlet:[${jetty.version}]:jar:test</include> <include>org.eclipse.jetty:jetty-servlets:[${jetty.version}]:jar:test</include> <include>org.eclipse.jetty:jetty-util-ajax:[${jetty.version}]:jar:test</include> + <include>org.eclipse.jetty:jetty-util:[${jetty.version}]:jar:test</include> <include>org.hamcrest:hamcrest-core:1.3:jar:test</include> <include>org.hdrhistogram:HdrHistogram:2.1.8:jar:test</include> + <include>org.json:json:20090211:jar:test</include> <include>org.junit.jupiter:junit-jupiter-api:[${junit5.version}]:jar:test</include> <include>org.junit.jupiter:junit-jupiter-engine:[${junit5.version}]:jar:test</include> <include>org.junit.platform:junit-platform-commons:[${junit5.platform.version}]:jar:test</include> <include>org.junit.platform:junit-platform-engine:[${junit5.platform.version}]:jar:test</include> <include>org.junit.vintage:junit-vintage-engine:[${junit5.version}]:jar:test</include> <include>org.kohsuke:libpam4j:1.11:jar:test</include> + <include>org.lz4:lz4-java:[${org.lz4.version}]:jar:test</include> <include>org.opentest4j:opentest4j:1.2.0:jar:test</include> <include>software.amazon.ion:ion-java:1.0.2:jar:test</include> <include>xerces:xercesImpl:2.12.1:jar:test</include> diff --git a/cloud-tenant-base/pom.xml b/cloud-tenant-base/pom.xml index bb4171bd096..64ed23b6c33 100644 --- a/cloud-tenant-base/pom.xml +++ b/cloud-tenant-base/pom.xml @@ -7,7 +7,7 @@ <artifactId>cloud-tenant-base</artifactId> <name>Vespa Cloud tenant base</name> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <description>Parent POM for all Vespa Cloud applications.</description> <url>https://github.com/vespa-engine</url> <packaging>pom</packaging> @@ -15,7 +15,7 @@ <parent> <artifactId>hosted-tenant-base</artifactId> <groupId>com.yahoo.vespa</groupId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../hosted-tenant-base/pom.xml</relativePath> </parent> diff --git a/cloud-tenant-cd/pom.xml b/cloud-tenant-cd/pom.xml index ec23337f82b..baccbab54f5 100644 --- a/cloud-tenant-cd/pom.xml +++ b/cloud-tenant-cd/pom.xml @@ -13,7 +13,7 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent</relativePath> </parent> diff --git a/clustercontroller-apps/pom.xml b/clustercontroller-apps/pom.xml index 4b90f3c6ca9..fa0f5c064ad 100644 --- a/clustercontroller-apps/pom.xml +++ b/clustercontroller-apps/pom.xml @@ -5,11 +5,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>clustercontroller-apps</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> diff --git a/clustercontroller-core/pom.xml b/clustercontroller-core/pom.xml index 7a3ccaf61f4..243b69bf1e2 100644 --- a/clustercontroller-core/pom.xml +++ b/clustercontroller-core/pom.xml @@ -5,11 +5,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>clustercontroller-core</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/clustercontroller-reindexer/pom.xml b/clustercontroller-reindexer/pom.xml index f39922e0379..42e746c1bae 100644 --- a/clustercontroller-reindexer/pom.xml +++ b/clustercontroller-reindexer/pom.xml @@ -6,7 +6,7 @@ <parent> <artifactId>parent</artifactId> <groupId>com.yahoo.vespa</groupId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <modelVersion>4.0.0</modelVersion> diff --git a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java index 94b65a01e57..d5324a3f0b8 100644 --- a/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java +++ b/clustercontroller-reindexer/src/main/java/ai/vespa/reindexing/ReindexingCurator.java @@ -135,7 +135,7 @@ public class ReindexingCurator implements Closeable { private Reindexing deserialize(byte[] json) { return new Reindexing(SlimeUtils.entriesStream(SlimeUtils.jsonToSlimeOrThrow(json).get().field(STATUS)) - .filter(object -> require(TYPE, object, field -> types.hasDataType(field.asString()))) // Forget unknown documents. + .filter(object -> require(TYPE, object, field -> types.hasDocumentType(field.asString()))) // Forget unknown documents. .collect(toUnmodifiableMap(object -> require(TYPE, object, field -> types.getDocumentType(field.asString())), object -> new Status(require(STARTED_MILLIS, object, field -> Instant.ofEpochMilli(field.asLong())), get(ENDED_MILLIS, object, field -> Instant.ofEpochMilli(field.asLong())), diff --git a/clustercontroller-utils/pom.xml b/clustercontroller-utils/pom.xml index bf97dbd79ae..03a368ed667 100644 --- a/clustercontroller-utils/pom.xml +++ b/clustercontroller-utils/pom.xml @@ -6,11 +6,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>clustercontroller-utils</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> diff --git a/component/pom.xml b/component/pom.xml index d4d8400f397..a4330957069 100755 --- a/component/pom.xml +++ b/component/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>component</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <dependency> <groupId>junit</groupId> @@ -44,12 +44,6 @@ <scope>provided</scope> </dependency> <dependency> - <!-- Needed by guava, which has an optional dependency on jsr305. - Not optional here, to avoid the need to add it to a lot of dependent modules. --> - <groupId>com.google.code.findbugs</groupId> - <artifactId>jsr305</artifactId> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>config-lib</artifactId> <version>${project.version}</version> diff --git a/config-application-package/pom.xml b/config-application-package/pom.xml index 869e161f57d..3d471f1c94b 100644 --- a/config-application-package/pom.xml +++ b/config-application-package/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-application-package</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 9e72efab07b..826cf85798a 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -166,12 +166,6 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { } @Override - @SuppressWarnings("deprecation") - public String getApplicationName() { - return metaData.getApplicationId().application().value(); - } - - @Override public ApplicationId getApplicationId() { return metaData.getApplicationId(); } @Override diff --git a/config-bundle/pom.xml b/config-bundle/pom.xml index e8409bc1bbe..50205f9fe71 100644 --- a/config-bundle/pom.xml +++ b/config-bundle/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-bundle</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/config-class-plugin/pom.xml b/config-class-plugin/pom.xml index 50471514cc0..26dac276ea2 100644 --- a/config-class-plugin/pom.xml +++ b/config-class-plugin/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-class-plugin</artifactId> <packaging>maven-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>config-class-plugin (Vespa ConfigGen Plugin)</name> <dependencies> <dependency> diff --git a/config-lib/abi-spec.json b/config-lib/abi-spec.json index dd4cf995f64..45290714777 100644 --- a/config-lib/abi-spec.json +++ b/config-lib/abi-spec.json @@ -102,9 +102,7 @@ "public static java.lang.String getDefName(java.lang.Class)", "public static java.lang.String getDefNamespace(java.lang.Class)", "public static java.util.List serialize(com.yahoo.config.InnerNode)", - "public static void serialize(com.yahoo.config.InnerNode, com.yahoo.config.Serializer)", - "public java.lang.String getConfigMd5()", - "public void setConfigMd5(java.lang.String)" + "public static void serialize(com.yahoo.config.InnerNode, com.yahoo.config.Serializer)" ], "fields": [] }, @@ -202,8 +200,8 @@ "public void postInitialize(java.lang.String)", "public boolean equals(java.lang.Object)", "public int hashCode()", - "protected java.util.Map getChildren()", - "protected java.util.Map getChildrenWithVectorsFlattened()", + "protected final java.util.Map getChildren()", + "protected final java.util.Map getChildrenWithVectorsFlattened()", "protected static java.util.Map getAllDescendantLeafNodes(com.yahoo.config.InnerNode)" ], "fields": [] diff --git a/config-lib/pom.xml b/config-lib/pom.xml index 7c13bd22227..ea72afb568d 100644 --- a/config-lib/pom.xml +++ b/config-lib/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-lib</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>${project.artifactId}</name> <dependencies> <dependency> diff --git a/config-lib/src/main/java/com/yahoo/config/ConfigInstance.java b/config-lib/src/main/java/com/yahoo/config/ConfigInstance.java index 5f06bc27dec..f37ed8a99d0 100644 --- a/config-lib/src/main/java/com/yahoo/config/ConfigInstance.java +++ b/config-lib/src/main/java/com/yahoo/config/ConfigInstance.java @@ -36,8 +36,6 @@ public abstract class ConfigInstance extends InnerNode { public interface Producer {} - private String configMd5 = ""; - @SuppressWarnings("unused") // Used by reflection from ConfigInstanceUtil String configId; @@ -110,24 +108,6 @@ public abstract class ConfigInstance extends InnerNode { } } - /** - * @deprecated do not use - */ - @Deprecated - // TODO: Remove in Vespa 8 - public String getConfigMd5() { - return configMd5; - } - - /** - * @deprecated do not use - */ - @Deprecated - // TODO: Remove in Vespa 8 - public void setConfigMd5(String configMd5) { - this.configMd5 = configMd5; - } - private static String getStaticStringField(Class<?> type, String fieldName) { try { return (String) type.getField(fieldName).get(null); diff --git a/config-lib/src/main/java/com/yahoo/config/InnerNode.java b/config-lib/src/main/java/com/yahoo/config/InnerNode.java index 597a734e6a3..610221094d2 100644 --- a/config-lib/src/main/java/com/yahoo/config/InnerNode.java +++ b/config-lib/src/main/java/com/yahoo/config/InnerNode.java @@ -88,10 +88,9 @@ public abstract class InnerNode extends Node { return res; } - // TODO Make final before Vespa 8 as correct order is required - protected Map<String, Object> getChildren() { + protected final Map<String, Object> getChildren() { HashMap<String, Object> ret = new LinkedHashMap<>(); - Field fields[] = getClass().getDeclaredFields(); + Field[] fields = getClass().getDeclaredFields(); for (Field field : fields) { field.setAccessible(true); Object fieldValue; @@ -111,11 +110,10 @@ public abstract class InnerNode extends Node { /** * Returns a flat map of this node's direct children, including all NodeVectors' elements. * Keys are the node name, including index for vector elements, e.g. 'arr[0]'. - * TODO Make final before Vespa 8 as correct order is required */ @SuppressWarnings("unchecked") - protected Map<String, Node> getChildrenWithVectorsFlattened() { - HashMap<String, Node> ret = new LinkedHashMap<>(); + protected final Map<String, Node> getChildrenWithVectorsFlattened() { + Map<String, Node> ret = new LinkedHashMap<>(); Map<String, Object> children = getChildren(); for (Map.Entry<String, Object> childEntry : children.entrySet()) { @@ -132,13 +130,13 @@ public abstract class InnerNode extends Node { return ret; } - private static void addMapEntries(HashMap<String, Node> ret, String name, Map<String, Node> map) { + private static void addMapEntries(Map<String, Node> ret, String name, Map<String, Node> map) { for (Map.Entry<String, Node> entry : map.entrySet()) ret.put(name + "{" + entry.getKey() + "}", entry.getValue()); } - private static void addNodeVectorEntries(HashMap<String, Node> ret, String name, NodeVector<?> vector) { + private static void addNodeVectorEntries(Map<String, Node> ret, String name, NodeVector<?> vector) { for (int j = 0; j < vector.length(); j++) ret.put(name + "[" + j + "]", (Node) vector.get(j)); } @@ -161,7 +159,7 @@ public abstract class InnerNode extends Node { for (Map.Entry<String, Node> childEntry : children.entrySet()) { String name = childEntry.getKey(); String prefixedName = prefix + name; - name=name.replaceAll("\\[.*", ""); + name.replaceAll("\\[.*", ""); Node child = childEntry.getValue(); if (child instanceof LeafNode) { ret.put(prefixedName, (LeafNode<?>) child); diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index ea59c4409c3..47228cf7768 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -97,13 +97,11 @@ "abstract" ], "methods": [ - "public abstract java.lang.String getApplicationName()", "public abstract com.yahoo.config.provision.ApplicationId getApplicationId()", "public abstract java.io.Reader getServices()", "public abstract java.io.Reader getHosts()", "public java.util.List getUserIncludeDirs()", "public void validateIncludeDir(java.lang.String)", - "public java.util.Collection searchDefinitionContents()", "public abstract java.util.Map getAllExistingConfigDefs()", "public abstract java.util.List getFiles(com.yahoo.path.Path, java.lang.String, boolean)", "public java.util.List getFiles(com.yahoo.path.Path, java.lang.String)", @@ -128,7 +126,6 @@ "public java.util.Optional getAllocatedHosts()", "public java.util.Map getFileRegistries()", "public java.util.Map legacyOverrides()", - "public java.util.Collection getSearchDefinitions()", "public abstract java.util.Collection getSchemas()", "public com.yahoo.config.application.api.ApplicationPackage preprocess(com.yahoo.config.provision.Zone, com.yahoo.config.application.api.DeployLogger)" ], diff --git a/config-model-api/pom.xml b/config-model-api/pom.xml index 3e83a6dfddf..1d28965fa3c 100644 --- a/config-model-api/pom.xml +++ b/config-model-api/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-model-api</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index 0600a31e9c5..883b59be410 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -73,15 +73,6 @@ public interface ApplicationPackage { String PERMANENT_SERVICES = "permanent-services.xml"; - /** - * The name of the application package - * - * @return the name of the application (i.e the directory where the application package was deployed from) - * @deprecated do not use - */ - @Deprecated // TODO: Remove in Vespa 8 - String getApplicationName(); - ApplicationId getApplicationId(); /** @@ -112,15 +103,6 @@ public interface ApplicationPackage { } /** - * Readers for all the search definition files for this. - * @deprecated use {@link #getSchemas()} instead - * @return a list of readers for search definitions - */ - @Deprecated - // TODO: Remove in Vespa 8 - default Collection<NamedReader> searchDefinitionContents() { return getSchemas(); } - - /** * Returns all the config definitions available in this package as unparsed data. */ Map<ConfigDefinitionKey, UnparsedConfigDefinition> getAllExistingConfigDefs(); @@ -242,13 +224,6 @@ public interface ApplicationPackage { } /** - * @deprecated use {@link #getSchemas()} instead - */ - @Deprecated - // TODO: Remove in Vespa 8 - default Collection<NamedReader> getSearchDefinitions() { return getSchemas(); } - - /** * Readers for all the schema files. * @return a collection of readers for schemas */ diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index 96cafd4ec0f..44880d052b6 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-model-fat</artifactId> <packaging>bundle</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/config-model/CMakeLists.txt b/config-model/CMakeLists.txt index 8f59bb77ef2..cecbfe7a646 100644 --- a/config-model/CMakeLists.txt +++ b/config-model/CMakeLists.txt @@ -2,8 +2,8 @@ install_jar(config-model-jar-with-dependencies.jar) vespa_install_script(src/main/perl/vespa-deploy bin) -vespa_install_script(src/main/perl/vespa-expand-config.pl bin) install(DIRECTORY src/main/resources/schema DESTINATION share/vespa PATTERN ".gitignore" EXCLUDE) -install(DIRECTORY src/main/resources/schema DESTINATION share/vespa/schema/version/7.x PATTERN ".gitignore" EXCLUDE) - +install(DIRECTORY src/main/resources/schema DESTINATION share/vespa/schema/version/8.x PATTERN ".gitignore" EXCLUDE) +# TODO: Remove when Vespa > 8 and no apps are left on 7 in hosted Vespa +install(DIRECTORY src/main/resources/schema/version/7.x DESTINATION share/vespa/schema/version/7.x PATTERN ".gitignore" EXCLUDE) diff --git a/config-model/pom.xml b/config-model/pom.xml index e50c1177e7e..b08ff7f7f2e 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-model</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -41,11 +41,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava-testlib</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.assertj</groupId> <artifactId>assertj-core</artifactId> <scope>test</scope> @@ -201,12 +196,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>statistics</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>container-messagebus</artifactId> <version>${project.version}</version> <scope>provided</scope> @@ -304,6 +293,10 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> + </dependency> </dependencies> <build> diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index 19805adc637..6a474b84161 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -22,7 +22,6 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.DistributionConfig; -import com.yahoo.vespa.config.content.LoadTypeConfig; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; import com.yahoo.vespa.documentmodel.DocumentModel; @@ -33,7 +32,6 @@ import com.yahoo.vespa.model.PortsMeta; import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.admin.Admin; -import com.yahoo.vespa.model.clients.Clients; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; import com.yahoo.vespa.model.routing.Routing; @@ -213,15 +211,6 @@ public class ApplicationConfigProducerRoot extends AbstractConfigProducer<Abstra } @Override - public void getConfig(LoadTypeConfig.Builder builder) { - VespaModel model = (VespaModel) getRoot(); - Clients clients = model.getClients(); - if (clients != null) { - clients.getConfig(builder); - } - } - - @Override public void getConfig(ClusterListConfig.Builder builder) { VespaModel model = (VespaModel) getRoot(); for (ContentCluster cluster : model.getContentClusters().values()) { diff --git a/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java index f8632a6b187..22e505b066c 100644 --- a/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java @@ -4,7 +4,6 @@ package com.yahoo.config.model; import com.yahoo.cloud.config.ApplicationIdConfig; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.vespa.config.content.DistributionConfig; -import com.yahoo.vespa.config.content.LoadTypeConfig; import com.yahoo.cloud.config.log.LogdConfig; import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.cloud.config.ClusterListConfig; @@ -30,7 +29,6 @@ public interface CommonConfigsProducer extends DocumentmanagerConfig.Producer, LogdConfig.Producer, SlobroksConfig.Producer, ZookeepersConfig.Producer, - LoadTypeConfig.Producer, ClusterListConfig.Producer, DistributionConfig.Producer, AllClustersBucketSpacesConfig.Producer, diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java index 8985dac91aa..01b66057038 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java @@ -16,7 +16,6 @@ import com.yahoo.path.Path; import com.yahoo.text.XML; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.builder.VespaModelBuilder; -import com.yahoo.vespa.model.clients.Clients; import com.yahoo.vespa.model.content.Content; import com.yahoo.vespa.model.routing.Routing; import org.w3c.dom.Document; @@ -244,15 +243,6 @@ public class ConfigModelRepo implements ConfigModelRepoAdder, Serializable, Iter return modelsOfModelClass; } - public Clients getClients() { - for (ConfigModel m : configModels) { - if (m instanceof Clients) { - return (Clients)m; - } - } - return null; - } - public Routing getRouting() { for (ConfigModel m : configModels) { if (m instanceof Routing) { diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 11d84884194..4be664034dd 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -144,7 +144,7 @@ public class DeployState implements ConfigDefinitionStore { this.zone = zone; this.queryProfiles = queryProfiles; // TODO: Remove this by seeing how pagetemplates are propagated this.semanticRules = semanticRules; // TODO: Remove this by seeing how pagetemplates are propagated - this.importedModels = importMlModels(applicationPackage, modelImporters, deployLogger, executor); + this.importedModels = importMlModels(applicationPackage, modelImporters, executor); this.validationOverrides = applicationPackage.getValidationOverrides().map(ValidationOverrides::fromXml) .orElse(ValidationOverrides.empty); @@ -210,15 +210,11 @@ public class DeployState implements ConfigDefinitionStore { private static ImportedMlModels importMlModels(ApplicationPackage applicationPackage, Collection<MlModelImporter> modelImporters, - DeployLogger deployLogger, ExecutorService executor) { File importFrom = applicationPackage.getFileReference(ApplicationPackage.MODELS_DIR); ImportedMlModels importedModels = new ImportedMlModels(importFrom, executor, modelImporters); - for (var entry : importedModels.getSkippedModels().entrySet()) { - // TODO: Vespa 8: Throw IllegalArgumentException instead - deployLogger.logApplicationPackage(Level.WARNING, "Skipping import of model " + entry.getKey() + " as an exception " + - "occurred during import: " + entry.getValue()); - } + for (var entry : importedModels.getSkippedModels().entrySet()) + throw new IllegalArgumentException("Could not import model '" + entry.getKey() + "': " + entry.getValue()); return importedModels; } diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java index 23aa7e80d74..b86230c97e6 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java @@ -6,7 +6,6 @@ import com.yahoo.config.ConfigInstance; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.subscription.ConfigInstanceUtil; -import java.util.logging.Level; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ConfigPayloadBuilder; @@ -19,7 +18,6 @@ import com.yahoo.vespa.model.SimpleConfigProducer; import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.utils.FreezableMap; - import java.io.PrintStream; import java.io.Serializable; import java.util.ArrayList; @@ -27,6 +25,7 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -233,7 +232,6 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce return didApply; } - @SuppressWarnings("removal") // TODO Vespa 8: remove private void applyUserConfig(ConfigInstance.Builder builder, ConfigPayloadBuilder payloadBuilder) { ConfigInstance.Builder override; if (builder instanceof GenericConfig.GenericConfigBuilder) { diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java index 5a6a8e86e6a..acad7532404 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java @@ -27,7 +27,7 @@ public class HostsXmlProvisioner implements HostProvisioner { @Override public HostSpec allocateHost(String alias) { - // Some special rules to allow no admin elements as well as jdisc element without nodes. + // Some special rules to allow no admin elements as well as container element without nodes. if (alias.equals(IMPLICIT_ADMIN_HOSTALIAS)) { if (hosts.asCollection().size() > 1) { throw new IllegalArgumentException("More than 1 host specified (" + hosts.asCollection().size() + ") and <admin> not specified"); diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index a6bcf6b0fd2..102023ba1fd 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.ProvisionLogger; @@ -67,6 +68,8 @@ public class InMemoryProvisioner implements HostProvisioner { private Provisioned provisioned = new Provisioned(); + private Environment environment = Environment.prod; + /** Creates this with a number of nodes with resources 1, 3, 9, 1 */ public InMemoryProvisioner(int nodeCount, boolean sharedHosts) { this(nodeCount, defaultResources, sharedHosts); @@ -115,6 +118,12 @@ public class InMemoryProvisioner implements HostProvisioner { this.retiredHostNames = Set.of(retiredHostNames); } + /** May affect e.g. the number of nodes/cluster. */ + public InMemoryProvisioner setEnvironment(Environment environment) { + this.environment = environment; + return this; + } + private static Collection<Host> toHostInstances(String[] hostnames) { return Arrays.stream(hostnames).map(Host::new).collect(Collectors.toList()); } @@ -137,6 +146,10 @@ public class InMemoryProvisioner implements HostProvisioner { @Override public List<HostSpec> prepare(ClusterSpec cluster, Capacity requested, ProvisionLogger logger) { provisioned.add(cluster.id(), requested); + if (environment == Environment.dev) { + requested = requested.withLimits(requested.minResources().withNodes(1), + requested.maxResources().withNodes(1)); + } if (useMaxResources) return prepare(cluster, requested.maxResources(), requested.isRequired(), requested.canFail()); else diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 8892de0a88d..abd7ba3be9f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -95,14 +95,8 @@ public class MockApplicationPackage implements ApplicationPackage { /** Returns the root of this application package relative to the current dir */ protected File root() { return root; } - @SuppressWarnings("deprecation") // not redundant @Override - public String getApplicationName() { - return "mock-application"; - } - - @Override - public ApplicationId getApplicationId() { return ApplicationId.from("default", getApplicationName(), "default"); } + public ApplicationId getApplicationId() { return ApplicationId.from("default", "mock-application", "default"); } @Override public Reader getServices() { diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index 7f15da3179f..8c644279281 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -91,25 +91,6 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp public DataType getContentStruct() { return contentStruct; } public Collection<NewDocumentType> getInherited() { return inherits.values(); } public NewDocumentType getInherited(Name inherited) { return inherits.get(inherited.getId()); } - public NewDocumentType removeInherited(Name inherited) { return inherits.remove(inherited.getId()); } - - /** - * Data type of the header fields of this and all inherited document types - * Used by DocumentGenMojo - * @return merged {@link StructDataType} - */ - public StructDataType allHeader() { - StructDataType ret = new StructDataType(contentStruct.getName()); - for (Field f : contentStruct.getFields()) { - ret.addField(f); - } - for (NewDocumentType inherited : getInherited()) { - for (Field f : ((StructDataType) inherited.getContentStruct()).getFields()) { - ret.addField(f); - } - } - return ret; - } @Override public Class<Document> getValueClass() { @@ -132,7 +113,7 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp return false; } - private boolean verifyInheritance(NewDocumentType inherited) { + private void verifyInheritance(NewDocumentType inherited) { for (Field f : getFields()) { Field inhF = inherited.getField(f.getName()); if (inhF != null && !inhF.equals(f)) { @@ -151,7 +132,6 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp } } } - return true; } public void inherit(NewDocumentType inherited) { @@ -213,9 +193,7 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp } public Collection<Field> getFields() { - Collection<Field> collection = new LinkedList<>(); - collection.addAll(contentStruct.getFields()); - return Collections.unmodifiableCollection(collection); + return contentStruct.getFields(); } @Override @@ -311,20 +289,6 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp } return null; } - public AnnotationType getAnnotationType(int id) { - AnnotationType a = annotations.getType(id); - if (a != null) { - return a; - } else { - for (NewDocumentType dt : getInherited()) { - a = dt.getAnnotationType(id); - if (a != null) { - return a; - } - } - } - return null; - } public NewDocumentType add(AnnotationType type) { annotations.register(type); @@ -386,7 +350,6 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp private NewDocumentReferenceDataType refToThis = null; - @SuppressWarnings("deprecation") public NewDocumentReferenceDataType getReferenceDataType() { if (refToThis == null) { refToThis = new NewDocumentReferenceDataType(this); diff --git a/config-model/src/main/java/com/yahoo/schema/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/schema/DocumentModelBuilder.java index 1a6134b410f..be3788a36c3 100644 --- a/config-model/src/main/java/com/yahoo/schema/DocumentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/schema/DocumentModelBuilder.java @@ -8,7 +8,6 @@ import com.yahoo.document.DocumentType; import com.yahoo.document.Field; import com.yahoo.document.MapDataType; import com.yahoo.document.StructDataType; -import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; @@ -190,11 +189,6 @@ public class DocumentModelBuilder { } } - private static String descT(DataType type) { - if (type == null) { return "<null>"; } - return "'" + type.getName() + "' [" + type.getId() + "] {"+type.getClass() + "}"; - } - private void addDocumentTypes(List<SDDocumentType> docList) { LinkedList<NewDocumentType> lst = new LinkedList<>(); for (SDDocumentType doc : docList) { @@ -232,9 +226,6 @@ public class DocumentModelBuilder { return replacements.get(type); } DataType original = type; - if (type instanceof TemporaryStructuredDataType) { - throw new IllegalArgumentException("Cannot handle temporary: " + type); - } if (type instanceof TemporaryUnknownType) { // must be a known struct or document type DataType other = repo.getDataType(type.getId()); @@ -252,10 +243,9 @@ public class DocumentModelBuilder { if (other == null || other == type) { throw new IllegalArgumentException("No replacement found for temporary type: " + type); } - if (other instanceof OwnedStructDataType) { + if (other instanceof OwnedStructDataType otherOwned) { var owned = (OwnedTemporaryType) type; String ownedBy = owned.getOwnerName(); - var otherOwned = (OwnedStructDataType) other; String otherOwnedBy = otherOwned.getOwnerName(); if (! ownedBy.equals(otherOwnedBy)) { throw new IllegalArgumentException("Wrong document for type: " + otherOwnedBy + " but expected " + ownedBy); @@ -270,19 +260,18 @@ public class DocumentModelBuilder { type = other; } else if (type != DataType.DOCUMENT) { throw new IllegalArgumentException - ("Can not handle nested document definitions. Undefined document type: " + type.toString()); + ("Can not handle nested document definitions. Undefined document type: " + type); } } else if (type instanceof NewDocumentType) { DataType other = getDocumentType(docs, type.getName()); if (other != null) { type = other; } - } else if (type instanceof StructDataType) { + } else if (type instanceof StructDataType sdt) { // trick avoids infinite recursion: var old = replacements.put(original, type); assert(old == null); - StructDataType dt = (StructDataType) type; - for (com.yahoo.document.Field field : dt.getFields()) { + for (com.yahoo.document.Field field : sdt.getFields()) { var ft = field.getDataType(); var newft = resolveTemporariesRecurse(ft, repo, docs, replacements); if (ft != newft) { @@ -293,37 +282,33 @@ public class DocumentModelBuilder { old = replacements.remove(original); assert(old == type); } - else if (type instanceof MapDataType) { - MapDataType t = (MapDataType) type; - var old_kt = t.getKeyType(); - var old_vt = t.getValueType(); + else if (type instanceof MapDataType mdt) { + var old_kt = mdt.getKeyType(); + var old_vt = mdt.getValueType(); var kt = resolveTemporariesRecurse(old_kt, repo, docs, replacements); var vt = resolveTemporariesRecurse(old_vt, repo, docs, replacements); if (kt != old_kt || vt != old_vt) { - type = new MapDataType(kt, vt, t.getId()); + type = new MapDataType(kt, vt, mdt.getId()); } } - else if (type instanceof ArrayDataType) { - ArrayDataType t = (ArrayDataType) type; - var old_nt = t.getNestedType(); + else if (type instanceof ArrayDataType adt) { + var old_nt = adt.getNestedType(); var nt = resolveTemporariesRecurse(old_nt, repo, docs, replacements); if (nt != old_nt) { - type = new ArrayDataType(nt, t.getId()); + type = new ArrayDataType(nt, adt.getId()); } } - else if (type instanceof WeightedSetDataType) { - WeightedSetDataType t = (WeightedSetDataType) type; - var old_nt = t.getNestedType(); + else if (type instanceof WeightedSetDataType wdt) { + var old_nt = wdt.getNestedType(); var nt = resolveTemporariesRecurse(old_nt, repo, docs, replacements); if (nt != old_nt) { - boolean c = t.createIfNonExistent(); - boolean r = t.removeIfZero(); - type = new WeightedSetDataType(nt, c, r, t.getId()); + boolean c = wdt.createIfNonExistent(); + boolean r = wdt.removeIfZero(); + type = new WeightedSetDataType(nt, c, r, wdt.getId()); } } - else if (type instanceof NewDocumentReferenceDataType) { - var t = (NewDocumentReferenceDataType) type; - var doc = getDocumentType(docs, t.getTargetTypeName()); + else if (type instanceof NewDocumentReferenceDataType rft) { + var doc = getDocumentType(docs, rft.getTargetTypeName()); type = doc.getReferenceDataType(); } if (type != original) { @@ -382,7 +367,7 @@ public class DocumentModelBuilder { if (type.isStruct()) { handleStruct(type); } else { - throw new IllegalArgumentException("Data type '" + type.getName() + "' is not a struct => tostring='" + type.toString() + "'."); + throw new IllegalArgumentException("Data type '" + type.getName() + "' is not a struct => tostring='" + type + "'."); } } for (SDDocumentType type : sdoc.getTypes()) { @@ -454,19 +439,16 @@ public class DocumentModelBuilder { return; } inProgress.put(type, this); - if (type instanceof StructDataType) { - StructDataType tmp = (StructDataType) type; - extractDataTypesFromFields(tmp.getFieldsThisTypeOnly()); - } else if (type instanceof CollectionDataType) { - CollectionDataType tmp = (CollectionDataType) type; - extractNestedTypes(tmp.getNestedType()); - addType(tmp.getNestedType()); - } else if (type instanceof MapDataType) { - MapDataType tmp = (MapDataType) type; - extractNestedTypes(tmp.getKeyType()); - extractNestedTypes(tmp.getValueType()); - addType(tmp.getKeyType()); - addType(tmp.getValueType()); + if (type instanceof StructDataType sdt) { + extractDataTypesFromFields(sdt.getFieldsThisTypeOnly()); + } else if (type instanceof CollectionDataType cdt) { + extractNestedTypes(cdt.getNestedType()); + addType(cdt.getNestedType()); + } else if (type instanceof MapDataType mdt) { + extractNestedTypes(mdt.getKeyType()); + extractNestedTypes(mdt.getValueType()); + addType(mdt.getKeyType()); + addType(mdt.getValueType()); } else if (type instanceof TemporaryAnnotationReferenceDataType) { throw new IllegalArgumentException(type.toString()); } @@ -474,7 +456,7 @@ public class DocumentModelBuilder { private boolean testAddType(DataType type) { return internalAddType(type, true); } - private boolean addType(DataType type) { return internalAddType(type, false); } + private void addType(DataType type) { internalAddType(type, false); } private boolean internalAddType(DataType type, boolean dryRun) { DataType oldType = targetDt.getDataTypeRecursive(type.getId()); @@ -488,11 +470,9 @@ public class DocumentModelBuilder { return false; } if (targetDt.getDataType(type.getId()) == null) { - if ((oldType instanceof OwnedStructDataType) - && (type instanceof OwnedStructDataType)) + if ((oldType instanceof OwnedStructDataType oldOwned) + && (type instanceof OwnedStructDataType newOwned)) { - var oldOwned = (OwnedStructDataType) oldType; - var newOwned = (OwnedStructDataType) type; if (newOwned.getOwnerName().equals(targetDt.getName()) && ! oldOwned.getOwnerName().equals(targetDt.getName())) { @@ -503,10 +483,8 @@ public class DocumentModelBuilder { } } } - if ((type instanceof StructDataType) && (oldType instanceof StructDataType)) { - StructDataType s = (StructDataType) type; - StructDataType os = (StructDataType) oldType; - if ((os.getFieldCount() == 0) && (s.getFieldCount() > os.getFieldCount())) { + if ((type instanceof StructDataType sdt) && (oldType instanceof StructDataType oldSdt)) { + if ((oldSdt.getFieldCount() == 0) && (sdt.getFieldCount() > oldSdt.getFieldCount())) { if ( ! dryRun) { targetDt.replace(type); } @@ -528,8 +506,7 @@ public class DocumentModelBuilder { private DataType specialHandleAnnotationReferenceRecurse(String fieldName, DataType dataType) { - if (dataType instanceof TemporaryAnnotationReferenceDataType) { - TemporaryAnnotationReferenceDataType refType = (TemporaryAnnotationReferenceDataType)dataType; + if (dataType instanceof TemporaryAnnotationReferenceDataType refType) { if (refType.getId() != 0) { return null; } @@ -542,42 +519,38 @@ public class DocumentModelBuilder { addType(dataType); return dataType; } - else if (dataType instanceof MapDataType) { - MapDataType t = (MapDataType)dataType; - DataType valueType = specialHandleAnnotationReferenceRecurse(fieldName, t.getValueType()); + else if (dataType instanceof MapDataType mdt) { + DataType valueType = specialHandleAnnotationReferenceRecurse(fieldName, mdt.getValueType()); if (valueType == null) { return null; } - var mapType = new MapDataType(t.getKeyType(), valueType, t.getId()); + var mapType = new MapDataType(mdt.getKeyType(), valueType, mdt.getId()); addType(mapType); return mapType; } - else if (dataType instanceof ArrayDataType) { - ArrayDataType t = (ArrayDataType) dataType; - DataType nestedType = specialHandleAnnotationReferenceRecurse(fieldName, t.getNestedType()); + else if (dataType instanceof ArrayDataType adt) { + DataType nestedType = specialHandleAnnotationReferenceRecurse(fieldName, adt.getNestedType()); if (nestedType == null) { return null; } - var lstType = new ArrayDataType(nestedType, t.getId()); + var lstType = new ArrayDataType(nestedType, adt.getId()); addType(lstType); return lstType; } - else if (dataType instanceof WeightedSetDataType) { - WeightedSetDataType t = (WeightedSetDataType) dataType; - DataType nestedType = specialHandleAnnotationReferenceRecurse(fieldName, t.getNestedType()); + else if (dataType instanceof WeightedSetDataType wdt) { + DataType nestedType = specialHandleAnnotationReferenceRecurse(fieldName, wdt.getNestedType()); if (nestedType == null) { return null; } - boolean c = t.createIfNonExistent(); - boolean r = t.removeIfZero(); - var lstType = new WeightedSetDataType(nestedType, c, r, t.getId()); + boolean c = wdt.createIfNonExistent(); + boolean r = wdt.removeIfZero(); + var lstType = new WeightedSetDataType(nestedType, c, r, wdt.getId()); addType(lstType); return lstType; } return null; } - @SuppressWarnings("deprecation") private StructDataType handleStruct(SDDocumentType type) { if (type.isStruct()) { var st = type.getStruct(); @@ -614,7 +587,7 @@ public class DocumentModelBuilder { } private static Set<NewDocumentType.Name> convertDocumentReferencesToNames(Optional<DocumentReferences> documentReferences) { - if (!documentReferences.isPresent()) { + if (documentReferences.isEmpty()) { return Set.of(); } return documentReferences.get().referenceMap().values().stream() diff --git a/config-model/src/main/java/com/yahoo/schema/MapEvaluationTypeContext.java b/config-model/src/main/java/com/yahoo/schema/MapEvaluationTypeContext.java index c6c807f2dbb..cbf120e1ee0 100644 --- a/config-model/src/main/java/com/yahoo/schema/MapEvaluationTypeContext.java +++ b/config-model/src/main/java/com/yahoo/schema/MapEvaluationTypeContext.java @@ -267,7 +267,7 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement String modelConfigName = OnnxModelTransformer.getModelConfigName(reference); String modelOutput = OnnxModelTransformer.getModelOutput(reference, null); - reference = new Reference("onnxModel", new Arguments(new ReferenceNode(modelConfigName)), modelOutput); + reference = new Reference("onnx", new Arguments(new ReferenceNode(modelConfigName)), modelOutput); if ( ! featureTypes.containsKey(reference)) { throw new IllegalArgumentException("Missing onnx-model config for '" + configOrFileName + "'"); } diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java index 5479ecf323f..56786c733ec 100644 --- a/config-model/src/main/java/com/yahoo/schema/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java @@ -1089,11 +1089,11 @@ public class RankProfile implements Cloneable { Map<String, TensorType> inputTypes = resolveOnnxInputTypes(model, context); TensorType defaultOutputType = model.getTensorType(model.getDefaultOutput(), inputTypes); - context.setType(new Reference("onnxModel", args, null), defaultOutputType); + context.setType(new Reference("onnx", args, null), defaultOutputType); for (Map.Entry<String, String> mapping : model.getOutputMap().entrySet()) { TensorType type = model.getTensorType(mapping.getKey(), inputTypes); - context.setType(new Reference("onnxModel", args, mapping.getValue()), type); + context.setType(new Reference("onnx", args, mapping.getValue()), type); } } return context; diff --git a/config-model/src/main/java/com/yahoo/schema/Schema.java b/config-model/src/main/java/com/yahoo/schema/Schema.java index c733b6012f9..c1556bcfccc 100644 --- a/config-model/src/main/java/com/yahoo/schema/Schema.java +++ b/config-model/src/main/java/com/yahoo/schema/Schema.java @@ -187,8 +187,7 @@ public class Schema implements ImmutableSchema { */ public boolean isRawAsBase64() { if (rawAsBase64 != null) return rawAsBase64; - // TODO Vespa 8: flip default: - if (inherited.isEmpty()) return false; + if (inherited.isEmpty()) return true; return requireInherited().isRawAsBase64(); } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java index 99f73a75669..b378d127430 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.schema.derived; -import com.yahoo.config.subscription.ConfigInstanceUtil; import com.yahoo.schema.Schema; import com.yahoo.schema.document.Attribute; import com.yahoo.schema.document.Case; @@ -16,8 +15,6 @@ import com.yahoo.vespa.indexinglanguage.expressions.ToPositionExpression; import java.util.Collection; import java.util.Collections; import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -183,15 +180,6 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce return "attributes"; } - @SuppressWarnings("removal") // TODO Vespa 8: remove - private Map<String, AttributesConfig.Attribute.Builder> toMap(List<AttributesConfig.Attribute.Builder> ls) { - Map<String, AttributesConfig.Attribute.Builder> ret = new LinkedHashMap<>(); - for (AttributesConfig.Attribute.Builder builder : ls) { - ret.put((String) ConfigInstanceUtil.getField(builder, "name"), builder); - } - return ret; - } - @Override public void getConfig(AttributesConfig.Builder builder) { //TODO This is just to get some exporting tests to work, Should be undone and removed @@ -225,9 +213,6 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce if (attribute.isMutable()) { aaB.ismutable(true); } - if (attribute.isHuge()) { - aaB.huge(true); - } if (attribute.isPaged()) { aaB.paged(true); } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java index 7f6c824b979..0e58b4cdb3b 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java @@ -8,7 +8,6 @@ import com.yahoo.document.StructuredDataType; import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.schema.Schema; -import com.yahoo.schema.document.BooleanIndexDefinition; import com.yahoo.schema.document.FieldSet; import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.config.search.IndexschemaConfig; @@ -112,14 +111,13 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { @Override public void getConfig(IndexschemaConfig.Builder icB) { - for (int i = 0; i < fields.size(); ++i) { - IndexField f = fields.get(i); + for (IndexField f : fields) { IndexschemaConfig.Indexfield.Builder ifB = new IndexschemaConfig.Indexfield.Builder() .name(f.getName()) .datatype(IndexschemaConfig.Indexfield.Datatype.Enum.valueOf(f.getType())) .prefix(f.hasPrefix()) - .phrases(f.hasPhrases()) - .positions(f.hasPositions()) + .phrases(false) + .positions(true) .interleavedfeatures(f.useInterleavedFeatures()); if (!f.getCollectionType().equals("SINGLE")) { ifB.collectiontype(IndexschemaConfig.Indexfield.Collectiontype.Enum.valueOf(f.getCollectionType())); @@ -137,7 +135,6 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { } } - @SuppressWarnings("deprecation") static List<Field> flattenField(Field field) { DataType fieldType = field.getDataType(); if (fieldType.getPrimitiveType() != null){ @@ -172,14 +169,11 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { * Representation of an index field with name and data type. */ public static class IndexField { - private String name; - private Index.Type type; - private com.yahoo.schema.Index.Type sdType; // The index type in "user intent land" - private DataType sdFieldType; + private final String name; + private final Index.Type type; + private final DataType sdFieldType; private boolean prefix = false; - private boolean phrases = false; // TODO dead, but keep a while to ensure config compatibility? - private boolean positions = true;// TODO dead, but keep a while to ensure config compatibility? - private BooleanIndexDefinition boolIndex = null; + // Whether the posting lists of this index field should have interleaved features (num occs, field length) in document id stream. private boolean interleavedFeatures = false; @@ -193,11 +187,8 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { prefix = index.isPrefix(); interleavedFeatures = index.useInterleavedFeatures(); } - sdType = index.getType(); - boolIndex = index.getBooleanIndexDefiniton(); } public String getName() { return name; } - public Index.Type getRawType() { return type; } public String getType() { return type.equals(Index.Type.INT64) ? "INT64" : "STRING"; @@ -212,21 +203,7 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { : "SINGLE"; } public boolean hasPrefix() { return prefix; } - public boolean hasPhrases() { return phrases; } - public boolean hasPositions() { return positions; } public boolean useInterleavedFeatures() { return interleavedFeatures; } - - public BooleanIndexDefinition getBooleanIndexDefinition() { - return boolIndex; - } - - /** - * The user set index type - * @return the type - */ - public com.yahoo.schema.Index.Type getSdType() { - return sdType; - } } /** diff --git a/config-model/src/main/java/com/yahoo/schema/document/Attribute.java b/config-model/src/main/java/com/yahoo/schema/document/Attribute.java index 5593c3f8cdf..ca8279752ac 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/schema/document/Attribute.java @@ -56,7 +56,6 @@ public final class Attribute implements Cloneable, Serializable { private boolean fastRank = false; private boolean fastSearch = false; private boolean fastAccess = false; - private boolean huge = false; private boolean mutable = false; private boolean paged = false; private int arity = BooleanIndexDefinition.DEFAULT_ARITY; @@ -182,6 +181,9 @@ public final class Attribute implements Cloneable, Serializable { public boolean isPrefetch() { if (prefetch!=null) return prefetch.booleanValue(); + if (tensorType.isPresent()) { + return false; + } if (CollectionType.SINGLE.equals(collectionType)) { return true; } @@ -199,7 +201,6 @@ public final class Attribute implements Cloneable, Serializable { public boolean isFastSearch() { return fastSearch; } public boolean isFastRank() { return fastRank; } public boolean isFastAccess() { return fastAccess; } - public boolean isHuge() { return huge; } public boolean isPaged() { return paged; } public boolean isPosition() { return isPosition; } public boolean isMutable() { return mutable; } @@ -245,7 +246,6 @@ public final class Attribute implements Cloneable, Serializable { throw badGen.get(); } public void setFastSearch(boolean fastSearch) { this.fastSearch = fastSearch; } - public void setHuge(boolean huge) { this.huge = huge; } public void setPaged(boolean paged) { this.paged = paged; } public void setFastAccess(boolean fastAccess) { this.fastAccess = fastAccess; } public void setPosition(boolean position) { this.isPosition = position; } @@ -379,7 +379,7 @@ public final class Attribute implements Cloneable, Serializable { public int hashCode() { return Objects.hash( name, type, collectionType, sorting, dictionary, isPrefetch(), fastAccess, removeIfZero, - createIfNonExistent, isPosition, huge, mutable, paged, enableBitVectors, enableOnlyBitVector, + createIfNonExistent, isPosition, mutable, paged, enableBitVectors, enableOnlyBitVector, tensorType, referenceDocumentType, distanceMetric, hnswIndexParams); } @@ -402,7 +402,6 @@ public final class Attribute implements Cloneable, Serializable { if (this.enableBitVectors != other.enableBitVectors) return false; if (this.enableOnlyBitVector != other.enableOnlyBitVector) return false; if (this.fastSearch != other.fastSearch) return false; - if (this.huge != other.huge) return false; if (this.mutable != other.mutable) return false; if (this.paged != other.paged) return false; if (! this.sorting.equals(other.sorting)) return false; diff --git a/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java index d300bd08bfd..d19b62ce555 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java +++ b/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java @@ -15,8 +15,6 @@ import com.yahoo.schema.DocumentReferences; import com.yahoo.schema.FieldSets; import com.yahoo.schema.Schema; -import java.io.Serializable; -import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; import java.util.LinkedHashMap; @@ -33,7 +31,7 @@ import java.util.Set; * @author Thomas Gundersen * @author bratseth */ -public class SDDocumentType implements Cloneable, Serializable { +public class SDDocumentType implements Cloneable { public static final SDDocumentType VESPA_DOCUMENT; private final Map<DataTypeName, SDDocumentType> inheritedTypes = new LinkedHashMap<>(); @@ -45,7 +43,7 @@ public class SDDocumentType implements Cloneable, Serializable { // to ensure that the full Search and this SDDocumentType is built first. private FieldSets fieldSets; // Document references - private Optional<DocumentReferences> documentReferences = Optional.empty(); + private DocumentReferences documentReferences; private TemporaryImportedFields temporaryImportedFields; static { @@ -72,7 +70,7 @@ public class SDDocumentType implements Cloneable, Serializable { NewDocumentType.Name name = new NewDocumentType.Name(dt.getName()); if (getType(name) != null) throw new IllegalArgumentException("Data type '" + name + "' has already been used."); - if (name.getName() == docType.getName()) + if (name.getName().equals(docType.getName())) throw new IllegalArgumentException("Data type '" + name + "' can not have same name as its defining document."); ownedTypes.put(name, dt); return this; @@ -80,9 +78,6 @@ public class SDDocumentType implements Cloneable, Serializable { public final SDDocumentType getOwnedType(String name) { return getOwnedType(new NewDocumentType.Name(name)); } - public SDDocumentType getOwnedType(DataTypeName name) { - return getOwnedType(name.getName()); - } public SDDocumentType getOwnedType(NewDocumentType.Name name) { return ownedTypes.get(name); @@ -105,9 +100,8 @@ public class SDDocumentType implements Cloneable, Serializable { return type; } - public SDDocumentType addAnnotation(AnnotationType annotation) { + public void addAnnotation(AnnotationType annotation) { annotationTypes.register(annotation); - return this; } /** Returns all owned datatypes. */ @@ -117,15 +111,6 @@ public class SDDocumentType implements Cloneable, Serializable { public Map<String, AnnotationType> getAnnotations() { return annotationTypes.getTypes(); } public AnnotationType findAnnotation(String name) { return annotationTypes.getType(name); } - public Collection<SDDocumentType> getAllTypes() { - Collection<SDDocumentType> list = new ArrayList<>(); - list.addAll(getTypes()); - for (SDDocumentType inherited : inheritedTypes.values()) { - list.addAll(inherited.getAllTypes()); - } - return list; - } - public Map<NewDocumentType.Name, SDDocumentType> allTypes() { Map<NewDocumentType.Name, SDDocumentType> map = new LinkedHashMap<>(); for (SDDocumentType inherited : inheritedTypes.values()) @@ -219,8 +204,7 @@ public class SDDocumentType implements Cloneable, Serializable { String subFieldName = name.substring(name.indexOf(".")+1); Field f = docType.getField(superFieldName); if (f != null) { - if (f instanceof SDField) { - SDField superField = (SDField)f; + if (f instanceof SDField superField) { return superField.getStructField(subFieldName); } else { throw new IllegalArgumentException("Field " + f.getName() + " is not an SDField"); @@ -268,7 +252,7 @@ public class SDDocumentType implements Cloneable, Serializable { return field; } - public Field addField(String fName, DataType dataType, boolean header, int code) { + public Field addField(String fName, DataType dataType, int code) { SDField field = new SDField(this, fName, code, dataType); addField(field); return field; @@ -294,10 +278,6 @@ public class SDDocumentType implements Cloneable, Serializable { return new LinkedHashSet<>(map.values()); } - public Iterator<Field> fieldIterator() { - return fieldSet().iterator(); - } - /** Returns the number of fields in this only, not including inherited fields */ // TODO: Remove public int getFieldCount() { @@ -329,11 +309,11 @@ public class SDDocumentType implements Cloneable, Serializable { } public Optional<DocumentReferences> getDocumentReferences() { - return documentReferences; + return Optional.ofNullable(documentReferences); } public void setDocumentReferences(DocumentReferences documentReferences) { - this.documentReferences = Optional.of(documentReferences); + this.documentReferences = documentReferences; } public TemporaryImportedFields getTemporaryImportedFields() { diff --git a/config-model/src/main/java/com/yahoo/schema/document/SDField.java b/config-model/src/main/java/com/yahoo/schema/document/SDField.java index fbb2b2dc21f..943d6c6fc14 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/SDField.java @@ -33,9 +33,7 @@ import com.yahoo.vespa.indexinglanguage.parser.ParseException; import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.LinkedList; import java.util.List; -import java.util.ListIterator; import java.util.Map; import java.util.TreeMap; @@ -111,7 +109,7 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { private final Map<String,SDField> structFields = new java.util.LinkedHashMap<>(0); /** The document that this field was declared in, or null */ - private SDDocumentType repoDocType = null; + private final SDDocumentType repoDocType; /** The aliases declared for this field. May pertain to indexes or attributes */ private final Map<String, String> aliasToName = new HashMap<>(); @@ -263,7 +261,6 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { private boolean doneStructFields = false; - @SuppressWarnings("deprecation") private void actuallyMakeStructFields() { if (doneStructFields) return; if (getFirstStructOrMapRecursive() == null) { @@ -283,8 +280,7 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { structFields.put(fieldName, subField); }; - if (dataType instanceof MapDataType) { - MapDataType mdt = (MapDataType) dataType; + if (dataType instanceof MapDataType mdt) { supplyStructField.accept("key", mdt.getKeyType()); supplyStructField.accept("value", mdt.getValueType()); } else { @@ -311,8 +307,7 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { for (Field field : subType.fieldSet()) { supplyStructField.accept(field.getName(), field.getDataType()); } - } else if (dataType instanceof StructDataType) { - var sdt = (StructDataType) dataType; + } else if (dataType instanceof StructDataType sdt) { for (Field field : sdt.getFields()) { supplyStructField.accept(field.getName(), field.getDataType()); } @@ -325,13 +320,12 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { // populate struct fields with matching if (subType != null) { for (Field f : subType.fieldSet()) { - if (f instanceof SDField) { - SDField field = (SDField) f; - SDField subField = structFields.get(field.getName()); + if (f instanceof SDField sdField) { + SDField subField = structFields.get(sdField.getName()); if (subField != null) { // we just made this with a copy of our matching (see above) Matching subFieldMatching = subField.getMatching(); - subFieldMatching.merge(field.getMatching()); + subFieldMatching.merge(sdField.getMatching()); subField.setMatching(subFieldMatching); } } else { @@ -344,8 +338,6 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { doneStructFields = true; } - private Matching matchingForStructFields = null; - public void setId(int fieldId, DocumentType owner) { super.setId(fieldId, owner); idOverride = true; diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/OnnxModelTransformer.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/OnnxModelTransformer.java index 4c38c257602..8797deefcb6 100644 --- a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/OnnxModelTransformer.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/OnnxModelTransformer.java @@ -20,17 +20,16 @@ import java.util.List; /** * Transforms ONNX model features of the forms: * - * onnxModel(config_name) - * onnxModel(config_name).output - * onnxModel("path/to/model") - * onnxModel("path/to/model").output - * onnxModel("path/to/model", "path/to/output") - * onnxModel("path/to/model", "unused", "path/to/output") // signature is unused - * onnx(...) // same as with onnxModel, onnx is an alias of onnxModel + * onnx(config_name) + * onnx(config_name).output + * onnx("path/to/model") + * onnx("path/to/model").output + * onnx("path/to/model", "path/to/output") + * onnx("path/to/model", "unused", "path/to/output") // signature is unused * * To the format expected by the backend: * - * onnxModel(config_name).output + * onnx(config_name).output * * @author lesters */ @@ -84,7 +83,7 @@ public class OnnxModelTransformer extends ExpressionTransformer<RankProfileTrans throw new IllegalArgumentException(featureName + " argument '" + output + "' output not found in model '" + onnxModel.getFileName() + "'"); } - return new ReferenceNode("onnxModel", List.of(new ReferenceNode(modelConfigName)), output); + return new ReferenceNode("onnx", List.of(new ReferenceNode(modelConfigName)), output); } public static String getModelConfigName(Reference reference) { diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/TensorFlowFeatureConverter.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/TensorFlowFeatureConverter.java index fd19c97036e..731f09011c4 100644 --- a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/TensorFlowFeatureConverter.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/TensorFlowFeatureConverter.java @@ -23,9 +23,6 @@ import java.util.Map; */ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfileTransformContext> { - /** A cache of imported models indexed by model path. This avoids importing the same model multiple times. */ - private final Map<Path, ConvertedModel> convertedTensorFlowModels = new HashMap<>(); - public TensorFlowFeatureConverter() {} @Override @@ -39,28 +36,10 @@ public class TensorFlowFeatureConverter extends ExpressionTransformer<RankProfil } private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { - if ( ! feature.getName().equals("tensorflow")) return feature; - - try { - FeatureArguments arguments = asFeatureArguments(feature.getArguments()); - ConvertedModel convertedModel = - convertedTensorFlowModels.computeIfAbsent(arguments.path(), - path -> ConvertedModel.fromSourceOrStore(path, false, context)); - return convertedModel.expression(arguments, context); - } - catch (IllegalArgumentException | UncheckedIOException e) { - throw new IllegalArgumentException("Could not use tensorflow model from " + feature, e); + if ( ! feature.getName().equals("tensorflow")) { + return feature; } - } - - private FeatureArguments asFeatureArguments(Arguments arguments) { - if (arguments.isEmpty()) - throw new IllegalArgumentException("A tensorflow node must take an argument pointing to " + - "the tensorflow model directory under [application]/models"); - if (arguments.expressions().size() > 3) - throw new IllegalArgumentException("A tensorflow feature can have at most 3 arguments"); - - return new FeatureArguments(arguments); + throw new IllegalArgumentException("Import of TensorFlow models is no longer supported"); } } diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java index fa656b72530..3008dac4f29 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java @@ -76,7 +76,6 @@ public class ConvertParsedFields { field.addAttribute(attribute); } } - attribute.setHuge(parsed.getHuge()); attribute.setPaged(parsed.getPaged()); attribute.setFastSearch(parsed.getFastSearch()); if (parsed.getFastRank()) { diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedTypes.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedTypes.java index 9f1203ffc9f..efe1547f146 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedTypes.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedTypes.java @@ -28,11 +28,6 @@ public class ConvertParsedTypes { private final List<ParsedSchema> orderedInput; private final DocumentTypeManager docMan; - ConvertParsedTypes(List<ParsedSchema> input) { - this.orderedInput = input; - this.docMan = new DocumentTypeManager(); - } - public ConvertParsedTypes(List<ParsedSchema> input, DocumentTypeManager docMan) { this.orderedInput = input; this.docMan = docMan; @@ -46,9 +41,9 @@ public class ConvertParsedTypes { } } - private Map<String, DocumentType> documentsFromSchemas = new HashMap<>(); - private Map<String, StructDataType> structsFromSchemas = new HashMap<>(); - private Map<String, SDAnnotationType> annotationsFromSchemas = new HashMap<>(); + private final Map<String, DocumentType> documentsFromSchemas = new HashMap<>(); + private final Map<String, StructDataType> structsFromSchemas = new HashMap<>(); + private final Map<String, SDAnnotationType> annotationsFromSchemas = new HashMap<>(); private void startDataTypes() { for (var schema : orderedInput) { @@ -170,10 +165,10 @@ public class ConvertParsedTypes { for (var fieldset : schema.getFieldSets()) { fieldSets.put(fieldset.name(), fieldset.getFieldNames()); } - docToFill.addFieldSets(fieldSets); for (String inherit : doc.getInherited()) { docToFill.inherit(findDocFromSchemas(inherit)); } + docToFill.addFieldSets(fieldSets); } } @@ -297,7 +292,6 @@ public class ConvertParsedTypes { throw new IllegalArgumentException("unknown type named '" + name + "' in context "+context); } - @SuppressWarnings("deprecation") private void registerDataTypes() { for (DataType t : structsFromSchemas.values()) { docMan.register(t); diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedAttribute.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedAttribute.java index be8d20fbe93..de06436a1eb 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedAttribute.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedAttribute.java @@ -19,7 +19,6 @@ class ParsedAttribute extends ParsedBlock { private boolean enableFastAccess = false; private boolean enableFastRank = false; private boolean enableFastSearch = false; - private boolean enableHuge = false; private boolean enableMutable = false; private boolean enablePaged = false; private final Map<String, String> aliases = new LinkedHashMap<>(); @@ -38,7 +37,6 @@ class ParsedAttribute extends ParsedBlock { boolean getFastAccess() { return this.enableFastAccess; } boolean getFastRank() { return this.enableFastRank; } boolean getFastSearch() { return this.enableFastSearch; } - boolean getHuge() { return this.enableHuge; } boolean getMutable() { return this.enableMutable; } boolean getPaged() { return this.enablePaged; } Optional<ParsedSorting> getSorting() { return Optional.ofNullable(sortSettings); } @@ -63,7 +61,6 @@ class ParsedAttribute extends ParsedBlock { void setFastAccess(boolean value) { this.enableFastAccess = true; } void setFastRank(boolean value) { this.enableFastRank = true; } void setFastSearch(boolean value) { this.enableFastSearch = true; } - void setHuge(boolean value) { this.enableHuge = true; } void setMutable(boolean value) { this.enableMutable = true; } void setPaged(boolean value) { this.enablePaged = true; } } diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedDocumentSummary.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedDocumentSummary.java index 93469a86fe3..7aaabaef865 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedDocumentSummary.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedDocumentSummary.java @@ -30,8 +30,7 @@ class ParsedDocumentSummary extends ParsedBlock { ParsedSummaryField addField(ParsedSummaryField field) { String fieldName = field.name(); - // TODO disallow this on Vespa 8 - // verifyThat(! fields.containsKey(fieldName), "already has field", fieldName); + verifyThat(! fields.containsKey(fieldName), "already has field", fieldName); return fields.put(fieldName, field); } diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedType.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedType.java index 9c3206a333a..5613278ee85 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedType.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedType.java @@ -172,10 +172,7 @@ class ParsedType { case "float16": case "float": case "double": - /* TODO Vespa 8: throw new IllegalArgumentException("weightedset of inexact type '" + vt + "' is not supported"); - */ - break; default: throw new IllegalArgumentException("weightedset of unknown type '" + vt + "' is not supported"); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AdjustPositionSummaryFields.java b/config-model/src/main/java/com/yahoo/schema/processing/AdjustPositionSummaryFields.java index 6c2d62f37cb..fa663fbec96 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/AdjustPositionSummaryFields.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/AdjustPositionSummaryFields.java @@ -42,6 +42,16 @@ public class AdjustPositionSummaryFields extends Processor { } } + static String getPositionSummaryFieldName(String fieldName) { + // Only used in v7 legacy mode, remove in Vespa 9 + return fieldName + ".position"; + } + + static String getDistanceSummaryFieldName(String fieldName) { + // Only used in v7 legacy mode, remove in Vespa 9 + return fieldName + ".distance"; + } + private void scanSummary(DocumentSummary summary) { for (SummaryField summaryField : summary.getSummaryFields().values()) { if ( ! GeoPos.isAnyPos(summaryField.getDataType())) continue; @@ -75,12 +85,12 @@ public class AdjustPositionSummaryFields extends Processor { summaryField.getSources().clear(); summaryField.addSource(source); ensureSummaryField(summary, - PositionDataType.getPositionSummaryFieldName(summaryField.getName()), + getPositionSummaryFieldName(summaryField.getName()), DataType.getArray(DataType.STRING), source, SummaryTransform.POSITIONS); ensureSummaryField(summary, - PositionDataType.getDistanceSummaryFieldName(summaryField.getName()), + getDistanceSummaryFieldName(summaryField.getName()), DataType.INT, source, SummaryTransform.DISTANCE); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java index 5bb5079fab6..e39b78d0d9f 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java @@ -78,11 +78,11 @@ public class CreatePositionZCurve extends Processor { Collection<String> summaryTo = removeSummaryTo(field); if (! useV8GeoPositions) { ensureCompatibleSummary(field, zName, - PositionDataType.getPositionSummaryFieldName(fieldName), + AdjustPositionSummaryFields.getPositionSummaryFieldName(fieldName), DataType.getArray(DataType.STRING), // will become "xmlstring" SummaryTransform.POSITIONS, summaryTo, validate); ensureCompatibleSummary(field, zName, - PositionDataType.getDistanceSummaryFieldName(fieldName), + AdjustPositionSummaryFields.getDistanceSummaryFieldName(fieldName), DataType.INT, SummaryTransform.DISTANCE, summaryTo, validate); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaries.java index 4080e37003f..cb28e74b4b1 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaries.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaries.java @@ -14,6 +14,7 @@ import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; +import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; /** @@ -91,12 +92,12 @@ public class ImplicitSummaries extends Processor { if (field.doesSummarying()) { for (Attribute attribute : field.getAttributes().values()) { if ( ! attribute.isPosition()) continue; - var distField = field.getSummaryField(PositionDataType.getDistanceSummaryFieldName(fieldName)); + var distField = field.getSummaryField(AdjustPositionSummaryFields.getDistanceSummaryFieldName(fieldName)); if (distField != null) { DocumentSummary attributePrefetchSummary = getOrCreateAttributePrefetchSummary(schema); attributePrefetchSummary.add(distField); } - var posField = field.getSummaryField(PositionDataType.getPositionSummaryFieldName(fieldName)); + var posField = field.getSummaryField(AdjustPositionSummaryFields.getPositionSummaryFieldName(fieldName)); if (posField != null) { DocumentSummary attributePrefetchSummary = getOrCreateAttributePrefetchSummary(schema); attributePrefetchSummary.add(posField); @@ -119,9 +120,9 @@ public class ImplicitSummaries extends Processor { } private DocumentSummary getOrCreateAttributePrefetchSummary(Schema schema) { - DocumentSummary summary = schema.getSummariesInThis().get("attributeprefetch"); + DocumentSummary summary = schema.getSummariesInThis().get(SORTABLE_ATTRIBUTES_SUMMARY_CLASS); if (summary == null) { - summary = new DocumentSummary("attributeprefetch", schema); + summary = new DocumentSummary(SORTABLE_ATTRIBUTES_SUMMARY_CLASS, schema); schema.addSummary(summary); } return summary; @@ -142,7 +143,7 @@ public class ImplicitSummaries extends Processor { DocumentSummary summary = getOrCreateAttributePrefetchSummary(schema); SummaryField attributeSummaryField = new SummaryField(attribute.getName(), attribute.getDataType()); attributeSummaryField.addSource(attribute.getName()); - attributeSummaryField.addDestination("attributeprefetch"); + attributeSummaryField.addDestination(SORTABLE_ATTRIBUTES_SUMMARY_CLASS); attributeSummaryField.setTransform(SummaryTransform.ATTRIBUTE); summary.add(attributeSummaryField); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexFieldNames.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexFieldNames.java index 27101c47c7a..7ec1a11fb22 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IndexFieldNames.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexFieldNames.java @@ -35,7 +35,7 @@ public class IndexFieldNames extends Processor { /** * In {@link CreatePositionZCurve} we add some .position and .distance fields for pos fields. Make an exception for those for now. - * TODO Vespa 8: Rename to _position and _distance and delete this method. + * TODO Vespa 9: delete this method. * * @param field an {@link com.yahoo.schema.document.SDField} * @return true if allowed diff --git a/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java index 280eae3d88b..0362dc39c4c 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java @@ -24,6 +24,8 @@ import com.yahoo.vespa.indexinglanguage.expressions.SetVarExpression; import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; import com.yahoo.vespa.model.container.search.QueryProfiles; +import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; + import java.util.ArrayList; import java.util.List; @@ -68,7 +70,7 @@ public class PredicateProcessor extends Processor { attribute.setDensePostingListThreshold(threshold); addPredicateOptimizationIlScript(field, booleanDefinition); } - DocumentSummary summary = schema.getSummariesInThis().get("attributeprefetch"); + DocumentSummary summary = schema.getSummariesInThis().get(SORTABLE_ATTRIBUTES_SUMMARY_CLASS); if (summary != null) { summary.remove(attribute.getName()); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ReferenceFieldsProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/ReferenceFieldsProcessor.java index 43e39b1e546..67c07aaeaf4 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/ReferenceFieldsProcessor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ReferenceFieldsProcessor.java @@ -10,6 +10,8 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; +import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; + /** * Class that processes reference fields and removes attribute aspect of such fields from summary. * @@ -51,7 +53,7 @@ public class ReferenceFieldsProcessor extends Processor { } private void removeFromAttributePrefetchSummaryClass(SDField field) { - DocumentSummary summary = schema.getSummariesInThis().get("attributeprefetch"); + DocumentSummary summary = schema.getSummariesInThis().get(SORTABLE_ATTRIBUTES_SUMMARY_CLASS); if (summary != null) { summary.remove(field.getName()); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TagType.java b/config-model/src/main/java/com/yahoo/schema/processing/TagType.java index f511d572bc6..112d3c4cad0 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/TagType.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/TagType.java @@ -2,8 +2,9 @@ package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.document.WeightedSetDataType; import com.yahoo.schema.RankProfileRegistry; -import com.yahoo.document.*; import com.yahoo.schema.Schema; import com.yahoo.schema.document.Matching; import com.yahoo.schema.document.MatchType; diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java index 1974774f7b5..e54ed1f1a8b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java @@ -1,8 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.configmodel.producers; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.MapDataType; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.StructDataType; +import com.yahoo.document.TensorDataType; +import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.document.*; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; import com.yahoo.documentmodel.NewDocumentReferenceDataType; @@ -68,9 +75,9 @@ public class DocumentManager { } static private class IdxMap { - private Map<Integer, Boolean> doneMap = new HashMap<>(); - private Map<String, Integer> map = new HashMap<>(); - private DataTypeRecognizer recognizer = new DataTypeRecognizer(); + private final Map<Integer, Boolean> doneMap = new HashMap<>(); + private final Map<String, Integer> map = new HashMap<>(); + private final DataTypeRecognizer recognizer = new DataTypeRecognizer(); private void add(String name) { // the adding of "10000" here is mostly to make it more @@ -111,8 +118,7 @@ public class DocumentManager { } static private <T> List<T> sortedList(Collection<T> unsorted, Comparator<T> cmp) { - var list = new ArrayList<T>(); - list.addAll(unsorted); + var list = new ArrayList<>(unsorted); list.sort(cmp); return list; } @@ -174,7 +180,6 @@ public class DocumentManager { builder.annotationtype(annBuilder); } - @SuppressWarnings("deprecation") private void docTypeBuildAnyType(DataType type, DocumentmanagerConfig.Doctype.Builder documentBuilder, IdxMap indexMap) { if (indexMap.isDone(type)) { return; @@ -188,9 +193,7 @@ public class DocumentManager { return; } indexMap.setDone(type); - if (type instanceof TemporaryStructuredDataType) { - throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName()); - } else if (type instanceof TemporaryUnknownType) { + if (type instanceof TemporaryUnknownType) { throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName()); } else if (type instanceof OwnedTemporaryType) { throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName()); diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java index bb57cd189c9..549b94adc02 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java @@ -1,7 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.configmodel.producers; -import com.yahoo.document.*; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.MapDataType; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.document.StructDataType; +import com.yahoo.document.TensorDataType; +import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.config.DocumenttypesConfig; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; @@ -12,7 +19,15 @@ import com.yahoo.documentmodel.TemporaryUnknownType; import com.yahoo.documentmodel.VespaDocumentType; import com.yahoo.schema.document.FieldSet; import com.yahoo.vespa.documentmodel.DocumentModel; -import java.util.*; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; /** * @author baldersheim @@ -37,8 +52,7 @@ public class DocumentTypes { } static private <T> List<T> sortedList(Collection<T> unsorted, Comparator<T> cmp) { - var list = new ArrayList<T>(); - list.addAll(unsorted); + var list = new ArrayList<>(unsorted); list.sort(cmp); return list; } @@ -58,8 +72,8 @@ public class DocumentTypes { } static private class IdxMap { - private Map<Integer, Boolean> doneMap = new HashMap<>(); - private Map<Object, Integer> map = new IdentityHashMap<>(); + private final Map<Integer, Boolean> doneMap = new HashMap<>(); + private final Map<Object, Integer> map = new IdentityHashMap<>(); void add(Object someType) { assert(someType != null); // the adding of "10000" here is mostly to make it more @@ -153,7 +167,6 @@ public class DocumentTypes { builder.annotationtype(annBuilder); } - @SuppressWarnings("deprecation") private void docTypeBuildAnyType(DataType type, DocumenttypesConfig.Doctype.Builder documentBuilder, IdxMap indexMap) { if (indexMap.isDone(type)) { return; @@ -167,9 +180,7 @@ public class DocumentTypes { return; } indexMap.setDone(type); - if (type instanceof TemporaryStructuredDataType) { - throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName()); - } else if (type instanceof TemporaryUnknownType) { + if (type instanceof TemporaryUnknownType) { throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName()); } else if (type instanceof OwnedTemporaryType) { throw new IllegalArgumentException("Can not create config for temporary data type: " + type.getName()); diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java index 92a75cb051b..ed6668f0d0d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java @@ -22,7 +22,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey { */ public static class Source implements Serializable { - private String name; + private final String name; private boolean override = false; public Source(String name) { this.name = name; @@ -54,7 +54,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey { } /** The transform to perform on the stored source */ - private SummaryTransform transform=SummaryTransform.NONE; + private SummaryTransform transform; /** The command used per field in vsmsummary */ private VsmCommand vsmCommand = VsmCommand.NONE; @@ -156,7 +156,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey { return destinations; } - private String toString(Collection<?> collection) { + public String toString(Collection<?> collection) { StringBuilder buffer=new StringBuilder(); for (Iterator<?> i=collection.iterator(); i.hasNext(); ) { buffer.append(i.next().toString()); @@ -283,8 +283,8 @@ public class SummaryField extends Field implements Cloneable, TypedKey { FLATTENSPACE("FLATTENSPACE"), FLATTENJUNIPER("FLATTENJUNIPER"); - private String cmd=""; - private VsmCommand(String cmd) { + private final String cmd; + VsmCommand(String cmd) { this.cmd=cmd; } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaConfigModelRegistry.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaConfigModelRegistry.java index 03750dbaa6e..ca2d6a9da76 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaConfigModelRegistry.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaConfigModelRegistry.java @@ -5,12 +5,10 @@ import com.yahoo.config.model.ConfigModelRegistry; import com.yahoo.config.model.admin.AdminModel; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; -import com.yahoo.vespa.model.builder.xml.dom.DomClientsBuilder; import com.yahoo.vespa.model.builder.xml.dom.DomRoutingBuilder; import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking; import com.yahoo.vespa.model.content.Content; -import com.yahoo.vespa.model.generic.GenericServicesBuilder; import java.util.ArrayList; import java.util.Collection; @@ -33,10 +31,8 @@ public class VespaConfigModelRegistry extends ConfigModelRegistry { builderList.add(new AdminModel.BuilderV2()); builderList.add(new AdminModel.BuilderV4()); builderList.add(new DomRoutingBuilder()); - builderList.add(new DomClientsBuilder()); builderList.add(new Content.Builder()); builderList.add(new ContainerModelBuilder(false, Networking.enable)); - builderList.add(new GenericServicesBuilder()); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index a919c19d754..7f5a78fa917 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -45,7 +45,6 @@ import com.yahoo.vespa.model.InstanceResolver.PackagePrefix; import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.builder.VespaModelBuilder; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; -import com.yahoo.vespa.model.clients.Clients; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -53,7 +52,6 @@ import com.yahoo.vespa.model.content.Content; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; import com.yahoo.vespa.model.filedistribution.FileReferencesRepository; -import com.yahoo.vespa.model.generic.service.ServiceCluster; import com.yahoo.vespa.model.ml.ConvertedModel; import com.yahoo.vespa.model.ml.ModelName; import com.yahoo.vespa.model.ml.OnnxModelInfo; @@ -116,9 +114,6 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri private final ApplicationPackage applicationPackage; - /** Generic service instances - service clusters which have no specific model */ - private final List<ServiceCluster> serviceClusters = new ArrayList<>(); - /** The global rank profiles of this model */ private final RankProfileList rankProfileList; @@ -183,7 +178,6 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri if (complete) { // create a completed, frozen model root.useFeatureFlags(deployState.getProperties().featureFlags()); configModelRepo.readConfigModels(deployState, this, builder, root, new VespaConfigModelRegistry(configModelRegistry)); - addServiceClusters(deployState, builder); setupRouting(deployState); getAdmin().addPerHostServices(hostSystem.getHosts(), deployState); freezeModelTopology(); @@ -261,11 +255,6 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri } } - /** Adds generic application specific clusters of services */ - private void addServiceClusters(DeployState deployState, VespaModelBuilder builder) { - serviceClusters.addAll(builder.getClusters(deployState, this)); - } - /** * Creates a rank profile not attached to any search definition, for each imported model in the application package, * and adds it to the given rank profile registry. @@ -619,10 +608,6 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri id2producer.put(configId, descendant); } - public Clients getClients() { - return configModelRepo.getClients(); - } - public List<SearchCluster> getSearchClusters() { return Content.getSearchClusters(configModelRepo()); } @@ -656,11 +641,6 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri return root.getFileDistributionConfigProducer(); } - /** The clusters of application specific generic services */ - public List<ServiceCluster> serviceClusters() { - return serviceClusters; - } - /** Returns an unmodifiable view of the mapping of config id to {@link ConfigProducer} */ public Map<String, ConfigProducer> id2producer() { return Collections.unmodifiableMap(id2producer); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index e299689d1c7..79f65264249 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -189,7 +189,6 @@ public class Admin extends AbstractConfigProducer<Admin> implements Serializable else { builder. logserver(new LogdConfig.Logserver.Builder(). - userpc(true). use(logServerContainerCluster.isPresent() || !isHostedVespa). host(logserver.getHostName()). rpcport(logserver.getRelativePort(0))); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java index 26e3d952080..cd6de7ac7fb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java @@ -2,8 +2,6 @@ package com.yahoo.vespa.model.admin.monitoring; import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; @@ -41,9 +39,9 @@ public class AutoscalingMetrics { // Write rate metrics.add("feed.http-requests.rate"); // container - metrics.add("vds.filestor.alldisks.allthreads.put.sum.count.rate"); // content - metrics.add("vds.filestor.alldisks.allthreads.remove.sum.count.rate"); // content - metrics.add("vds.filestor.alldisks.allthreads.update.sum.count.rate"); // content + metrics.add("vds.filestor.allthreads.put.count.rate"); // content + metrics.add("vds.filestor.allthreads.remove.count.rate"); // content + metrics.add("vds.filestor.allthreads.update.count.rate"); // content return new MetricSet("autoscaling", toMetrics(metrics)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 95307456f80..09e2e7680f8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -93,7 +93,6 @@ public class VespaMetricSet { metrics.add(new Metric("configserver.latency.max")); metrics.add(new Metric("configserver.latency.sum")); metrics.add(new Metric("configserver.latency.count")); - metrics.add(new Metric("configserver.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("configserver.cacheConfigElems.last")); metrics.add(new Metric("configserver.cacheChecksumElems.last")); metrics.add(new Metric("configserver.hosts.last")); @@ -118,22 +117,15 @@ public class VespaMetricSet { metrics.add(new Metric("handled.latency.max")); metrics.add(new Metric("handled.latency.sum")); metrics.add(new Metric("handled.latency.count")); - metrics.add(new Metric("handled.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("serverRejectedRequests.rate")); metrics.add(new Metric("serverRejectedRequests.count")); - metrics.add(new Metric("serverThreadPoolSize.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("serverThreadPoolSize.min")); // TODO: Remove in Vespa 8 metrics.add(new Metric("serverThreadPoolSize.max")); - metrics.add(new Metric("serverThreadPoolSize.rate")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("serverThreadPoolSize.count")); // TODO: Remove in Vespa 8 metrics.add(new Metric("serverThreadPoolSize.last")); - metrics.add(new Metric("serverActiveThreads.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("serverActiveThreads.min")); metrics.add(new Metric("serverActiveThreads.max")); - metrics.add(new Metric("serverActiveThreads.rate")); // TODO: Remove in Vespa 8 metrics.add(new Metric("serverActiveThreads.sum")); metrics.add(new Metric("serverActiveThreads.count")); metrics.add(new Metric("serverActiveThreads.last")); @@ -171,11 +163,9 @@ public class VespaMetricSet { metrics.add(new Metric("httpapi_latency.max")); metrics.add(new Metric("httpapi_latency.sum")); metrics.add(new Metric("httpapi_latency.count")); - metrics.add(new Metric("httpapi_latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("httpapi_pending.max")); metrics.add(new Metric("httpapi_pending.sum")); metrics.add(new Metric("httpapi_pending.count")); - metrics.add(new Metric("httpapi_pending.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("httpapi_num_operations.rate")); metrics.add(new Metric("httpapi_num_updates.rate")); metrics.add(new Metric("httpapi_num_removes.rate")); @@ -223,17 +213,13 @@ public class VespaMetricSet { metrics.add(new Metric("http.status.3xx.rate")); metrics.add(new Metric("http.status.4xx.rate")); metrics.add(new Metric("http.status.5xx.rate")); - metrics.add(new Metric("http.status.401.rate")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("http.status.403.rate")); // TODO: Remove in Vespa 8 metrics.add(new Metric("jdisc.http.request.uri_length.max")); metrics.add(new Metric("jdisc.http.request.uri_length.sum")); metrics.add(new Metric("jdisc.http.request.uri_length.count")); - metrics.add(new Metric("jdisc.http.request.uri_length.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("jdisc.http.request.content_size.max")); metrics.add(new Metric("jdisc.http.request.content_size.sum")); metrics.add(new Metric("jdisc.http.request.content_size.count")); - metrics.add(new Metric("jdisc.http.request.content_size.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("jdisc.http.ssl.handshake.failure.missing_client_cert.rate")); metrics.add(new Metric("jdisc.http.ssl.handshake.failure.expired_client_cert.rate")); @@ -317,22 +303,18 @@ public class VespaMetricSet { metrics.add(new Metric("search_connections.max")); metrics.add(new Metric("search_connections.sum")); metrics.add(new Metric("search_connections.count")); - metrics.add(new Metric("search_connections.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("feed.latency.max")); metrics.add(new Metric("feed.latency.sum")); metrics.add(new Metric("feed.latency.count")); - metrics.add(new Metric("feed.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("feed.http-requests.count")); metrics.add(new Metric("feed.http-requests.rate")); metrics.add(new Metric("queries.rate")); metrics.add(new Metric("query_container_latency.max")); metrics.add(new Metric("query_container_latency.sum")); metrics.add(new Metric("query_container_latency.count")); - metrics.add(new Metric("query_container_latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("query_latency.max")); metrics.add(new Metric("query_latency.sum")); metrics.add(new Metric("query_latency.count")); - metrics.add(new Metric("query_latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("query_latency.95percentile")); metrics.add(new Metric("query_latency.99percentile")); metrics.add(new Metric("failed_queries.rate")); @@ -340,7 +322,6 @@ public class VespaMetricSet { metrics.add(new Metric("hits_per_query.max")); metrics.add(new Metric("hits_per_query.sum")); metrics.add(new Metric("hits_per_query.count")); - metrics.add(new Metric("hits_per_query.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("hits_per_query.95percentile")); metrics.add(new Metric("hits_per_query.99percentile")); metrics.add(new Metric("query_hit_offset.max")); @@ -356,7 +337,6 @@ public class VespaMetricSet { metrics.add(new Metric("totalhits_per_query.max")); metrics.add(new Metric("totalhits_per_query.sum")); metrics.add(new Metric("totalhits_per_query.count")); - metrics.add(new Metric("totalhits_per_query.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("totalhits_per_query.95percentile")); metrics.add(new Metric("totalhits_per_query.99percentile")); metrics.add(new Metric("empty_results.rate")); @@ -365,13 +345,10 @@ public class VespaMetricSet { metrics.add(new Metric("relevance.at_1.sum")); metrics.add(new Metric("relevance.at_1.count")); - metrics.add(new Metric("relevance.at_1.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("relevance.at_3.sum")); metrics.add(new Metric("relevance.at_3.count")); - metrics.add(new Metric("relevance.at_3.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("relevance.at_10.sum")); metrics.add(new Metric("relevance.at_10.count")); - metrics.add(new Metric("relevance.at_10.average")); // TODO: Remove in Vespa 8 // Errors from qrserver metrics.add(new Metric("error.timeout.rate")); @@ -394,7 +371,6 @@ public class VespaMetricSet { metrics.add(new Metric(prefix + ".queuesize.max")); metrics.add(new Metric(prefix + ".queuesize.sum")); metrics.add(new Metric(prefix + ".queuesize.count")); - metrics.add(new Metric(prefix + ".maxpending.last")); // TODO: Remove in Vespa 8 metrics.add(new Metric(prefix + ".accepted.rate")); metrics.add(new Metric(prefix + ".wakeups.rate")); metrics.add(new Metric(prefix + ".utilization.max")); @@ -419,11 +395,9 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.docsum.latency.max")); metrics.add(new Metric("content.proton.docsum.latency.sum")); metrics.add(new Metric("content.proton.docsum.latency.count")); - metrics.add(new Metric("content.proton.docsum.latency.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("content.proton.transport.query.latency.max")); metrics.add(new Metric("content.proton.transport.query.latency.sum")); metrics.add(new Metric("content.proton.transport.query.latency.count")); - metrics.add(new Metric("content.proton.transport.query.latency.average")); // TODO: Remove in Vespa 8 // Search protocol metrics.add(new Metric("content.proton.search_protocol.query.latency.max")); @@ -503,16 +477,11 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.resource_usage.memory_usage.total.max")); metrics.add(new Metric("content.proton.resource_usage.memory_usage.total_utilization.max")); metrics.add(new Metric("content.proton.resource_usage.memory_usage.transient.max")); - metrics.add(new Metric("content.proton.resource_usage.disk_utilization.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("content.proton.resource_usage.memory_utilization.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("content.proton.resource_usage.transient_memory.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("content.proton.resource_usage.transient_disk.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("content.proton.resource_usage.memory_mappings.max")); metrics.add(new Metric("content.proton.resource_usage.open_file_descriptors.max")); metrics.add(new Metric("content.proton.resource_usage.feeding_blocked.max")); metrics.add(new Metric("content.proton.resource_usage.malloc_arena.max")); metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.address_space.max")); - metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.feeding_blocked.last")); // TODO: Remove in Vespa 8 metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.feeding_blocked.max")); // CPU util @@ -592,11 +561,6 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.matching.query_latency.max")); metrics.add(new Metric("content.proton.documentdb.matching.query_latency.sum")); metrics.add(new Metric("content.proton.documentdb.matching.query_latency.count")); - metrics.add(new Metric("content.proton.documentdb.matching.query_latency.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.max")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.sum")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.count")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.max")); metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.sum")); metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.count")); @@ -611,11 +575,6 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.max")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.sum")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.count")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.max")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.sum")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.count")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.average")); // TODO: Remove in Vespa 8 metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.max")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.sum")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.count")); @@ -625,8 +584,7 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.max")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.sum")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.count")); - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.docs_matched.rate")); // TODO: Consider remove in Vespa 8 + metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.docs_matched.rate")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.docs_matched.count")); metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.limited_queries.rate")); @@ -650,136 +608,120 @@ public class VespaMetricSet { // TODO: disregarded when considering metric names. Consider cleaning up for Vespa 9. metrics.add(new Metric("vds.datastored.alldisks.docs.average")); metrics.add(new Metric("vds.datastored.alldisks.bytes.average")); - metrics.add(new Metric("vds.visitor.allthreads.averagevisitorlifetime.sum.max")); - metrics.add(new Metric("vds.visitor.allthreads.averagevisitorlifetime.sum.sum")); - metrics.add(new Metric("vds.visitor.allthreads.averagevisitorlifetime.sum.count")); - metrics.add(new Metric("vds.visitor.allthreads.averagevisitorlifetime.sum.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.visitor.allthreads.averagequeuewait.sum.max")); - metrics.add(new Metric("vds.visitor.allthreads.averagequeuewait.sum.sum")); - metrics.add(new Metric("vds.visitor.allthreads.averagequeuewait.sum.count")); - metrics.add(new Metric("vds.visitor.allthreads.averagequeuewait.sum.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.max")); - metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.sum")); - metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.count")); - metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.visitor.allthreads.completed.sum.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.visitor.allthreads.completed.sum.rate")); - metrics.add(new Metric("vds.visitor.allthreads.created.sum.rate")); - metrics.add(new Metric("vds.visitor.allthreads.failed.sum.rate")); - metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.max")); - metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.sum")); - metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.count")); - metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.max")); - metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.sum")); - metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.count")); - metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.average")); // TODO: Remove in Vespa 8 - - metrics.add(new Metric("vds.filestor.alldisks.queuesize.max")); - metrics.add(new Metric("vds.filestor.alldisks.queuesize.sum")); - metrics.add(new Metric("vds.filestor.alldisks.queuesize.count")); - metrics.add(new Metric("vds.filestor.alldisks.queuesize.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.max")); - metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.sum")); - metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.count")); - metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.filestor.alldisks.active_operations.size.max")); - metrics.add(new Metric("vds.filestor.alldisks.active_operations.size.sum")); - metrics.add(new Metric("vds.filestor.alldisks.active_operations.size.count")); - metrics.add(new Metric("vds.filestor.alldisks.active_operations.latency.max")); - metrics.add(new Metric("vds.filestor.alldisks.active_operations.latency.sum")); - metrics.add(new Metric("vds.filestor.alldisks.active_operations.latency.count")); - metrics.add(new Metric("vds.filestor.alldisks.throttle_window_size.max")); - metrics.add(new Metric("vds.filestor.alldisks.throttle_window_size.sum")); - metrics.add(new Metric("vds.filestor.alldisks.throttle_window_size.count")); - metrics.add(new Metric("vds.filestor.alldisks.throttle_waiting_threads.max")); - metrics.add(new Metric("vds.filestor.alldisks.throttle_waiting_threads.sum")); - metrics.add(new Metric("vds.filestor.alldisks.throttle_waiting_threads.count")); - metrics.add(new Metric("vds.filestor.alldisks.throttle_active_tokens.max")); - metrics.add(new Metric("vds.filestor.alldisks.throttle_active_tokens.sum")); - metrics.add(new Metric("vds.filestor.alldisks.throttle_active_tokens.count")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergemetadatareadlatency.max")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergemetadatareadlatency.sum")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergemetadatareadlatency.count")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatareadlatency.max")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatareadlatency.sum")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatareadlatency.count")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatawritelatency.max")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatawritelatency.sum")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatawritelatency.count")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.put_latency.max")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.put_latency.sum")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.put_latency.count")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove_latency.max")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove_latency.sum")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove_latency.count")); - metrics.add(new Metric("vds.filestor.alldisks.allstripes.throttled_rpc_direct_dispatches.rate")); - metrics.add(new Metric("vds.filestor.alldisks.allstripes.throttled_persistence_thread_polls.rate")); - metrics.add(new Metric("vds.filestor.alldisks.allstripes.timeouts_waiting_for_throttle_token.rate")); + metrics.add(new Metric("vds.visitor.allthreads.averagevisitorlifetime.max")); + metrics.add(new Metric("vds.visitor.allthreads.averagevisitorlifetime.sum")); + metrics.add(new Metric("vds.visitor.allthreads.averagevisitorlifetime.count")); + metrics.add(new Metric("vds.visitor.allthreads.averagequeuewait.max")); + metrics.add(new Metric("vds.visitor.allthreads.averagequeuewait.sum")); + metrics.add(new Metric("vds.visitor.allthreads.averagequeuewait.count")); + metrics.add(new Metric("vds.visitor.allthreads.queuesize.max")); + metrics.add(new Metric("vds.visitor.allthreads.queuesize.sum")); + metrics.add(new Metric("vds.visitor.allthreads.queuesize.count")); + metrics.add(new Metric("vds.visitor.allthreads.completed.rate")); + metrics.add(new Metric("vds.visitor.allthreads.created.rate")); + metrics.add(new Metric("vds.visitor.allthreads.failed.rate")); + metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.max")); + metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum")); + metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.count")); + metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.max")); + metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum")); + metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.count")); + + metrics.add(new Metric("vds.filestor.queuesize.max")); + metrics.add(new Metric("vds.filestor.queuesize.sum")); + metrics.add(new Metric("vds.filestor.queuesize.count")); + metrics.add(new Metric("vds.filestor.averagequeuewait.max")); + metrics.add(new Metric("vds.filestor.averagequeuewait.sum")); + metrics.add(new Metric("vds.filestor.averagequeuewait.count")); + metrics.add(new Metric("vds.filestor.active_operations.size.max")); + metrics.add(new Metric("vds.filestor.active_operations.size.sum")); + metrics.add(new Metric("vds.filestor.active_operations.size.count")); + metrics.add(new Metric("vds.filestor.active_operations.latency.max")); + metrics.add(new Metric("vds.filestor.active_operations.latency.sum")); + metrics.add(new Metric("vds.filestor.active_operations.latency.count")); + metrics.add(new Metric("vds.filestor.throttle_window_size.max")); + metrics.add(new Metric("vds.filestor.throttle_window_size.sum")); + metrics.add(new Metric("vds.filestor.throttle_window_size.count")); + metrics.add(new Metric("vds.filestor.throttle_waiting_threads.max")); + metrics.add(new Metric("vds.filestor.throttle_waiting_threads.sum")); + metrics.add(new Metric("vds.filestor.throttle_waiting_threads.count")); + metrics.add(new Metric("vds.filestor.throttle_active_tokens.max")); + metrics.add(new Metric("vds.filestor.throttle_active_tokens.sum")); + metrics.add(new Metric("vds.filestor.throttle_active_tokens.count")); + metrics.add(new Metric("vds.filestor.allthreads.mergemetadatareadlatency.max")); + metrics.add(new Metric("vds.filestor.allthreads.mergemetadatareadlatency.sum")); + metrics.add(new Metric("vds.filestor.allthreads.mergemetadatareadlatency.count")); + metrics.add(new Metric("vds.filestor.allthreads.mergedatareadlatency.max")); + metrics.add(new Metric("vds.filestor.allthreads.mergedatareadlatency.sum")); + metrics.add(new Metric("vds.filestor.allthreads.mergedatareadlatency.count")); + metrics.add(new Metric("vds.filestor.allthreads.mergedatawritelatency.max")); + metrics.add(new Metric("vds.filestor.allthreads.mergedatawritelatency.sum")); + metrics.add(new Metric("vds.filestor.allthreads.mergedatawritelatency.count")); + metrics.add(new Metric("vds.filestor.allthreads.put_latency.max")); + metrics.add(new Metric("vds.filestor.allthreads.put_latency.sum")); + metrics.add(new Metric("vds.filestor.allthreads.put_latency.count")); + metrics.add(new Metric("vds.filestor.allthreads.remove_latency.max")); + metrics.add(new Metric("vds.filestor.allthreads.remove_latency.sum")); + metrics.add(new Metric("vds.filestor.allthreads.remove_latency.count")); + metrics.add(new Metric("vds.filestor.allstripes.throttled_rpc_direct_dispatches.rate")); + metrics.add(new Metric("vds.filestor.allstripes.throttled_persistence_thread_polls.rate")); + metrics.add(new Metric("vds.filestor.allstripes.timeouts_waiting_for_throttle_token.rate")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.count.rate")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.failed.rate")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.test_and_set_failed.rate")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.max")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.sum")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.count")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.request_size.max")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.request_size.sum")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.request_size.count")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.count.rate")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.failed.rate")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.test_and_set_failed.rate")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.max")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.sum")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.count")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.request_size.max")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.request_size.sum")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.request_size.count")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.count.rate")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.failed.rate")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.max")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.sum")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.count")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.request_size.max")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.request_size.sum")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.request_size.count")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.count.rate")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.failed.rate")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.test_and_set_failed.rate")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.max")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.sum")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.count")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.request_size.max")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.request_size.sum")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.request_size.count")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.createiterator.latency.max")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.createiterator.latency.sum")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.createiterator.latency.count")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.createiterator.latency.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.filestor.alldisks.allthreads.createiterator.count.rate")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.visit.sum.count.rate")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.visit.sum.latency.max")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.visit.sum.latency.sum")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.visit.sum.latency.count")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.visit.sum.latency.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove_location.sum.count.rate")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove_location.sum.latency.max")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove_location.sum.latency.sum")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove_location.sum.latency.count")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove_location.sum.latency.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.filestor.alldisks.allthreads.splitbuckets.count.rate")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.joinbuckets.count.rate")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.count.rate")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.failed.rate")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.max")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.sum")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.count")); - metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.filestor.alldisks.allthreads.setbucketstates.count.rate")); + metrics.add(new Metric("vds.filestor.allthreads.put.count.rate")); + metrics.add(new Metric("vds.filestor.allthreads.put.failed.rate")); + metrics.add(new Metric("vds.filestor.allthreads.put.test_and_set_failed.rate")); + metrics.add(new Metric("vds.filestor.allthreads.put.latency.max")); + metrics.add(new Metric("vds.filestor.allthreads.put.latency.sum")); + metrics.add(new Metric("vds.filestor.allthreads.put.latency.count")); + metrics.add(new Metric("vds.filestor.allthreads.put.request_size.max")); + metrics.add(new Metric("vds.filestor.allthreads.put.request_size.sum")); + metrics.add(new Metric("vds.filestor.allthreads.put.request_size.count")); + metrics.add(new Metric("vds.filestor.allthreads.remove.count.rate")); + metrics.add(new Metric("vds.filestor.allthreads.remove.failed.rate")); + metrics.add(new Metric("vds.filestor.allthreads.remove.test_and_set_failed.rate")); + metrics.add(new Metric("vds.filestor.allthreads.remove.latency.max")); + metrics.add(new Metric("vds.filestor.allthreads.remove.latency.sum")); + metrics.add(new Metric("vds.filestor.allthreads.remove.latency.count")); + metrics.add(new Metric("vds.filestor.allthreads.remove.request_size.max")); + metrics.add(new Metric("vds.filestor.allthreads.remove.request_size.sum")); + metrics.add(new Metric("vds.filestor.allthreads.remove.request_size.count")); + metrics.add(new Metric("vds.filestor.allthreads.get.count.rate")); + metrics.add(new Metric("vds.filestor.allthreads.get.failed.rate")); + metrics.add(new Metric("vds.filestor.allthreads.get.latency.max")); + metrics.add(new Metric("vds.filestor.allthreads.get.latency.sum")); + metrics.add(new Metric("vds.filestor.allthreads.get.latency.count")); + metrics.add(new Metric("vds.filestor.allthreads.get.request_size.max")); + metrics.add(new Metric("vds.filestor.allthreads.get.request_size.sum")); + metrics.add(new Metric("vds.filestor.allthreads.get.request_size.count")); + metrics.add(new Metric("vds.filestor.allthreads.update.count.rate")); + metrics.add(new Metric("vds.filestor.allthreads.update.failed.rate")); + metrics.add(new Metric("vds.filestor.allthreads.update.test_and_set_failed.rate")); + metrics.add(new Metric("vds.filestor.allthreads.update.latency.max")); + metrics.add(new Metric("vds.filestor.allthreads.update.latency.sum")); + metrics.add(new Metric("vds.filestor.allthreads.update.latency.count")); + metrics.add(new Metric("vds.filestor.allthreads.update.request_size.max")); + metrics.add(new Metric("vds.filestor.allthreads.update.request_size.sum")); + metrics.add(new Metric("vds.filestor.allthreads.update.request_size.count")); + metrics.add(new Metric("vds.filestor.allthreads.createiterator.latency.max")); + metrics.add(new Metric("vds.filestor.allthreads.createiterator.latency.sum")); + metrics.add(new Metric("vds.filestor.allthreads.createiterator.latency.count")); + metrics.add(new Metric("vds.filestor.allthreads.createiterator.count.rate")); + metrics.add(new Metric("vds.filestor.allthreads.visit.count.rate")); + metrics.add(new Metric("vds.filestor.allthreads.visit.latency.max")); + metrics.add(new Metric("vds.filestor.allthreads.visit.latency.sum")); + metrics.add(new Metric("vds.filestor.allthreads.visit.latency.count")); + metrics.add(new Metric("vds.filestor.allthreads.remove_location.count.rate")); + metrics.add(new Metric("vds.filestor.allthreads.remove_location.latency.max")); + metrics.add(new Metric("vds.filestor.allthreads.remove_location.latency.sum")); + metrics.add(new Metric("vds.filestor.allthreads.remove_location.latency.count")); + metrics.add(new Metric("vds.filestor.allthreads.splitbuckets.count.rate")); + metrics.add(new Metric("vds.filestor.allthreads.joinbuckets.count.rate")); + metrics.add(new Metric("vds.filestor.allthreads.deletebuckets.count.rate")); + metrics.add(new Metric("vds.filestor.allthreads.deletebuckets.failed.rate")); + metrics.add(new Metric("vds.filestor.allthreads.deletebuckets.latency.max")); + metrics.add(new Metric("vds.filestor.allthreads.deletebuckets.latency.sum")); + metrics.add(new Metric("vds.filestor.allthreads.deletebuckets.latency.count")); + metrics.add(new Metric("vds.filestor.allthreads.setbucketstates.count.rate")); return metrics; } private static Set<Metric> getDistributorMetrics() { @@ -818,66 +760,61 @@ public class VespaMetricSet { metrics.add(new Metric("vds.idealstate.garbage_collection.documents_removed.count")); metrics.add(new Metric("vds.idealstate.garbage_collection.documents_removed.rate")); - metrics.add(new Metric("vds.distributor.puts.sum.latency.max")); - metrics.add(new Metric("vds.distributor.puts.sum.latency.sum")); - metrics.add(new Metric("vds.distributor.puts.sum.latency.count")); - metrics.add(new Metric("vds.distributor.puts.sum.latency.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.distributor.puts.sum.ok.rate")); - metrics.add(new Metric("vds.distributor.puts.sum.failures.total.rate")); - metrics.add(new Metric("vds.distributor.puts.sum.failures.notfound.rate")); - metrics.add(new Metric("vds.distributor.puts.sum.failures.test_and_set_failed.rate")); - metrics.add(new Metric("vds.distributor.puts.sum.failures.concurrent_mutations.rate")); - metrics.add(new Metric("vds.distributor.puts.sum.failures.notconnected.rate")); - metrics.add(new Metric("vds.distributor.puts.sum.failures.notready.rate")); - metrics.add(new Metric("vds.distributor.puts.sum.failures.wrongdistributor.rate")); - metrics.add(new Metric("vds.distributor.puts.sum.failures.safe_time_not_reached.rate")); - metrics.add(new Metric("vds.distributor.puts.sum.failures.storagefailure.rate")); - metrics.add(new Metric("vds.distributor.puts.sum.failures.timeout.rate")); - metrics.add(new Metric("vds.distributor.puts.sum.failures.busy.rate")); - metrics.add(new Metric("vds.distributor.puts.sum.failures.inconsistent_bucket.rate")); - metrics.add(new Metric("vds.distributor.removes.sum.latency.max")); - metrics.add(new Metric("vds.distributor.removes.sum.latency.sum")); - metrics.add(new Metric("vds.distributor.removes.sum.latency.count")); - metrics.add(new Metric("vds.distributor.removes.sum.latency.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.distributor.removes.sum.ok.rate")); - metrics.add(new Metric("vds.distributor.removes.sum.failures.total.rate")); - metrics.add(new Metric("vds.distributor.removes.sum.failures.notfound.rate")); - metrics.add(new Metric("vds.distributor.removes.sum.failures.test_and_set_failed.rate")); - metrics.add(new Metric("vds.distributor.removes.sum.failures.concurrent_mutations.rate")); - metrics.add(new Metric("vds.distributor.updates.sum.latency.max")); - metrics.add(new Metric("vds.distributor.updates.sum.latency.sum")); - metrics.add(new Metric("vds.distributor.updates.sum.latency.count")); - metrics.add(new Metric("vds.distributor.updates.sum.latency.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.distributor.updates.sum.ok.rate")); - metrics.add(new Metric("vds.distributor.updates.sum.failures.total.rate")); - metrics.add(new Metric("vds.distributor.updates.sum.failures.notfound.rate")); - metrics.add(new Metric("vds.distributor.updates.sum.failures.test_and_set_failed.rate")); - metrics.add(new Metric("vds.distributor.updates.sum.failures.concurrent_mutations.rate")); - metrics.add(new Metric("vds.distributor.updates.sum.diverging_timestamp_updates.rate")); - metrics.add(new Metric("vds.distributor.removelocations.sum.ok.rate")); - metrics.add(new Metric("vds.distributor.removelocations.sum.failures.total.rate")); - metrics.add(new Metric("vds.distributor.gets.sum.latency.max")); - metrics.add(new Metric("vds.distributor.gets.sum.latency.sum")); - metrics.add(new Metric("vds.distributor.gets.sum.latency.count")); - metrics.add(new Metric("vds.distributor.gets.sum.latency.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.distributor.gets.sum.ok.rate")); - metrics.add(new Metric("vds.distributor.gets.sum.failures.total.rate")); - metrics.add(new Metric("vds.distributor.gets.sum.failures.notfound.rate")); - metrics.add(new Metric("vds.distributor.visitor.sum.latency.max")); - metrics.add(new Metric("vds.distributor.visitor.sum.latency.sum")); - metrics.add(new Metric("vds.distributor.visitor.sum.latency.count")); - metrics.add(new Metric("vds.distributor.visitor.sum.latency.average")); // TODO: Remove in Vespa 8 - metrics.add(new Metric("vds.distributor.visitor.sum.ok.rate")); - metrics.add(new Metric("vds.distributor.visitor.sum.failures.total.rate")); - metrics.add(new Metric("vds.distributor.visitor.sum.failures.notready.rate")); - metrics.add(new Metric("vds.distributor.visitor.sum.failures.notconnected.rate")); - metrics.add(new Metric("vds.distributor.visitor.sum.failures.wrongdistributor.rate")); - metrics.add(new Metric("vds.distributor.visitor.sum.failures.safe_time_not_reached.rate")); - metrics.add(new Metric("vds.distributor.visitor.sum.failures.storagefailure.rate")); - metrics.add(new Metric("vds.distributor.visitor.sum.failures.timeout.rate")); - metrics.add(new Metric("vds.distributor.visitor.sum.failures.busy.rate")); - metrics.add(new Metric("vds.distributor.visitor.sum.failures.inconsistent_bucket.rate")); - metrics.add(new Metric("vds.distributor.visitor.sum.failures.notfound.rate")); + metrics.add(new Metric("vds.distributor.puts.latency.max")); + metrics.add(new Metric("vds.distributor.puts.latency.sum")); + metrics.add(new Metric("vds.distributor.puts.latency.count")); + metrics.add(new Metric("vds.distributor.puts.ok.rate")); + metrics.add(new Metric("vds.distributor.puts.failures.total.rate")); + metrics.add(new Metric("vds.distributor.puts.failures.notfound.rate")); + metrics.add(new Metric("vds.distributor.puts.failures.test_and_set_failed.rate")); + metrics.add(new Metric("vds.distributor.puts.failures.concurrent_mutations.rate")); + metrics.add(new Metric("vds.distributor.puts.failures.notconnected.rate")); + metrics.add(new Metric("vds.distributor.puts.failures.notready.rate")); + metrics.add(new Metric("vds.distributor.puts.failures.wrongdistributor.rate")); + metrics.add(new Metric("vds.distributor.puts.failures.safe_time_not_reached.rate")); + metrics.add(new Metric("vds.distributor.puts.failures.storagefailure.rate")); + metrics.add(new Metric("vds.distributor.puts.failures.timeout.rate")); + metrics.add(new Metric("vds.distributor.puts.failures.busy.rate")); + metrics.add(new Metric("vds.distributor.puts.failures.inconsistent_bucket.rate")); + metrics.add(new Metric("vds.distributor.removes.latency.max")); + metrics.add(new Metric("vds.distributor.removes.latency.sum")); + metrics.add(new Metric("vds.distributor.removes.latency.count")); + metrics.add(new Metric("vds.distributor.removes.ok.rate")); + metrics.add(new Metric("vds.distributor.removes.failures.total.rate")); + metrics.add(new Metric("vds.distributor.removes.failures.notfound.rate")); + metrics.add(new Metric("vds.distributor.removes.failures.test_and_set_failed.rate")); + metrics.add(new Metric("vds.distributor.removes.failures.concurrent_mutations.rate")); + metrics.add(new Metric("vds.distributor.updates.latency.max")); + metrics.add(new Metric("vds.distributor.updates.latency.sum")); + metrics.add(new Metric("vds.distributor.updates.latency.count")); + metrics.add(new Metric("vds.distributor.updates.ok.rate")); + metrics.add(new Metric("vds.distributor.updates.failures.total.rate")); + metrics.add(new Metric("vds.distributor.updates.failures.notfound.rate")); + metrics.add(new Metric("vds.distributor.updates.failures.test_and_set_failed.rate")); + metrics.add(new Metric("vds.distributor.updates.failures.concurrent_mutations.rate")); + metrics.add(new Metric("vds.distributor.updates.diverging_timestamp_updates.rate")); + metrics.add(new Metric("vds.distributor.removelocations.ok.rate")); + metrics.add(new Metric("vds.distributor.removelocations.failures.total.rate")); + metrics.add(new Metric("vds.distributor.gets.latency.max")); + metrics.add(new Metric("vds.distributor.gets.latency.sum")); + metrics.add(new Metric("vds.distributor.gets.latency.count")); + metrics.add(new Metric("vds.distributor.gets.ok.rate")); + metrics.add(new Metric("vds.distributor.gets.failures.total.rate")); + metrics.add(new Metric("vds.distributor.gets.failures.notfound.rate")); + metrics.add(new Metric("vds.distributor.visitor.latency.max")); + metrics.add(new Metric("vds.distributor.visitor.latency.sum")); + metrics.add(new Metric("vds.distributor.visitor.latency.count")); + metrics.add(new Metric("vds.distributor.visitor.ok.rate")); + metrics.add(new Metric("vds.distributor.visitor.failures.total.rate")); + metrics.add(new Metric("vds.distributor.visitor.failures.notready.rate")); + metrics.add(new Metric("vds.distributor.visitor.failures.notconnected.rate")); + metrics.add(new Metric("vds.distributor.visitor.failures.wrongdistributor.rate")); + metrics.add(new Metric("vds.distributor.visitor.failures.safe_time_not_reached.rate")); + metrics.add(new Metric("vds.distributor.visitor.failures.storagefailure.rate")); + metrics.add(new Metric("vds.distributor.visitor.failures.timeout.rate")); + metrics.add(new Metric("vds.distributor.visitor.failures.busy.rate")); + metrics.add(new Metric("vds.distributor.visitor.failures.inconsistent_bucket.rate")); + metrics.add(new Metric("vds.distributor.visitor.failures.notfound.rate")); metrics.add(new Metric("vds.distributor.docsstored.average")); metrics.add(new Metric("vds.distributor.bytesstored.average")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java index cfb641882f6..5c3905e3311 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java @@ -114,7 +114,6 @@ public class AttributeChangeValidator { validateAttributePredicate(id, current, next, Attribute::isFastAccess, "fast-access", result); validateAttributeProperty(id, current, next, AttributeChangeValidator::extractDictionaryType, "dictionary: btree/hash", result); validateAttributeProperty(id, current, next, AttributeChangeValidator::extractDictionaryCase, "dictionary: cased/uncased", result); - validateAttributePredicate(id, current, next, Attribute::isHuge, "huge", result); validateAttributePredicate(id, current, next, Attribute::isPaged, "paged", result); validatePagedAttributeRemoval(current, next); validateAttributeProperty(id, current, next, Attribute::densePostingListThreshold, "dense-posting-list-threshold", result); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java index 6f94b9295ae..b9ed8a3c97c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java @@ -1,16 +1,19 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder; -import com.yahoo.config.model.deploy.ConfigDefinitionStore; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.deploy.ConfigDefinitionStore; import com.yahoo.config.model.producer.UserConfigRepo; -import java.util.logging.Level; import com.yahoo.text.XML; -import com.yahoo.vespa.config.*; +import com.yahoo.vespa.config.ConfigDefinition; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.ConfigPayloadBuilder; import com.yahoo.vespa.model.builder.xml.dom.DomConfigPayloadBuilder; import org.w3c.dom.Element; - -import java.util.*; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -39,7 +42,7 @@ public class UserConfigBuilder { logger.logApplicationPackage(Level.WARNING, "Unable to find config definition '" + key.asFileName() + "'. Please ensure that the name is spelled correctly, and that the def file is included in a bundle."); } - ConfigPayloadBuilder payloadBuilder = new DomConfigPayloadBuilder(def.orElse(null), logger).build(element); + ConfigPayloadBuilder payloadBuilder = new DomConfigPayloadBuilder(def.orElse(null)).build(element); ConfigPayloadBuilder old = builderMap.get(key); if (old != null) { logger.logApplicationPackage(Level.WARNING, "Multiple overrides for " + key + " found. Applying in the order they are discovered"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java index ff41a700230..2cf32f1e8ff 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java @@ -6,9 +6,6 @@ import com.yahoo.config.model.ConfigModelRepo; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.ApplicationConfigProducerRoot; -import com.yahoo.vespa.model.generic.service.ServiceCluster; - -import java.util.List; /** * Base class for classes capable of building vespa model. @@ -18,7 +15,6 @@ import java.util.List; public abstract class VespaModelBuilder { public abstract ApplicationConfigProducerRoot getRoot(String name, DeployState deployState, AbstractConfigProducer parent); - public abstract List<ServiceCluster> getClusters(DeployState pkg, AbstractConfigProducer parent); /** * Processing that requires access across plugins diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index 8e7c543b67b..e081848d01c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -107,13 +107,8 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { SimpleConfigProducer<?> configServers = new SimpleConfigProducer<>(parent, "configservers"); Element configserversE = XML.getChild(adminE, "configservers"); if (configserversE == null) { - Element configserverE = XML.getChild(adminE, "configserver"); - if (configserverE == null) - configserverE = XML.getChild(adminE, "adminserver"); - else - deployState.getDeployLogger().logApplicationPackage(Level.INFO, - "Specifying configserver without parent element configservers in services.xml is deprecated and will be removed in Vespa 8"); - return List.of(new ConfigserverBuilder(0, configServerSpecs).build(deployState, configServers, configserverE)); + Element adminserver = XML.getChild(adminE, "adminserver"); + return List.of(new ConfigserverBuilder(0, configServerSpecs).build(deployState, configServers, adminserver)); } else { List<Configserver> configservers = new ArrayList<>(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientsBuilder.java deleted file mode 100644 index fe22f818e32..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientsBuilder.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.builder.xml.dom; - -import com.yahoo.config.model.ConfigModelContext; -import com.yahoo.config.model.builder.xml.ConfigModelId; -import com.yahoo.vespa.model.clients.Clients; -import org.w3c.dom.Element; - -import java.util.Arrays; -import java.util.List; - -/** - * Builds the Clients plugin - * - * @author hmusum - */ -public class DomClientsBuilder extends LegacyConfigModelBuilder<Clients> { - - public DomClientsBuilder() { - super(Clients.class); - } - - @Override - public List<ConfigModelId> handlesElements() { - return Arrays.asList(ConfigModelId.fromNameAndVersion("clients", "2.0")); - } - - @Override - public void doBuild(Clients clients, Element clientsE, ConfigModelContext modelContext) { - String version = clientsE.getAttribute("version"); - if (version.startsWith("2.")) { - DomV20ClientsBuilder parser = new DomV20ClientsBuilder(clients, version); - parser.build(clientsE); - } else { - throw new IllegalArgumentException("Version '" + version + "' of 'clients' not supported."); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java index 2174616bacf..7ff01cbf82e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java @@ -3,19 +3,14 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.collections.Tuple2; import com.yahoo.config.ConfigurationRuntimeException; -import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.text.XML; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayloadBuilder; -import com.yahoo.yolean.Exceptions; -import com.yahoo.text.XML; - import com.yahoo.vespa.config.util.ConfigUtils; +import com.yahoo.yolean.Exceptions; import org.w3c.dom.Element; - import java.util.List; -import java.util.Optional; -import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -34,11 +29,9 @@ public class DomConfigPayloadBuilder { /** The config definition, not null if not found */ private final ConfigDefinition configDefinition; - private final Optional<DeployLogger> logger; - public DomConfigPayloadBuilder(ConfigDefinition configDefinition, DeployLogger logger) { + public DomConfigPayloadBuilder(ConfigDefinition configDefinition) { this.configDefinition = configDefinition; - this.logger = Optional.ofNullable(logger); } /** @@ -135,12 +128,7 @@ public class DomConfigPayloadBuilder { throw new ConfigurationRuntimeException("Element '" + name + "' must have either children or a value"); } - if (element.hasAttribute("operation")) { - // leaf array, currently the only supported operation is 'append' - verifyLegalOperation(element); - ConfigPayloadBuilder.Array a = payloadBuilder.getArray(name); - a.append(value); - } else if ("item".equals(name)) { + if ("item".equals(name)) { if (parentName == null) throw new ConfigurationRuntimeException("<item> is a reserved keyword for array and map elements"); if (element.hasAttribute("key")) { @@ -157,16 +145,7 @@ public class DomConfigPayloadBuilder { private void parseComplex(Element element, List<Element> children, ConfigPayloadBuilder payloadBuilder, String parentName) { String name = extractName(element); // Inner value - if (element.hasAttribute("operation")) { - // inner array, currently the only supported operation is 'append' - verifyLegalOperation(element); - ConfigPayloadBuilder childPayloadBuilder = payloadBuilder.getArray(name).append(); - //Cursor array = node.setArray(name); - for (Element child : children) { - //Cursor struct = array.addObject(); - parseElement(child, childPayloadBuilder, name); - } - } else if ("item".equals(name)) { + if ("item".equals(name)) { // Reserved item means array/map element as struct if (element.hasAttribute("key")) { ConfigPayloadBuilder childPayloadBuilder = payloadBuilder.getMap(parentName).get(element.getAttribute("key")); @@ -222,14 +201,4 @@ public class DomConfigPayloadBuilder { } } - private void verifyLegalOperation(Element currElem) { - logger.ifPresent(log -> log.logApplicationPackage( - Level.WARNING, "The 'operation' attribute is deprecated for removal in Vespa 8. Use 'item' instead.")); - - String operation = currElem.getAttribute("operation"); - if (! operation.equalsIgnoreCase("append")) - throw new ConfigurationRuntimeException("The only supported array operation is 'append', got '" - + operation + "' at XML node '" + XML.getNodePath(currElem, " > ") + "'."); - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java index bbc18ab2254..025a6377b04 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java @@ -51,14 +51,6 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder< for (Element binding : XML.getChildren(handlerElement, "binding")) addServerBinding(handler, UserBindingPattern.fromPattern(XML.getValue(binding)), deployState.getDeployLogger()); - List<Element> clientBindingsElements = XML.getChildren(handlerElement, "clientBinding"); - if (! clientBindingsElements.isEmpty()) { - deployState.getDeployLogger().logApplicationPackage( - Level.WARNING, "The 'clientBindings' element is deprecated for removal in Vespa 8, with no replacement"); - } - for (Element clientBinding : clientBindingsElements) - handler.addClientBindings(UserBindingPattern.fromPattern(XML.getValue(clientBinding))); - DomComponentBuilder.addChildren(deployState, parent, handlerElement, handler); return handler; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java deleted file mode 100644 index 040291cfa71..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.builder.xml.dom; - -import com.yahoo.text.XML; -import com.yahoo.vespa.model.clients.Clients; -import org.w3c.dom.Element; -import org.w3c.dom.NodeList; - -/** - * Builds the Clients plugin - * - * @author vegardh - */ -public class DomV20ClientsBuilder { - - // The parent docproc plugin to register data with. - private final Clients clients; - - DomV20ClientsBuilder(Clients clients, String version) { - if ( ! version.equals("2.0")) - throw new IllegalArgumentException("Version '" + version + "' of 'clients' not supported."); - this.clients = clients; - } - - public void build(Element spec) { - NodeList children = spec.getElementsByTagName("load-types"); - for (int i = 0; i < children.getLength(); i++) { - createLoadTypes((Element) children.item(i), clients); - } - } - - private void createLoadTypes(Element element, Clients clients) { - for (Element e : XML.getChildren(element, "type")) { - String priority = e.getAttribute("default-priority"); - clients.getLoadTypes().addType(e.getAttribute("name"), priority.length() > 0 ? priority : null); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index 640bede6b62..472e15add98 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -60,13 +60,17 @@ public class NodesSpecification { /** The cloud account to use for nodes in this spec, if any */ private final Optional<CloudAccount> cloudAccount; + /* Whether the count attribute was present on the nodes element. */ + private final boolean hasCountAttribute; + private NodesSpecification(ClusterResources min, ClusterResources max, boolean dedicated, Version version, boolean required, boolean canFail, boolean exclusive, Optional<DockerImage> dockerImageRepo, Optional<String> combinedId, - Optional<CloudAccount> cloudAccount) { + Optional<CloudAccount> cloudAccount, + boolean hasCountAttribute) { if (max.smallerThan(min)) throw new IllegalArgumentException("Min resources must be larger or equal to max resources, but " + max + " is smaller than " + min); @@ -89,6 +93,7 @@ public class NodesSpecification { this.dockerImageRepo = dockerImageRepo; this.combinedId = combinedId; this.cloudAccount = cloudAccount; + this.hasCountAttribute = hasCountAttribute; } private static NodesSpecification create(boolean dedicated, boolean canFail, Version version, @@ -97,6 +102,7 @@ public class NodesSpecification { var resolvedElement = resolveElement(nodesElement); var combinedId = findCombinedId(nodesElement, resolvedElement); var resources = toResources(resolvedElement); + boolean hasCountAttribute = resolvedElement.stringAttribute("count") != null; return new NodesSpecification(resources.getFirst(), resources.getSecond(), dedicated, @@ -106,7 +112,8 @@ public class NodesSpecification { resolvedElement.booleanAttribute("exclusive", false), dockerImageToUse(resolvedElement, dockerImageRepo), combinedId, - cloudAccount); + cloudAccount, + hasCountAttribute); } private static Pair<ClusterResources, ClusterResources> toResources(ModelElement nodesElement) { @@ -180,7 +187,8 @@ public class NodesSpecification { false, context.getDeployState().getWantedDockerImageRepo(), Optional.empty(), - context.getDeployState().getProperties().cloudAccount()); + context.getDeployState().getProperties().cloudAccount(), + false); } /** Returns a requirement from <code>count</code> dedicated nodes in one group */ @@ -194,7 +202,8 @@ public class NodesSpecification { false, context.getDeployState().getWantedDockerImageRepo(), Optional.empty(), - context.getDeployState().getProperties().cloudAccount()); + context.getDeployState().getProperties().cloudAccount(), + false); } /** @@ -219,7 +228,8 @@ public class NodesSpecification { false, context.getDeployState().getWantedDockerImageRepo(), Optional.empty(), - context.getDeployState().getProperties().cloudAccount()); + context.getDeployState().getProperties().cloudAccount(), + false); } public ClusterResources minResources() { return min; } @@ -238,6 +248,11 @@ public class NodesSpecification { */ public boolean isExclusive() { return exclusive; } + /** Returns whether the count attribute was present on the {@code <nodes>} element. */ + public boolean hasCountAttribute() { + return hasCountAttribute; + } + public Map<HostResource, ClusterMembership> provision(HostSystem hostSystem, ClusterSpec.Type clusterType, ClusterSpec.Id clusterId, @@ -376,12 +391,8 @@ public class NodesSpecification { /** Returns the ID of the parent container element of nodesElement, if any */ private static Optional<String> containerIdOf(ModelElement nodesElement) { var element = nodesElement.getXml(); - for (var containerTag : List.of("container", "jdisc")) { - var container = findParentByTag(containerTag, element); - if (container.isEmpty()) continue; - return container.map(el -> el.getAttribute("id")); - } - return Optional.empty(); + var container = findParentByTag("container", element); + return container.map(el -> el.getAttribute("id")); } /** Returns the ID of the container element referencing nodesElement, if any */ @@ -451,5 +462,4 @@ public class NodesSpecification { return "specification of " + (dedicated ? "dedicated " : "") + (min.equals(max) ? min : "min " + min + " max " + max); } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java index b619210155f..e2bfcb66f37 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -20,15 +20,11 @@ import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import com.yahoo.vespa.model.content.Content; -import com.yahoo.vespa.model.generic.builder.DomServiceClusterBuilder; -import com.yahoo.vespa.model.generic.service.ServiceCluster; import com.yahoo.vespa.model.search.SearchCluster; -import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.logging.Level; @@ -41,7 +37,6 @@ import java.util.logging.Logger; */ public class VespaDomBuilder extends VespaModelBuilder { - public static final String JVMARGS_ATTRIB_NAME = "jvmargs"; public static final String JVM_OPTIONS = "jvm-options"; public static final String OPTIONS = "options"; public static final String JVM_GC_OPTIONS = "jvm-gc-options"; @@ -145,10 +140,6 @@ public class VespaDomBuilder extends VespaModelBuilder { if (producerSpec != null) { if (producerSpec.hasAttribute(JVM_OPTIONS)) { t.appendJvmOptions(producerSpec.getAttribute(JVM_OPTIONS)); - } else { - if (producerSpec.hasAttribute(JVMARGS_ATTRIB_NAME)) { - t.appendJvmOptions(producerSpec.getAttribute(JVMARGS_ATTRIB_NAME)); - } } if (producerSpec.hasAttribute(PRELOAD_ATTRIB_NAME)) { t.setPreLoad(producerSpec.getAttribute(PRELOAD_ATTRIB_NAME)); @@ -300,15 +291,4 @@ public class VespaDomBuilder extends VespaModelBuilder { } } - @Override - public List<ServiceCluster> getClusters(DeployState deployState, AbstractConfigProducer parent) { - List<ServiceCluster> clusters = new ArrayList<>(); - Document services = XmlHelper.getDocument(deployState.getApplicationPackage().getServices(), "services.xml"); - for (Element clusterSpec : XML.getChildren(services.getDocumentElement(), "cluster")) { - DomServiceClusterBuilder clusterBuilder = new DomServiceClusterBuilder(clusterSpec.getAttribute("name")); - clusters.add(clusterBuilder.build(deployState, parent.getRoot(), clusterSpec)); - } - return clusters; - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomChainsBuilder.java index 2bc7e8f2658..b40ee7f3136 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomChainsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomChainsBuilder.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom.chains; -import com.yahoo.config.application.Xml; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; @@ -26,16 +25,10 @@ class DomChainsBuilder<COMPONENT extends ChainedComponent<?>, CHAIN extends Chai extends VespaDomBuilder.DomConfigProducerBuilder<CHAINS> { private final Collection<ComponentType<COMPONENT>> allowedComponentTypes; - private final String appPkgChainsDir; - private final Element outerChainsElem; - protected DomChainsBuilder(Element outerChainsElem, - Collection<ComponentType<COMPONENT>> allowedComponentTypes, - String appPkgChainsDir) { + protected DomChainsBuilder(Collection<ComponentType<COMPONENT>> allowedComponentTypes) { - this.outerChainsElem = outerChainsElem; this.allowedComponentTypes = new ArrayList<>(allowedComponentTypes); - this.appPkgChainsDir = appPkgChainsDir; } protected abstract CHAINS newChainsInstance(AbstractConfigProducer<?> parent); @@ -58,13 +51,8 @@ class DomChainsBuilder<COMPONENT extends ChainedComponent<?>, CHAIN extends Chai private List<Element> allChainElements(DeployState deployState, Element chainsElement) { List<Element> chainsElements = new ArrayList<>(); - if (outerChainsElem != null) - chainsElements.add(outerChainsElem); chainsElements.add(chainsElement); - if (appPkgChainsDir != null) - chainsElements.addAll(Xml.allElemsFromPath(deployState.getApplicationPackage(), appPkgChainsDir)); - return chainsElements; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/InheritanceBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/InheritanceBuilder.java index 1d32c644f97..fb11ba32a1e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/InheritanceBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/InheritanceBuilder.java @@ -18,38 +18,22 @@ public class InheritanceBuilder { public InheritanceBuilder(Element spec) { inheritance = new ChainSpecification.Inheritance( - // XXX: for this to work, the tagname in the spec must match the tagname inside the 'inherits' elem, e.g. 'searchchain->inherits->searchchain' - read(spec, "inherits", spec.getTagName()), - read(spec, "excludes", "exclude")); + read(spec, "inherits"), + read(spec, "excludes")); } public ChainSpecification.Inheritance build() { return inheritance; } - private Set<ComponentSpecification> read(Element spec, String attributeName, String elementName) { + private Set<ComponentSpecification> read(Element spec, String attributeName) { Set<ComponentSpecification> componentSpecifications = new LinkedHashSet<>(); componentSpecifications.addAll(spaceSeparatedComponentSpecificationsFromAttribute(spec, attributeName)); - // TODO: the 'inherits' element is undocumented, and can be removed in an upcoming version of Vespa - componentSpecifications.addAll(idRefFromElements(XML.getChild(spec, "inherits"), elementName)); - - return componentSpecifications; } - private Collection<ComponentSpecification> idRefFromElements(Element spec, String elementName) { - Collection<ComponentSpecification> result = new ArrayList<>(); - if (spec == null) - return result; - - for (Element element : XML.getChildren(spec, elementName)) { - result.add(XmlHelper.getIdRef(element)); - } - return result; - } - private Collection<ComponentSpecification> spaceSeparatedComponentSpecificationsFromAttribute(Element spec, String attributeName) { return toComponentSpecifications(XmlHelper.spaceSeparatedSymbolsFromAttribute(spec, attributeName)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocprocChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocprocChainsBuilder.java index ae22a6971bb..85f9d305afc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocprocChainsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocprocChainsBuilder.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom.chains.docproc; -import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder.ComponentType; @@ -20,8 +19,8 @@ import java.util.Map; */ public class DomDocprocChainsBuilder extends DomChainsBuilder<DocumentProcessor, DocprocChain, DocprocChains> { public DomDocprocChainsBuilder(Element outerChainsElem, boolean supportDocprocChainsDir) { - super(outerChainsElem, List.of(ComponentType.documentprocessor), - supportDocprocChainsDir ? ApplicationPackage.DOCPROCCHAINS_DIR: null); + super(List.of(ComponentType.documentprocessor) + ); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessingBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessingBuilder.java index d29643c78e2..bd7b6dd4716 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessingBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessingBuilder.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom.chains.processing; -import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder; @@ -21,7 +20,7 @@ import java.util.Map; public class DomProcessingBuilder extends DomChainsBuilder<Processor, ProcessingChain, ProcessingChains> { public DomProcessingBuilder(Element outerChainsElem) { - super(outerChainsElem, List.of(ComponentsBuilder.ComponentType.processor), ApplicationPackage.PROCESSORCHAINS_DIR); + super(List.of(ComponentsBuilder.ComponentType.processor)); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilder.java index d4bd93f67c7..798986273d0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilder.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.builder.xml.dom.chains.search; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder.ComponentType; import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainsBuilder; import com.yahoo.vespa.model.container.search.searchchain.SearchChain; @@ -22,14 +21,8 @@ import java.util.Map; */ public class DomSearchChainsBuilder extends DomChainsBuilder<Searcher<?>, SearchChain, SearchChains> { - public DomSearchChainsBuilder(Element outerChainsElem, boolean supportSearchChainsDir) { - super(outerChainsElem, Arrays.asList(ComponentType.searcher, ComponentType.federation), - supportSearchChainsDir ? ApplicationPackage.SEARCHCHAINS_DIR: null); - } - - // For unit testing without outer chains public DomSearchChainsBuilder() { - this(null, false); + super(Arrays.asList(ComponentType.searcher, ComponentType.federation)); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java index e13e741dd5d..65fc585c7af 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java @@ -25,9 +25,8 @@ public class SearchChainsBuilder extends ChainsBuilder<Searcher<?>, SearchChain> private static final Map<String, Class<? extends DomChainBuilderBase<? extends Searcher<?>, ? extends SearchChain>>> chainType2builderClass = Collections.unmodifiableMap( - new LinkedHashMap<String, Class<? extends DomChainBuilderBase<? extends Searcher<?>, ? extends SearchChain>>>() {{ + new LinkedHashMap<>() {{ put("chain", DomSearchChainBuilder.class); - put("searchchain", DomSearchChainBuilder.class); put("provider", DomProviderBuilder.class); }}); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java deleted file mode 100644 index 8b8b9e5f40d..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.clients; - -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.config.model.ConfigModel; -import com.yahoo.config.model.ConfigModelContext; -import com.yahoo.documentapi.messagebus.loadtypes.LoadType; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; - -/** - * This is the clients plugin for the Vespa model. It is responsible for creating - * all clients services. - * - * @author Gunnar Gauslaa Bergem - */ -@SuppressWarnings("removal") // TODO: Remove on Vespa 8 -public class Clients extends ConfigModel { - - private static final long serialVersionUID = 1L; - private LoadTypeSet loadTypes = new LoadTypeSet(); - - public Clients(ConfigModelContext modelContext) { - super(modelContext); - } - - public LoadTypeSet getLoadTypes() { - return loadTypes; - } - - public void getConfig(LoadTypeConfig.Builder builder) { - for (LoadType t : loadTypes.getNameMap().values()) { - if (t != LoadType.DEFAULT) { - builder.type(getLoadTypeConfig(t)); - } - } - } - - private LoadTypeConfig.Type.Builder getLoadTypeConfig(LoadType loadType) { - LoadTypeConfig.Type.Builder builder = new LoadTypeConfig.Type.Builder(); - builder.name(loadType.getName()); - builder.id(loadType.getId()); - builder.priority(loadType.getPriority().toString()); - return builder; - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index ba5e8ed5544..52a51a2c323 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -38,7 +38,7 @@ public final class ApplicationContainer extends Container implements super(parent, name, retired, index, deployState); this.isHostedVespa = deployState.isHosted(); this.enableServerOcspStapling = deployState.featureFlags().enableServerOcspStapling(); - this.useQrserverServiceName = deployState.featureFlags().useQrserverServiceName(); + this.useQrserverServiceName = false; addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerHolder")); addComponent(new SimpleComponent("com.yahoo.container.jdisc.messagebus.NetworkMultiplexerProvider")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index bfa19cfe90c..2b6efab3389 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -108,7 +108,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat addSimpleComponent("com.yahoo.language.provider.DefaultLinguisticsProvider"); addSimpleComponent("com.yahoo.language.provider.DefaultEmbedderProvider"); addSimpleComponent("com.yahoo.container.jdisc.SecretStoreProvider"); - addSimpleComponent("com.yahoo.container.jdisc.DeprecatedSecretStoreProvider"); addSimpleComponent("com.yahoo.container.jdisc.CertificateStoreProvider"); addSimpleComponent("com.yahoo.container.jdisc.AthenzIdentityProviderProvider"); addSimpleComponent(com.yahoo.container.core.documentapi.DocumentAccessProvider.class.getName()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 7f7e608aa19..79412368eae 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -47,7 +47,6 @@ import com.yahoo.vespa.model.container.component.DiscBindingsConfigGenerator; import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SimpleComponent; -import com.yahoo.vespa.model.container.component.StatisticsComponent; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; import com.yahoo.vespa.model.container.configserver.ConfigserverCluster; @@ -143,8 +142,6 @@ public abstract class ContainerCluster<CONTAINER extends Container> private final Set<Path> platformBundles = new LinkedHashSet<>(); - private final List<String> serviceAliases = new ArrayList<>(); - private final List<String> endpointAliases = new ArrayList<>(); private final ComponentGroup<Component<?, ?>> componentGroup; private final boolean isHostedVespa; private final boolean zooKeeperLocalhostAffinity; @@ -174,12 +171,6 @@ public abstract class ContainerCluster<CONTAINER extends Container> componentGroup = new ComponentGroup<>(this, "component"); addCommonVespaBundles(); - - // TODO Vespa 8: remove LoggingRequestHandler.Context component if we can break binary compatibility - // (ThreadedHttpRequestHandler.Context is source compatible.) - addSimpleComponent("com.yahoo.container.jdisc.LoggingRequestHandler$Context"); - - addComponent(new StatisticsComponent()); addSimpleComponent(AccessLog.class); addComponent(new DefaultThreadpoolProvider(this, defaultPoolNumThreads)); addSimpleComponent(com.yahoo.concurrent.classlock.ClassLocking.class); @@ -195,7 +186,6 @@ public abstract class ContainerCluster<CONTAINER extends Container> addSimpleComponent(com.yahoo.container.handler.ClustersStatus.class.getName()); addSimpleComponent("com.yahoo.container.jdisc.DisabledConnectionLogProvider"); addSimpleComponent(com.yahoo.jdisc.http.server.jetty.Janitor.class); - addJaxProviders(); } public ClusterSpec.Id id() { return ClusterSpec.Id.from(getName()); } @@ -247,19 +237,6 @@ public abstract class ContainerCluster<CONTAINER extends Container> addComponent(vipHandler); } - @SuppressWarnings("deprecation") - private void addJaxProviders() { - addSimpleComponent(com.yahoo.container.xml.providers.DatatypeFactoryProvider.class); - addSimpleComponent(com.yahoo.container.xml.providers.DocumentBuilderFactoryProvider.class); - addSimpleComponent(com.yahoo.container.xml.providers.SAXParserFactoryProvider.class); - addSimpleComponent(com.yahoo.container.xml.providers.SchemaFactoryProvider.class); - addSimpleComponent(com.yahoo.container.xml.providers.TransformerFactoryProvider.class); - addSimpleComponent(com.yahoo.container.xml.providers.XMLEventFactoryProvider.class); - addSimpleComponent(com.yahoo.container.xml.providers.XMLInputFactoryProvider.class); - addSimpleComponent(com.yahoo.container.xml.providers.XMLOutputFactoryProvider.class); - addSimpleComponent(com.yahoo.container.xml.providers.XPathFactoryProvider.class); - } - public final void addComponent(Component<?, ?> component) { componentGroup.addComponent(component); } @@ -618,12 +595,6 @@ public abstract class ContainerCluster<CONTAINER extends Container> public Map<String, String> concreteDocumentTypes() { return concreteDocumentTypes; } - /** The configured service aliases for the service in this cluster */ - public List<String> serviceAliases() { return serviceAliases; } - - /** The configured endpoint aliases (fqdn) for the service in this cluster */ - public List<String> endpointAliases() { return endpointAliases; } - public void setHostClusterId(String clusterId) { hostClusterId = clusterId; } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java index 8a489ea9de4..e5125fe7e1d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java @@ -60,8 +60,6 @@ public class PlatformBundles { "com.yahoo.docproc.AbstractConcreteDocumentFactory", "com.yahoo.docproc.DocumentProcessor", "com.yahoo.docproc.SimpleDocumentProcessor", - "com.yahoo.docproc.util.JoinerDocumentProcessor", - "com.yahoo.docproc.util.SplitterDocumentProcessor", "com.yahoo.example.TimingSearcher", "com.yahoo.language.simple.SimpleLinguistics", "com.yahoo.prelude.cluster.ClusterSearcher", @@ -125,8 +123,6 @@ public class PlatformBundles { "com.yahoo.search.searchchain.ForkingSearcher", "com.yahoo.search.searchchain.example.ExampleSearcher", "com.yahoo.search.searchers.CacheControlSearcher", - "com.yahoo.search.statistics.PeakQpsSearcher", - "com.yahoo.search.statistics.TimingSearcher", "com.yahoo.vespa.streamingvisitors.MetricsSearcher", "com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher" ); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java index e29ca9399e5..cc0415a7630 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java @@ -38,7 +38,7 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL this(logType, compressionType, clusterName.isEmpty() ? String.format("logs/vespa/access/%s.%s", capitalize(logType.name()), "%Y%m%d%H%M%S") : - // TODO: Clean up after Vespa 8 + // TODO: Vespa > 8: Clean up VespaVersion.major == 7 ? String.format("logs/vespa/qrs/%s.%s.%s", capitalize(logType.name()), clusterName.get(), "%Y%m%d%H%M%S") : String.format("logs/vespa/access/%s.%s.%s", capitalize(logType.name()), clusterName.get(), "%Y%m%d%H%M%S"), null, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/HttpFilter.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/HttpFilter.java index 71446438b06..e77099a0598 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/HttpFilter.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/HttpFilter.java @@ -10,7 +10,7 @@ import com.yahoo.osgi.provider.model.ComponentModel; /** * This is only for the legacy certificate filter setup, outside http. * - * TODO: Remove when 'filter' directly under 'jdisc' can be removed from services.xml + * TODO: Remove when 'filter' directly under 'container' can be removed from services.xml * * @author Tony Vaagenes */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/StatisticsComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/StatisticsComponent.java deleted file mode 100644 index 2236d30083b..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/StatisticsComponent.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.component; - -import com.yahoo.vespa.model.admin.monitoring.Monitoring; -import com.yahoo.container.StatisticsConfig; - -/** - * @author Tony Vaagenes - */ -public class StatisticsComponent extends SimpleComponent implements StatisticsConfig.Producer { - - public StatisticsComponent() { - super("com.yahoo.statistics.StatisticsImpl"); - } - - @Override - public void getConfig(StatisticsConfig.Builder builder) { - Monitoring monitoring = getMonitoringService(); - if (monitoring != null) { - builder. - collectionintervalsec(monitoring.getIntervalSeconds().doubleValue()). - loggingintervalsec(monitoring.getIntervalSeconds().doubleValue()); - } - builder.values(new StatisticsConfig.Values.Builder(). - name("query_latency"). - operations(new StatisticsConfig.Values.Operations.Builder(). - name(StatisticsConfig.Values.Operations.Name.REGULAR). - arguments(new StatisticsConfig.Values.Operations.Arguments.Builder(). - key("limits"). - value("25,50,100,500")))); - - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java index 413c6f07efd..bbccdaa9453 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; -import com.yahoo.container.StatisticsConfig; import com.yahoo.container.core.VipStatusConfig; import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.net.HostName; @@ -31,7 +30,6 @@ public class ConfigserverCluster extends AbstractConfigProducer ConfigserverConfig.Producer, CuratorConfig.Producer, HealthMonitorConfig.Producer, - StatisticsConfig.Producer, VipStatusConfig.Producer, ZookeeperServerConfig.Producer { @@ -173,12 +171,6 @@ public class ConfigserverCluster extends AbstractConfigProducer } @Override - public void getConfig(StatisticsConfig.Builder builder) { - builder.collectionintervalsec(60.0); - builder.loggingintervalsec(60.0); - } - - @Override public void getConfig(HealthMonitorConfig.Builder builder) { builder.snapshot_interval(60.0); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/model/DocumentProcessorModel.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/model/DocumentProcessorModel.java index 669be44d0f6..27fea8a01b4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/model/DocumentProcessorModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/model/DocumentProcessorModel.java @@ -5,7 +5,6 @@ import com.yahoo.collections.Pair; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.component.chain.dependencies.Dependencies; import com.yahoo.component.chain.model.ChainedComponentModel; -import net.jcip.annotations.Immutable; import java.util.HashMap; import java.util.Map; @@ -13,7 +12,6 @@ import java.util.Map; /** * @author Einar M R Rosenvinge */ -@Immutable public class DocumentProcessorModel extends ChainedComponentModel { private final Map<Pair<String, String>, String> fieldNameSchemaMap = new HashMap<>(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java index 4bbc5f8f990..d85f00a5bb2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java @@ -3,9 +3,6 @@ package com.yahoo.vespa.model.container.http; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; -import com.yahoo.component.chain.dependencies.Dependencies; -import com.yahoo.component.chain.model.ChainedComponentModel; -import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerCluster; @@ -52,8 +49,6 @@ public class AccessControl { ); public static class Builder { private final String domain; - private boolean readEnabled = false; - private boolean writeEnabled = true; private ClientAuthentication clientAuthentication = ClientAuthentication.need; private final Set<BindingPattern> excludeBindings = new LinkedHashSet<>(); private Collection<Handler<?>> handlers = Collections.emptyList(); @@ -61,16 +56,6 @@ public class AccessControl { this.domain = domain; } - public Builder readEnabled(boolean readEnabled) { - this.readEnabled = readEnabled; - return this; - } - - public Builder writeEnabled(boolean writeEnabled) { - this.writeEnabled = writeEnabled; - return this; - } - public Builder excludeBinding(BindingPattern binding) { this.excludeBindings.add(binding); return this; @@ -87,26 +72,20 @@ public class AccessControl { } public AccessControl build() { - return new AccessControl(domain, writeEnabled, readEnabled, clientAuthentication, excludeBindings, handlers); + return new AccessControl(domain, clientAuthentication, excludeBindings, handlers); } } public final String domain; - public final boolean readEnabled; - public final boolean writeEnabled; public final ClientAuthentication clientAuthentication; private final Set<BindingPattern> excludedBindings; private final Collection<Handler<?>> handlers; private AccessControl(String domain, - boolean writeEnabled, - boolean readEnabled, ClientAuthentication clientAuthentication, Set<BindingPattern> excludedBindings, Collection<Handler<?>> handlers) { this.domain = domain; - this.readEnabled = readEnabled; - this.writeEnabled = writeEnabled; this.clientAuthentication = clientAuthentication; this.excludedBindings = Collections.unmodifiableSet(excludedBindings); this.handlers = handlers; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java index f5d1d3e6afd..eea3ec68cc0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java @@ -24,7 +24,7 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl private final List<FilterBinding> bindings = new CopyOnWriteArrayList<>(); private volatile JettyHttpServer httpServer; private volatile AccessControl accessControl; - private volatile boolean strictFiltering = false; // TODO Vespa 8: Enable strict filtering by default if filtering is enabled + private volatile Boolean strictFiltering; public Http(FilterChains chains) { super("http"); @@ -83,7 +83,12 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl .binding(binding.binding().patternString())); } populateDefaultFiltersConfig(builder, httpServer); - builder.strictFiltering(strictFiltering); + + // Enable strict filter by default if any filter chain/binding is configured + boolean strictFilter = this.strictFiltering == null + ? (!bindings.isEmpty() || !filterChains.allChains().allComponents().isEmpty()) + : strictFiltering; + builder.strictFiltering(strictFilter); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java index 33cbb328c52..7a463ee627f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java @@ -69,10 +69,9 @@ public class JettyHttpServer extends SimpleComponent implements ServerConfig.Pro // Enable connection log hosted Vespa builder.connectionLog(new ServerConfig.ConnectionLog.Builder().enabled(true)); } else { - // TODO Vespa 8: Remove legacy Yahoo headers builder.accessLog(new ServerConfig.AccessLog.Builder() - .remoteAddressHeaders(List.of("x-forwarded-for", "y-ra", "yahooremoteip", "client-ip")) - .remotePortHeaders(List.of("X-Forwarded-Port", "y-rp"))); + .remoteAddressHeaders(List.of("x-forwarded-for")) + .remotePortHeaders(List.of("X-Forwarded-Port"))); } configureJettyThreadpool(builder); builder.stopTimeout(300); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainsBuilder.java index c75d8225843..35a91fc95ab 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainsBuilder.java @@ -31,7 +31,7 @@ public class FilterChainsBuilder extends DomChainsBuilder<Filter, Chain<Filter>, HttpBuilder.RESPONSE_CHAIN_TAG_NAME, FilterChainBuilder.class); public FilterChainsBuilder() { - super(null, allowedComponentTypes, null); + super(allowedComponentTypes); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java index 13503906c04..f27e0b24c82 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java @@ -68,16 +68,22 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> return http; } - private AccessControl buildAccessControl(DeployState deployState, AbstractConfigProducer ancestor, Element accessControlElem) { + private AccessControl buildAccessControl(DeployState deployState, AbstractConfigProducer<?> ancestor, Element accessControlElem) { AthenzDomain domain = getAccessControlDomain(deployState, accessControlElem); AccessControl.Builder builder = new AccessControl.Builder(domain.value()); getContainerCluster(ancestor).ifPresent(builder::setHandlers); XmlHelper.getOptionalAttribute(accessControlElem, "read").ifPresent( - readAttr -> builder.readEnabled(Boolean.valueOf(readAttr))); + readAttr -> deployState.getDeployLogger() + .logApplicationPackage(Level.WARNING, + "The 'read' attribute of the 'access-control' element has no effect and is deprecated. " + + "Please remove the attribute from services.xml")); XmlHelper.getOptionalAttribute(accessControlElem, "write").ifPresent( - writeAttr -> builder.writeEnabled(Boolean.valueOf(writeAttr))); + writeAttr -> deployState.getDeployLogger() + .logApplicationPackage(Level.WARNING, + "The 'write' attribute of the 'access-control' element has no effect and is deprecated. " + + "Please remove the attribute from services.xml")); AccessControl.ClientAuthentication clientAuth = XmlHelper.getOptionalAttribute(accessControlElem, "tls-handshake-client-auth") @@ -98,7 +104,7 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> return builder.build(); } - // TODO Fail if domain is not provided through deploy properties + // TODO(tokle,bjorncs) Vespa > 8: Fail if domain is not provided through deploy properties private static AthenzDomain getAccessControlDomain(DeployState deployState, Element accessControlElem) { AthenzDomain tenantDomain = deployState.getProperties().athenzDomain().orElse(null); AthenzDomain explicitDomain = XmlHelper.getOptionalAttribute(accessControlElem, "domain") diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index dd0d74aa1e8..f971aa97e6a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -26,23 +26,21 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; import com.yahoo.container.logging.FileConnectionLog; import com.yahoo.osgi.provider.model.ComponentModel; -import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.schema.OnnxModel; import com.yahoo.schema.derived.RankProfileList; +import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.security.X509CertificateUtils; import com.yahoo.text.XML; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.HostSystem; -import com.yahoo.vespa.model.builder.xml.dom.DomClientProviderBuilder; import com.yahoo.vespa.model.builder.xml.dom.DomComponentBuilder; import com.yahoo.vespa.model.builder.xml.dom.DomHandlerBuilder; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; @@ -90,7 +88,6 @@ import com.yahoo.vespa.model.container.xml.embedder.EmbedderConfig; import com.yahoo.vespa.model.content.StorageGroup; import org.w3c.dom.Element; import org.w3c.dom.Node; - import java.net.URI; import java.security.cert.X509Certificate; import java.util.ArrayList; @@ -122,7 +119,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private static final String HOSTED_VESPA_STATUS_FILE_SETTING = "VESPA_LB_STATUS_FILE"; private static final String CONTAINER_TAG = "container"; - private static final String DEPRECATED_CONTAINER_TAG = "jdisc"; private static final String ENVIRONMENT_VARIABLES_ELEMENT = "environment-variables"; // The node count to enforce in a cluster running ZooKeeper @@ -138,8 +134,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private final boolean httpServerEnabled; protected DeployLogger log; - public static final List<ConfigModelId> configModelIds = - ImmutableList.of(ConfigModelId.fromName(CONTAINER_TAG), ConfigModelId.fromName(DEPRECATED_CONTAINER_TAG)); + public static final List<ConfigModelId> configModelIds = ImmutableList.of(ConfigModelId.fromName(CONTAINER_TAG)); private static final String xmlRendererId = RendererRegistry.xmlRendererId.getName(); private static final String jsonRendererId = RendererRegistry.jsonRendererId.getName(); @@ -164,8 +159,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { app = modelContext.getApplicationPackage(); checkVersion(spec); - checkTagName(spec, log); - checkDeprecatedAttributes(spec, log); ApplicationContainerCluster cluster = createContainerCluster(spec, modelContext); addClusterContent(cluster, spec, modelContext); @@ -175,16 +168,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { model.setCluster(cluster); } - private void checkDeprecatedAttributes(Element spec, DeployLogger log) { - String version = spec.getAttribute("jetty"); - if (!version.isEmpty()) { - log.logApplicationPackage(WARNING, - "The 'jetty' attribute is deprecated and will be removed in Vespa 8. " + - "It has no effect - Jetty is always enabled." + - "Please remove the attribute from the 'container'/'jdisc' element in services.xml."); - } - } - private ApplicationContainerCluster createContainerCluster(Element spec, ConfigModelContext modelContext) { return new VespaDomBuilder.DomConfigProducerBuilder<ApplicationContainerCluster>() { @Override @@ -218,10 +201,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addHttp(deployState, spec, cluster, context); addAccessLogs(deployState, cluster, spec); - addRoutingAliases(cluster, spec, deployState.zone().environment()); addNodes(cluster, spec, context); - addClientProviders(deployState, spec, cluster); addServerProviders(deployState, spec, cluster); // Must be added after nodes: @@ -370,21 +351,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { container.setProp("rotations", String.join(",", rotationsProperty)); } - private void addRoutingAliases(ApplicationContainerCluster cluster, Element spec, Environment environment) { - if (environment != Environment.prod) return; - - Element aliases = XML.getChild(spec, "aliases"); - if (aliases != null) { - log.logApplicationPackage(WARNING, "The 'aliases' element and its children has no effect. They have been deprecated for removal in Vespa 8"); - } - for (Element alias : XML.getChildren(aliases, "service-alias")) { - cluster.serviceAliases().add(XML.getValue(alias)); - } - for (Element alias : XML.getChildren(aliases, "endpoint-alias")) { - cluster.endpointAliases().add(XML.getValue(alias)); - } - } - private static void addEmbedderComponents(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { for (Element node : XML.getChildren(spec, "embedder")) { Element transformed = EmbedderConfig.transform(deployState, node); @@ -414,17 +380,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } } - private void addClientProviders(DeployState deployState, Element spec, ApplicationContainerCluster cluster) { - List<Element> clientElements = XML.getChildren(spec, "client"); - if (! clientElements.isEmpty()) { - log.logApplicationPackage( - Level.WARNING, "The 'client' element is deprecated for removal in Vespa 8, with no replacement"); - } - for (Element clientSpec : clientElements) { - cluster.addComponent(new DomClientProviderBuilder(cluster).build(deployState, cluster, clientSpec)); - } - } - private void addServerProviders(DeployState deployState, Element spec, ApplicationContainerCluster cluster) { addConfiguredComponents(deployState, cluster, spec, "server"); } @@ -447,7 +402,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // Add connection log if access log is configured if (cluster.getAllComponents().stream().anyMatch(component -> component instanceof AccessLogComponent)) { - // TODO: clean up after Vespa 8 + // TODO: Vespa > 8: Clean up if (cluster.isHostedVespa() || deployState.getVespaVersion().getMajor() == 8) { cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class, "access")); } else { @@ -554,8 +509,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { if (tenantDomain == null) return; // tenant domain not present, cannot add access control. this should eventually be a failure. new AccessControl.Builder(tenantDomain.value()) .setHandlers(cluster) - .readEnabled(false) - .writeEnabled(false) .clientAuthentication(AccessControl.ClientAuthentication.need) .build() .configureHttpFilterChains(http); @@ -649,7 +602,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } private ContainerSearch buildSearch(DeployState deployState, ApplicationContainerCluster containerCluster, Element producerSpec) { - SearchChains searchChains = new DomSearchChainsBuilder(null, false) + SearchChains searchChains = new DomSearchChainsBuilder() .build(deployState, containerCluster, producerSpec); ContainerSearch containerSearch = new ContainerSearch(containerCluster, searchChains, new ContainerSearch.Options()); @@ -680,13 +633,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { throw new IllegalArgumentException("Expected container version to be 1.0, but got " + version); } - private void checkTagName(Element spec, DeployLogger logger) { - if (spec.getTagName().equals(DEPRECATED_CONTAINER_TAG)) { - logger.logApplicationPackage(WARNING, "'" + DEPRECATED_CONTAINER_TAG + - "' is deprecated as tag name. Use '" + CONTAINER_TAG + "' instead."); - } - } - private void addNodes(ApplicationContainerCluster cluster, Element spec, ConfigModelContext context) { if (standaloneBuilder) addStandaloneNode(cluster, context.getDeployState()); @@ -699,12 +645,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.addContainers(Collections.singleton(container)); } - static boolean incompatibleGCOptions(String jvmargs) { - Pattern gcAlgorithm = Pattern.compile("-XX:[-+]Use.+GC"); - Pattern cmsArgs = Pattern.compile("-XX:[-+]*CMS"); - return (gcAlgorithm.matcher(jvmargs).find() || cmsArgs.matcher(jvmargs).find()); - } - private static String buildJvmGCOptions(ConfigModelContext context, String jvmGCOptions) { return new JvmGcOptions(context.getDeployState(), jvmGCOptions).build(); } @@ -767,7 +707,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { List<ApplicationContainer> nodes = createNodes(cluster, containerElement, nodesElement, context); extractJvmOptions(nodes, cluster, nodesElement, context); - applyRoutingAliasProperties(nodes, cluster); applyDefaultPreload(nodes, nodesElement); String environmentVars = getEnvironmentVariables(XML.getChild(nodesElement, ENVIRONMENT_VARIABLES_ELEMENT)); if (!environmentVars.isEmpty()) { @@ -802,19 +741,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { else // the non-hosted option return createNodesFromNodeList(context.getDeployState(), cluster, nodesElement); } - - private static void applyRoutingAliasProperties(List<ApplicationContainer> result, ApplicationContainerCluster cluster) { - if (!cluster.serviceAliases().isEmpty()) { - result.forEach(container -> { - container.setProp("servicealiases", cluster.serviceAliases().stream().collect(Collectors.joining(","))); - }); - } - if (!cluster.endpointAliases().isEmpty()) { - result.forEach(container -> { - container.setProp("endpointaliases", cluster.endpointAliases().stream().collect(Collectors.joining(","))); - }); - } - } private static void applyMemoryPercentage(ApplicationContainerCluster cluster, String memoryPercentage) { if (memoryPercentage == null || memoryPercentage.isEmpty()) return; @@ -1109,7 +1035,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } public static boolean isContainerTag(Element element) { - return CONTAINER_TAG.equals(element.getTagName()) || DEPRECATED_CONTAINER_TAG.equals(element.getTagName()); + return CONTAINER_TAG.equals(element.getTagName()); } /** @@ -1149,33 +1075,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } String buildLegacyOptions() { - String jvmOptions; + String jvmOptions = null; if (nodesElement.hasAttribute(VespaDomBuilder.JVM_OPTIONS)) { jvmOptions = nodesElement.getAttribute(VespaDomBuilder.JVM_OPTIONS); - if (nodesElement.hasAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME)) { - String jvmArgs = nodesElement.getAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME); - throw new IllegalArgumentException("You have specified both deprecated jvm-options='" + jvmOptions + "'" + - " and deprecated jvmargs='" + jvmArgs + - "'. 'jvm-options' and 'jvmargs' are deprecated and will be removed in Vespa 8." + - " Please merge 'jvmargs' into 'options' or 'gc-options' in 'jvm' element." + - " See https://docs.vespa.ai/en/reference/services-container.html#jvm"); - } if (! jvmOptions.isEmpty()) logger.logApplicationPackage(WARNING, "'jvm-options' is deprecated and will be removed in Vespa 8." + " Please merge 'jvm-options' into 'options' or 'gc-options' in 'jvm' element." + " See https://docs.vespa.ai/en/reference/services-container.html#jvm"); - } else { - jvmOptions = nodesElement.getAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME); - if (incompatibleGCOptions(jvmOptions)) { - logger.logApplicationPackage(WARNING, "You need to move your GC-related options from deprecated 'jvmargs'" + - " to 'gc-options' in 'jvm' element. 'jvmargs' is deprecated and will be removed in Vespa 8." + - " See https://docs.vespa.ai/en/reference/services-container.html#jvm"); - cluster.setJvmGCOptions(ContainerCluster.G1GC); - } - if (! jvmOptions.isEmpty()) - logger.logApplicationPackage(WARNING, "'jvmargs' is deprecated and will be removed in Vespa 8." + - " Please merge 'jvmargs' into 'options' or 'gc-options' in 'jvm' element." + - " See https://docs.vespa.ai/en/reference/services-container.html#jvm"); } validateJvmOptions(jvmOptions); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java index d32fa8f2902..d6ed6f4f7dd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.content; import com.yahoo.config.model.api.ModelContext; import com.yahoo.metrics.MetricsmanagerConfig; -import com.yahoo.vespa.config.content.LoadTypeConfig; import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig; import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.vespa.config.content.core.StorStatusConfig; @@ -16,7 +15,7 @@ import com.yahoo.vespa.model.application.validation.RestartConfigs; * Common class for config producers for storage and distributor nodes. */ @RestartConfigs({StorCommunicationmanagerConfig.class, StorStatusConfig.class, - StorServerConfig.class, LoadTypeConfig.class, MetricsmanagerConfig.class}) + StorServerConfig.class, MetricsmanagerConfig.class}) public abstract class ContentNode extends AbstractService implements StorCommunicationmanagerConfig.Producer, StorStatusConfig.Producer, StorServerConfig.Producer { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index b41d805f2a7..8a8d2742df1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -6,6 +6,7 @@ import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; import com.yahoo.vespa.config.content.StorDistributionConfig; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.HostSystem; @@ -203,9 +204,8 @@ public class StorageGroup { } public StorageGroup buildRootGroup(DeployState deployState, RedundancyBuilder redundancyBuilder, ContentCluster owner) { - Optional<Integer> maxRedundancy = Optional.empty(); if (owner.isHosted()) - maxRedundancy = validateRedundancyAndGroups(); + validateRedundancyAndGroups(deployState.zone().environment()); Optional<ModelElement> group = Optional.ofNullable(clusterElement.child("group")); Optional<ModelElement> nodes = getNodes(clusterElement); @@ -222,8 +222,7 @@ public class StorageGroup { : groupBuilder.buildNonHosted(deployState, owner, Optional.empty()); Redundancy redundancy = redundancyBuilder.build(owner.getName(), owner.isHosted(), storageGroup.subgroups.size(), - storageGroup.getNumberOfLeafGroups(), storageGroup.countNodes(false), - maxRedundancy); + storageGroup.getNumberOfLeafGroups(), storageGroup.countNodes(false)); owner.setRedundancy(redundancy); if (storageGroup.partitions.isEmpty() && (redundancy.groups() > 1)) { storageGroup.partitions = Optional.of(computePartitions(redundancy.finalRedundancy(), redundancy.groups())); @@ -231,27 +230,24 @@ public class StorageGroup { return storageGroup; } - private Optional<Integer> validateRedundancyAndGroups() { + private void validateRedundancyAndGroups(Environment environment) { var redundancyElement = clusterElement.child("redundancy"); - if (redundancyElement == null) return Optional.empty(); + if (redundancyElement == null) return; long redundancy = redundancyElement.asLong(); var nodesElement = clusterElement.child("nodes"); - if (nodesElement == null) return Optional.empty(); + if (nodesElement == null) return; var nodesSpec = NodesSpecification.from(nodesElement, context); + // Allow dev deployment of self-hosted app (w/o count attribute): absent count => 1 node + if (!nodesSpec.hasCountAttribute() && environment == Environment.dev) return; + int minNodesPerGroup = (int)Math.ceil((double)nodesSpec.minResources().nodes() / nodesSpec.minResources().groups()); - if (minNodesPerGroup < redundancy) { // TODO: Fail on this on Vespa 8, and simplify - context.getDeployLogger() - .logApplicationPackage(Level.WARNING, - "Cluster '" + clusterElement.stringAttribute("id") + "' " + - "specifies redundancy " + redundancy + " but cannot be higher than " + - "the minimum nodes per group, which is " + minNodesPerGroup); - return Optional.of(minNodesPerGroup); - } - else { - return Optional.empty(); + if (minNodesPerGroup < redundancy) { + throw new IllegalArgumentException("Cluster '" + clusterElement.stringAttribute("id") + "' " + + "specifies redundancy " + redundancy + ", but it cannot be higher than " + + "the minimum nodes per group, which is " + minNodesPerGroup); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java index d7cafafb87f..aebdfdf61cf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java @@ -27,12 +27,6 @@ public class DomTuningDispatchBuilder { builder.setDispatchPolicy(dispatchElement.childAsString("dispatch-policy")); builder.setMinActiveDocsCoverage(dispatchElement.childAsDouble("min-active-docs-coverage")); - if (dispatchElement.child("min-group-coverage") != null) // TODO: Remove on Vespa 8 - logger.logApplicationPackage(Level.WARNING, "Attribute 'min-group-coverage' is deprecated and ignored: " + - "Use min-active-docs-coverage instead."); - if (dispatchElement.child("use-local-node") != null) // TODO: Remove on Vespa 8 - logger.logApplicationPackage(Level.WARNING, "Attribute 'use-local-node' is deprecated and ignored: " + - "The local node will automatically be preferred when appropriate."); return builder.build(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java index 7b124593354..1948cc1bd71 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java @@ -5,8 +5,6 @@ import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.content.IndexedHierarchicDistributionValidator; import com.yahoo.vespa.model.content.Redundancy; -import java.util.Optional; - /** * Builds redundancy config for a content cluster. */ @@ -39,13 +37,7 @@ public class RedundancyBuilder { } } } - public Redundancy build(String clusterName, boolean isHosted, int subGroups, int leafGroups, int totalNodes, - Optional<Integer> maxRedundancy) { - if (maxRedundancy.isPresent()) { - initialRedundancy = Math.min(initialRedundancy, maxRedundancy.get()); - finalRedundancy = Math.min(finalRedundancy, maxRedundancy.get()); - readyCopies = Math.min(readyCopies, maxRedundancy.get()); - } + public Redundancy build(String clusterName, boolean isHosted, int subGroups, int leafGroups, int totalNodes) { if (isHosted) { return new Redundancy(initialRedundancy, finalRedundancy, readyCopies, leafGroups, totalNodes); } else { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java index c87fb8d120e..a3f32fcb44b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java @@ -80,18 +80,18 @@ public class StorageCluster extends AbstractConfigProducer<StorageNode> addedmetrics("vds.datastored.bucket_space.buckets_total"); ContentCluster.getMetricBuilder("log", builder). - addedmetrics("vds.filestor.alldisks.allthreads.put.sum"). - addedmetrics("vds.filestor.alldisks.allthreads.get.sum"). - addedmetrics("vds.filestor.alldisks.allthreads.remove.sum"). - addedmetrics("vds.filestor.alldisks.allthreads.update.sum"). + addedmetrics("vds.filestor.allthreads.put"). + addedmetrics("vds.filestor.allthreads.get"). + addedmetrics("vds.filestor.allthreads.remove"). + addedmetrics("vds.filestor.allthreads.update"). addedmetrics("vds.datastored.alldisks.docs"). addedmetrics("vds.datastored.alldisks.bytes"). - addedmetrics("vds.filestor.alldisks.queuesize"). - addedmetrics("vds.filestor.alldisks.averagequeuewait.sum"). + addedmetrics("vds.filestor.queuesize"). + addedmetrics("vds.filestor.averagequeuewait"). addedmetrics("vds.visitor.cv_queuewaittime"). addedmetrics("vds.visitor.allthreads.averagequeuewait"). addedmetrics("vds.visitor.allthreads.averagevisitorlifetime"). - addedmetrics("vds.visitor.allthreads.created.sum"); + addedmetrics("vds.visitor.allthreads.created"); } public String getClusterName() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/GenericServicesBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/GenericServicesBuilder.java deleted file mode 100644 index 7a0278faa48..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/generic/GenericServicesBuilder.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.generic; - -import com.yahoo.config.model.ConfigModelContext; -import com.yahoo.config.model.builder.xml.ConfigModelBuilder; -import com.yahoo.config.model.builder.xml.ConfigModelId; -import com.yahoo.vespa.model.generic.builder.DomServiceClusterBuilder; -import org.w3c.dom.Element; - -import java.util.Arrays; -import java.util.List; - -/** - * @author Ulf Lilleengen - * @since 5.1 - * - * TODO: remove in Vespa 8 - */ -public class GenericServicesBuilder extends ConfigModelBuilder<GenericServicesModel> { - - public GenericServicesBuilder() { - super(GenericServicesModel.class); - } - - @Override - public List<ConfigModelId> handlesElements() { - return List.of(ConfigModelId.fromName("service")); - } - - @Override - public void doBuild(GenericServicesModel model, Element spec, ConfigModelContext context) { - String name = spec.getAttribute("name"); - model.addCluster(new DomServiceClusterBuilder(name).build(context.getDeployState(), context.getParentProducer(), spec)); - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/GenericServicesModel.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/GenericServicesModel.java deleted file mode 100644 index 52f02aba021..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/generic/GenericServicesModel.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.generic; - -import com.yahoo.config.model.ConfigModel; -import com.yahoo.config.model.ConfigModelContext; -import com.yahoo.vespa.model.generic.service.ServiceCluster; - -import java.util.ArrayList; -import java.util.List; - -/** - * @author Ulf Lilleengen - * @since 5.1 - */ -public class GenericServicesModel extends ConfigModel { - private final List<ServiceCluster> clusters = new ArrayList<>(); - public GenericServicesModel(ConfigModelContext modelContext) { - super(modelContext); - } - - public void addCluster(ServiceCluster cluster) { - clusters.add(cluster); - } - - public List<ServiceCluster> serviceClusters() { - return clusters; - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomModuleBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomModuleBuilder.java deleted file mode 100644 index f2f930f201f..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomModuleBuilder.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.generic.builder; - -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.text.XML; -import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; -import com.yahoo.vespa.model.generic.service.Module; -import org.w3c.dom.Element; - -/** - * Produces sub services for generic services. - */ -public class DomModuleBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Module> { - - private final String name; - - public DomModuleBuilder(String name) { - this.name = name; - } - - private void addChildren(DeployState deployState, Module s, Element subServiceSpec) { - for (Element nodeSpec : XML.getChildren(subServiceSpec, "module")) { - new DomModuleBuilder(nodeSpec.getAttribute("name")).build(deployState, s, nodeSpec); - } - } - - @Override - protected Module doBuild(DeployState deployState, AbstractConfigProducer<?> ancestor, Element subServiceSpec) { - Module s = new Module(ancestor, name); - addChildren(deployState, s, subServiceSpec); - return s; - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomServiceBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomServiceBuilder.java deleted file mode 100644 index 52f13667d89..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomServiceBuilder.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.generic.builder; - -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.text.XML; -import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; -import com.yahoo.vespa.model.generic.service.Service; -import com.yahoo.vespa.model.generic.service.ServiceCluster; -import org.w3c.dom.Element; - -/** -* @author Ulf Lilleengen -* @since 5.1 -*/ -public class DomServiceBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Service> { - private final int i; - - public DomServiceBuilder(int i) { - this.i = i; - } - - @Override - protected com.yahoo.vespa.model.generic.service.Service doBuild(DeployState deployState, AbstractConfigProducer parent, - Element serviceSpec) { - ServiceCluster sc = (ServiceCluster) parent; - com.yahoo.vespa.model.generic.service.Service service = new com.yahoo.vespa.model.generic.service.Service(sc, i + ""); - for (Element subServiceSpec : XML.getChildren(serviceSpec, "module")) { - new DomModuleBuilder(subServiceSpec.getAttribute("name")).build(deployState, service, subServiceSpec); - } - return service; - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomServiceClusterBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomServiceClusterBuilder.java deleted file mode 100644 index 1ac668049f0..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomServiceClusterBuilder.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.generic.builder; - -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.text.XML; -import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; -import com.yahoo.vespa.model.generic.service.ServiceCluster; -import org.w3c.dom.Element; -import java.util.Map; -import java.util.logging.Level; - -/** - * @author Ulf Lilleengen - * - * TODO: remove in Vespa 8 - */ -public class DomServiceClusterBuilder extends VespaDomBuilder.DomConfigProducerBuilder<ServiceCluster> { - - private final String name; - - public DomServiceClusterBuilder(String name) { - this.name = name; - } - - @Override - protected ServiceCluster doBuild(DeployState deployState, AbstractConfigProducer<?> ancestor, Element spec) { - deployState.getDeployLogger().logApplicationPackage( - Level.WARNING, "The 'service' element is deprecated and will be removed in Vespa 8, without replacement."); - - ServiceCluster cluster = new ServiceCluster(ancestor, name, spec.getAttribute("command")); - int nodeIndex = 0; - for (Element nodeSpec : XML.getChildren(spec, "node")) { - com.yahoo.vespa.model.generic.service.Service service = new DomServiceBuilder(nodeIndex).build(deployState, cluster, nodeSpec); - - // TODO: Currently creates the config for each service. Should instead build module tree first - // and store them in ServiceCluster. Then have some way of referencing them from each service. - for (Element subServiceSpec : XML.getChildren(spec, "module")) { - String subServiceName = subServiceSpec.getAttribute("name"); - Map<String, AbstractConfigProducer<?>> map = service.getChildren(); - // Add only non-conflicting modules. Does not merge unspecified configs that are specified in root though. - if (!map.containsKey(subServiceName)) - new DomModuleBuilder(subServiceName).build(deployState, service, subServiceSpec); - } - nodeIndex++; - } - return cluster; - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/package-info.java deleted file mode 100644 index 3163a070607..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/generic/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.model.generic; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/service/Module.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/Module.java deleted file mode 100644 index d0b1520e905..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/generic/service/Module.java +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.generic.service; - -import com.yahoo.config.model.producer.AbstractConfigProducer; - -/** - * A simple sub service that is essentially just to have a node with a nice name - * in the tree. Could might as well have used an AbstractConfigProducer as well, - * but that makes the code very confusing to read. - * - * @author Ulf Lilleengen - */ -public class Module extends AbstractConfigProducer<Module> { - - public Module(AbstractConfigProducer<?> parent, String subId) { - super(parent, subId); - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/service/Service.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/Service.java deleted file mode 100644 index dbb2e6a45b3..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/generic/service/Service.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.generic.service; - -import com.yahoo.vespa.model.AbstractService; -import com.yahoo.vespa.model.HostResource; -import com.yahoo.vespa.model.PortAllocBridge; - -/** - * An application specific generic service - * @author vegardh - * - */ -public class Service extends AbstractService { - private static final long serialVersionUID = 1L; - - public Service(ServiceCluster parent, String id) { - super(parent, id); - setProp("clustertype", parent.getName()); - setProp("clustername", parent.getName()); - } - - @Override - public int getPortCount() { - return 0; - } - - @Override - public void allocatePorts(int start, PortAllocBridge from) { } - - @Override - public String getStartupCommand() { - return ((ServiceCluster) getParent()).getCommand(); - } - - private String getClusterName() { - return ((ServiceCluster) getParent()).getName(); - } - - /** - * Different services are represented using same class here, so we must take service name into account too - * - * @param host a host - * @return the index of the host - */ - protected int getIndex(HostResource host) { - int i = 0; - for (com.yahoo.vespa.model.Service s : host.getServices()) { - if (!s.getClass().equals(getClass())) continue; - Service other = (Service)s; - if (s!=this && other.getClusterName().equals(getClusterName())) { - i++; - } - } - return i + 1; - } - - @Override - public String getServiceType() { - return getClusterName(); - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/service/ServiceCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/ServiceCluster.java deleted file mode 100644 index 32daed92fac..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/generic/service/ServiceCluster.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.generic.service; - -import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.vespa.model.HostSystem; - -import java.util.ArrayList; -import java.util.Collection; - -/** - * A cluster of nodes running one application specific generic service. These are defined on the top level in the Vespa config - * @author vegardh - * - */ -public class ServiceCluster extends AbstractConfigProducer<ServiceCluster> { - - private static final long serialVersionUID = 1L; - private final String command; - private final String name; - private HostSystem hostSystem; // A generic cluster can resolve hosts for its nodes - - public ServiceCluster(AbstractConfigProducer<?> parent, String name, String command) { - super(parent, name); - this.command=command; - this.name=name; - } - - public String getName() { - return name; - } - - String getCommand() { - return command; - } - - public Collection<Service> services() { - Collection<Service> ret = new ArrayList<>(); - for (Object child : getChildren().values()) { - if (child instanceof Service) ret.add((Service) child); - } - return ret; - } - - @Override - public HostSystem hostSystem() { - if (hostSystem!=null) return hostSystem; - return super.hostSystem(); - } - - /** - * Sets the host system for this. - * @param hostSystem a {@link com.yahoo.vespa.model.HostSystem} - */ - public void setHostSystem(HostSystem hostSystem) { - this.hostSystem = hostSystem; - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/service/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/package-info.java deleted file mode 100644 index 2ad0df20a1a..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/generic/service/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.vespa.model.generic.service; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index e7c0968f99e..3461cab1201 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -7,7 +7,6 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.provision.NodeResources; import com.yahoo.metrics.MetricsmanagerConfig; import com.yahoo.searchlib.TranslogserverConfig; -import com.yahoo.vespa.config.content.LoadTypeConfig; import com.yahoo.vespa.config.content.StorFilestorConfig; import com.yahoo.vespa.config.content.core.StorBucketmoverConfig; import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig; @@ -44,7 +43,7 @@ import static com.yahoo.vespa.defaults.Defaults.getDefaults; @RestartConfigs({ProtonConfig.class, MetricsmanagerConfig.class, TranslogserverConfig.class, StorFilestorConfig.class, StorBucketmoverConfig.class, StorCommunicationmanagerConfig.class, StorStatusConfig.class, - StorServerConfig.class, LoadTypeConfig.class}) + StorServerConfig.class}) public class SearchNode extends AbstractService implements SearchInterface, ProtonConfig.Producer, diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj index 47b7909eb69..7ccaf956b43 100644 --- a/config-model/src/main/javacc/SchemaParser.jj +++ b/config-model/src/main/javacc/SchemaParser.jj @@ -190,12 +190,7 @@ TOKEN : | < MUTATE: "mutate" > | < QUERY: "query" > | < RANKPROFILE: "rank-profile" > -| < RANKDEGRADATIONFREQ: "rank-degradation-frequency" > -| < RANKDEGRADATION: "rank-degradation" > | < RAW_AS_BASE64_IN_SUMMARY: "raw-as-base64-in-summary" > -| < RPBINSIZE: "doc-frequency" > -| < RPBINLOW: "min-fullrank-docs"> -| < RPPOSBINSIZE: "occurrences-per-doc" > | < SUMMARY: "summary" > | < FULL: "full" > | < STATIC: "static" > @@ -229,8 +224,6 @@ TOKEN : | < CASED: "cased" > | < UNCASED: "uncased" > | < BOLDING: "bolding" > -| < BODY: "body" > -| < HEADER: "header" > | < NONE: "none" > | < ON: "on" > | < OFF: "off" > @@ -246,7 +239,6 @@ TOKEN : | < FILTER: "filter" > | < NORMAL: "normal" > | < EXACTTERMINATOR: "exact-terminator" > -| < INDEXINGREWRITE: "indexing-rewrite" > | < IGNOREDEFAULTRANKFEATURES: "ignore-default-rank-features" > | < ID: "id" > | < SOURCE: "source" > @@ -264,15 +256,9 @@ TOKEN : | < PAGED: "paged" > | < FASTRANK: "fast-rank" > | < FASTSEARCH: "fast-search" > -| < HUGE: "huge" > | < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])* ")" > | < TENSOR_VALUE_SL: "value" (" ")* ":" (" ")* ("{"<BRACE_SL_LEVEL_1>) ("\n")? > | < TENSOR_VALUE_ML: "value" (<SEARCHLIB_SKIP>)? "{" (["\n"," "])* ("{"<BRACE_ML_LEVEL_1>) (["\n"," "])* "}" ("\n")? > -| < COMPRESSION: "compression" > -| < COMPRESSIONLEVEL: "level" > -| < COMPRESSIONTHRESHOLD: "threshold" > -| < LZ4: "lz4" > -| < USEDOCUMENT: "use-document" > | < LBRACE: "{" > | < RBRACE: "}" > | < COLON: ":" > @@ -418,8 +404,7 @@ void rootSchemaItem(ParsedSchema schema) : { } | rawAsBase64(schema) | searchStemming(schema) | importField(schema) - | rankingConstant(schema) // Deprecated: TODO: Emit warning when on Vespa 8 - | useDocument(schema) + | rankingConstant(schema) // Deprecated: TODO: Vespa > 8: Emit warning | rankProfile(schema) | documentSummary(schema) | fieldOutsideDoc(schema) @@ -427,7 +412,7 @@ void rootSchemaItem(ParsedSchema schema) : { } | structOutside(schema) | annotationOutside(schema) | fieldSet(schema) - | onnxModelInSchema(schema) // Deprecated: TODO: Emit warning when on Vespa 8 + | onnxModelInSchema(schema) // Deprecated: TODO: Vespa > 8: Emit warning ) } @@ -466,17 +451,6 @@ ParsedSchema rootDocumentItem(ParsedSchema schema) : } /** - * Consumes a use-document statement. This currently does nothing. - * - * @param schema the schema object to modify. - */ -void useDocument(ParsedSchema schema) : { } -{ - <USEDOCUMENT> <COLON> identifier() - { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'use-document' is deprecated and has no effect."); } -} - -/** * Consumes a document element. The name defaults to the schema's name, but may be set. * * @param schema the schema object to add content to. @@ -519,10 +493,7 @@ ParsedDocument namedDocument() : */ void documentBody(ParsedDocument document) : { } { - ( compression() - | headercfg() - | bodycfg() - | annotation(document) + ( annotation(document) | structInside(document) | fieldInsideDoc(document) ) @@ -544,44 +515,6 @@ void rawAsBase64(ParsedSchema schema) : } /** - * Consumes a document head block. - */ -void headercfg() : { } -{ - <HEADER> lbrace() [compression() (<NL>)*] <RBRACE> -} - -/** - * Consumes a document body block. - */ -void bodycfg() : { } -{ - <BODY> lbrace() [compression() (<NL>)*] <RBRACE> -} - -/** - * Consumes a compression block. This can be set in both document header and -body block. - */ -void compression() : -{ - deployLogger.logApplicationPackage(Level.WARNING, "'compression' for a document is deprecated and ignored"); -} -{ - <COMPRESSION> lbrace() ( compressionItem() (<NL>)*)* <RBRACE> -} - -/** - * Consumes the body of a compression block. - */ -void compressionItem() : { } -{ - ( ( <TYPE> <COLON> <LZ4> ) - | (<COMPRESSIONTHRESHOLD> <COLON> <INTEGER>) - | (<COMPRESSIONLEVEL> <COLON> <INTEGER>) - ) -} - -/** * Consumes struct inheritance * * @param struct The struct type to modify. @@ -907,16 +840,13 @@ void fieldBody(ParsedField field) : { } { ( alias(field) | attribute(field) | - body() | bolding(field) | dictionary(field) | fieldStemming(field) | - header() | id(field) | summaryInField(field) | indexInsideField(field) | indexing(field) | - indexingRewrite(field) | matchSettings(field.matchSettings()) | normalizing(field) | queryCommand(field) | @@ -1115,8 +1045,7 @@ void attributeSetting(ParsedAttribute attribute) : } { ( - <HUGE> { attribute.setHuge(true); } - | <FASTRANK> { attribute.setFastRank(true); } + <FASTRANK> { attribute.setFastRank(true); } | <FASTSEARCH> { attribute.setFastSearch(true); } | <FASTACCESS> { attribute.setFastAccess(true); } | <MUTABLE> { attribute.setMutable(true); } @@ -1127,29 +1056,11 @@ void attributeSetting(ParsedAttribute attribute) : | <ALIAS> { String alias; String aliasedName=attribute.name(); } [aliasedName = identifier()] <COLON> alias = identifierWithDash() { attribute.addAlias(aliasedName, alias); } - | attributeTensorType(attribute) | <DISTANCEMETRIC> <COLON> str = identifierWithDash() { attribute.setDistanceMetric(str); } ) } /** - * This rule consumes a tensor type statement for an attribute element. - * - * @param attribute The attribute to modify. - */ -void attributeTensorType(ParsedAttribute attribute) : -{ - TensorType tensorType; -} -{ - tensorType = tensorType("For attribute field '" + attribute.name() + "'") - { - // TODO: Remove on Vespa 8 - deployLogger.logApplicationPackage(Level.WARNING, "In field '" + attribute.name() + "': Specifying tensor type on the attribute is deprecated and has no effect."); - } -} - -/** * This rule consumes a summary statement defined inside a document-summary block. * * @param docsum The document summary to modify. @@ -1235,7 +1146,7 @@ void summaryItem(ParsedSummaryField field) : { } | summaryBolding(field) | summarySourceList(field) | summaryDestinationList(field) - | summaryProperties(field) ) + ) } /** @@ -1294,30 +1205,6 @@ void summaryDestinationList(ParsedSummaryField field) : } /** - * This rule consumes properties for a summary field element. - * - * @param field The summary field to modify. - */ -void summaryProperties(ParsedSummaryField field) : { } -{ - <PROPERTIES> lbrace() (summaryProperty(field) <NL>)+ <RBRACE> -} - -/** - * This rule consumes a single summary property pair for a summary field element. - * - * @param field The summary field to modify. - */ -void summaryProperty(ParsedSummaryField field) : -{ - String name, value; -} -{ - name = identifierWithDash() <COLON> (value = identifierWithDash() | value = quotedString()) - { deployLogger.logApplicationPackage(Level.WARNING, "Specifying summary properties is deprecated and has no effect."); } -} - -/** * This rule consumes a stemming block of a field element. * * @param field The field to modify. @@ -1399,28 +1286,6 @@ void dictionarySetting(ParsedField field) : { } | <UNCASED> { field.dictionary(DictionaryOption.UNCASED); } ) } -/** - * This rule consumes a body statement of a field element. - */ -void body() : { } -{ - <BODY> - { - deployLogger.logApplicationPackage(Level.WARNING, "'body' is deprecated and has no effect."); - } -} - -/** - * This rule consumes a header statement of a field element. - */ -void header() : { } -{ - <HEADER> - { - deployLogger.logApplicationPackage(Level.WARNING, "'header' is deprecated and has no effect."); - } -} - void queryCommand(ParsedField field) : { String command; @@ -1547,17 +1412,6 @@ void id(ParsedField field) : } /** - * Consumes an indexing-rewrite statement of a field body block. - * - * @param field The field to modify. - */ -void indexingRewrite(ParsedField field) : { } -{ - <INDEXINGREWRITE> <COLON> <NONE> - { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'indexing-rewrite' is deprecated and has no effect."); } -} - -/** * Consumes a document-summary block from within a schema block. * * @param schema the schema object to add content to @@ -1774,7 +1628,7 @@ void rankingConstant(ParsedSchema schema) : { ( <CONSTANT> name = identifier() lbrace() (path = fileItem() { pathType = DistributableResource.PathType.FILE; } - | path = uriItem() { pathType = DistributableResource.PathType.URI; } // Undocumented. TODO: Remove on Vespa 8 + | path = uriItem() { pathType = DistributableResource.PathType.URI; } // Undocumented. Remove? | type = tensorTypeWithPrefix(rankingConstantErrorMessage(name)) (<NL>)* )+ <RBRACE> @@ -1852,7 +1706,6 @@ void rankProfileItem(ParsedSchema schema, ParsedRankProfile profile) : { } | rankFeatures(profile) | rankProperties(profile) | secondPhase(profile) - | rankDegradation() | inputs(profile) | constants(schema, profile) | matchFeatures(profile) @@ -1934,8 +1787,7 @@ void function(ParsedRankProfile profile) : func.setInline(inline); var old = profile.addOrReplaceFunction(func); if (old != null) { - /* TODO disallow this for Vespa 8 */ - deployLogger.logApplicationPackage(Level.WARNING, "Function '" + func.name() + throw new IllegalArgumentException("Function '" + func.name() + "' is defined twice in rank profile '" + profile.name() + "'"); } @@ -2375,68 +2227,6 @@ void fieldRankFilter(ParsedRankProfile profile) : } /** - * Consumes part of a rank-degradation statement of a rank profile. - */ -void rankDegradationBinSize() : -{ - double freq; -} -{ - <RPBINSIZE> <COLON> freq = floatValue() - { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'doc-frequency' in 'rank-degradation' is deprecated and has no effect."); } -} - - -/** - * Consumes part of a rank-degradation statement of a rank profile. - */ -void rankDegradationBinLow() : -{ - int n; -} -{ - <RPBINLOW> <COLON> n = integer() - { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'min-fullrank-docs' in 'rank-degradation' is deprecated and has no effect."); } -} - -/** - * Consumes part of a rank-degradation statement of a rank profile. - */ -void rankDegradationPosbinSize() : -{ - double avgOcc; -} -{ - <RPPOSBINSIZE> <COLON> avgOcc = floatValue() - { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'occurrences-per-doc' in 'rank-degradation' is deprecated and has no effect."); } -} - - -/** - * Consumes part of a rank-degradation statement of a rank profile. - */ -void rankDegradationItem() : { } -{ - ( rankDegradationBinSize() - | rankDegradationBinLow() - | rankDegradationPosbinSize() ) -} - -/** - * Consumes a rank-degradation statement of a rank profile. - */ -void rankDegradation() : -{ - double freq; -} -{ - ( <RANKDEGRADATIONFREQ> <COLON> freq = floatValue() - { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'rank-degradation-frequency' in 'rank-profile' is deprecated and has no effect."); } - | <RANKDEGRADATION> lbrace() ( rankDegradationItem() (<NL>)*)+ <RBRACE> - ) -} - -/** * Consumes a set of constants available in ranking expressions in the enclosing profile. */ void constants(ParsedSchema schema, ParsedRankProfile profile) : @@ -2468,7 +2258,7 @@ void constant(ParsedSchema schema, ParsedRankProfile profile) : profile.add(new RankProfile.Constant(name, type, valuePath)); } ) - | // Deprecated forms (TODO: Add warning on Vespa 8): + | // Deprecated forms (TODO: Vespa > 8: Add warning): ( constantValue(profile, name) | constantTensor(profile, name) ) ) ) @@ -2759,13 +2549,9 @@ String identifier() : { } | <AS> | <ASCENDING> | <ATTRIBUTE> - | <BODY> | <BOLDING> | <BTREE> | <CASED> - | <COMPRESSION> - | <COMPRESSIONLEVEL> - | <COMPRESSIONTHRESHOLD> | <CONSTANT> | <CONSTANTS> | <CONTEXT> @@ -2796,8 +2582,6 @@ String identifier() : { } | <FUNCTION> | <GRAM> | <HASH> - | <HEADER> - | <HUGE> | <ID> | <IDENTICAL> | <IDENTIFIER> @@ -2805,7 +2589,6 @@ String identifier() : { } | <IMPORT> | <INDEX> | <INDEXING> - | <INDEXINGREWRITE> | <INHERITS> | <INLINE> | <INPUTS> @@ -2880,7 +2663,6 @@ String identifier() : { } | <UNCASED> | <URI> | <UPPERBOUND> - | <USEDOCUMENT> | <VARIABLE> | <WEIGHT> | <WEIGHTEDSET> diff --git a/config-model/src/main/perl/vespa-deploy b/config-model/src/main/perl/vespa-deploy index 02a6737185d..dde074b5e8c 100755 --- a/config-model/src/main/perl/vespa-deploy +++ b/config-model/src/main/perl/vespa-deploy @@ -445,7 +445,7 @@ sub http_upload_lowlevel { $url = add_url_property_from_option($url, $opt_F, "from"); `$CURL_POST $url`; } else { - my $TAR="tar -C $app --dereference --exclude='.[a-zA-Z0-9]*' --exclude=ext -cf - . --transform=\"s#^#application/#\" "; + my $TAR="tar -C $app --dereference --exclude='.[a-zA-Z0-9]*' --exclude=ext -cf - ."; print "Uploading application '$app' using $url\n"; if (-f $app) { `cat $app | $CURL_POST_ZIP $url`; diff --git a/config-model/src/main/perl/vespa-expand-config.pl b/config-model/src/main/perl/vespa-expand-config.pl deleted file mode 100755 index 7fa2ce69707..00000000000 --- a/config-model/src/main/perl/vespa-expand-config.pl +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env perl -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#============================================================================ -# @version $Id: expand-config.pl,v 1.1 2006-07-26 15:52:43 gv Exp $ -# @project Vespa Admin -# @author Gj�ran Voldengen -# @date created 2005-04-15 -# -# Create a vespa config file from an application package config file -# that might contain "file=<filename>" statements. The output stream -# consists of the original file contents, with the expanded and escaped -# contents of the files given in file= statements. -#============================================================================= - -use strict; - -$| = 1; - -# Check for correct number of command line args -if ( int(@ARGV) != 0 ) { - die ("\nUsage: cat infile | $0 > outfile\n\n"); -} - - -#============================================================================ -# Global Constants -#============================================================================ - -# "Reserved keywords" to recognize in input file -my $FILE = "file"; - - -#============================================================================ -# Subroutines -#============================================================================ - -#--------------------------------------------------------------------------- -# Expand the contents of the input file into a one-line string, -# escaping special chars. -#--------------------------------------------------------------------------- -sub expandFile { - my ($filename) = @_; - - my $config = ""; - - # Read the complete input file into a single string - open (INFILE, "$filename") || die "Cannot open $filename\n"; - while (<INFILE>) { - $config .= $_; - } - - $config =~ s{\\}{\\\\}g; - $config =~ s{\"}{\\\"}g; #" emacs gets confused.. - $config =~ s{\n}{\\n}g; - - return $config; -} - - -#============================================================================ -# Main program -#============================================================================ -my $file = ""; - -while (<STDIN>) { - - # Comment lines are allowed, and must be preserved, along with all - # lines that don't contain "file=" - unless (m/^\#/) { - # Allow several files on one line - while (m{$FILE \s* = \s* ([^\s\"]+) }x) { #" - $file = $1; - $file =~ s{^\s+}{}; - $file =~ s{\s+ $ }{}x; - $_ = $` . expandFile($file) . $'; - } - } - print STDOUT ($_); -} - - - -############################### File end ################################ diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc index 2b9b414374c..4e916e0901f 100644 --- a/config-model/src/main/resources/schema/admin.rnc +++ b/config-model/src/main/resources/schema/admin.rnc @@ -7,7 +7,7 @@ AdminV2 = element adminserver { service.attlist } & GenericConfig* & LogServer? & - (ConfigServer | ConfigServers)? & + ConfigServers? & AdminSlobroks? & AdminMonitoring? & Metrics? & diff --git a/config-model/src/main/resources/schema/common.rnc b/config-model/src/main/resources/schema/common.rnc index 275c0f7f702..7259a3159ff 100644 --- a/config-model/src/main/resources/schema/common.rnc +++ b/config-model/src/main/resources/schema/common.rnc @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. service.attlist &= attribute hostalias { xsd:NCName } service.attlist &= attribute baseport { xsd:unsignedShort }? -service.attlist &= attribute jvmargs { text }? service.attlist &= attribute jvm-options { text }? service.attlist &= attribute jvm-gc-options { text }? # preload is for internal use only @@ -44,7 +43,6 @@ OptionalDedicatedNodes = element nodes { GenericConfig = element config { attribute name { text }, - attribute namespace { text }?, # TODO: Remove in Vespa 8 attribute version { text }?, anyElement* } diff --git a/config-model/src/main/resources/schema/container.rnc b/config-model/src/main/resources/schema/container.rnc index c16a5c4e3a5..2d145e170db 100644 --- a/config-model/src/main/resources/schema/container.rnc +++ b/config-model/src/main/resources/schema/container.rnc @@ -5,29 +5,21 @@ include "processing.rnc" Handler = element handler { ComponentDefinition & - ServerBindings & - element clientBinding {text}* + ServerBindings } Binding = element binding {text} ServerBindings = Binding* -Client = element client { - ComponentDefinition & - element binding {text}* & - element serverBinding {text}* -} - Server = element server { ComponentDefinition } AccessControl = element access-control { - attribute domain { xsd:NCName }? & # TODO Vespa 8 Remove - attribute read { string "true" | string "false" }? & # TODO Vespa 8 Remove - attribute write { string "true" | string "false" }? & # TODO Vespa 8 Remove + attribute domain { xsd:NCName }? & # TODO(tokle,bjorncs) Vespa > 8: Remove + attribute read { string "true" | string "false" }? & # TODO(tokle,bjorncs) Vespa > 8: Remove + attribute write { string "true" | string "false" }? & # TODO(tokle,bjorncs) Vespa > 8: Remove attribute tls-handshake-client-auth {string "want" | string "need"}? & - element vespa-domain { xsd:NCName }? & # TODO Remove after end of March 2020 element exclude { Binding+ }? diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 3fdbff84f6d..1ab3c9893bf 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -1,12 +1,10 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -ContainerCluster = element container | jdisc { +ContainerCluster = element container { attribute version { "1.0" } & attribute id { xsd:NCName }? & - attribute jetty { xsd:boolean }? & Include* & ContainerServices & DocumentBinding* & - Aliases? & NodesOfContainerCluster? & ClientAuthorize? } @@ -21,7 +19,6 @@ ContainerServices = Component* & Embedder* & Handler* & - Client* & Server* & Http? & AccessLog* & @@ -37,11 +34,6 @@ Components = element components { Component* } -Aliases = element aliases { - element service-alias { xsd:NCName }* & - element endpoint-alias { xsd:NCName }* -} - Include = element \include { attribute dir { text } } @@ -233,7 +225,6 @@ HttpClientApi = element http-client-api { # NODES: NodesOfContainerCluster = element nodes { - attribute jvmargs { text }? & attribute jvm-options { text }? & attribute jvm-gc-options { text }? & attribute preload { text }? & diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc index 8dd5f5c042f..ff45b127b8b 100644 --- a/config-model/src/main/resources/schema/content.rnc +++ b/config-model/src/main/resources/schema/content.rnc @@ -83,10 +83,8 @@ ClusterControllerTuning = element cluster-controller { DispatchTuning = element dispatch { element max-hits-per-partition { xsd:nonNegativeInteger }? & element dispatch-policy { string "round-robin" | string "adaptive" | string "random" }? & - element min-group-coverage { xsd:double }? & # TODO: Ignored, remove on Vespa 8 element min-active-docs-coverage { xsd:double }? & - element top-k-probability { xsd:double }? & - element use-local-node { string "true" | string "false" }? # TODO: Ignored, remove on Vespa 8 + element top-k-probability { xsd:double }? } ClusterTuning = element tuning { diff --git a/config-model/src/main/resources/schema/federation.rnc b/config-model/src/main/resources/schema/federation.rnc index 15b57b9dd7b..4a0d34149ea 100644 --- a/config-model/src/main/resources/schema/federation.rnc +++ b/config-model/src/main/resources/schema/federation.rnc @@ -2,7 +2,7 @@ # Schema for federation configuration inside the searchchains section. GenericSource = - GenericSearchChainInQrservers & + SearchChainInFederation & FederationOptions? Source = @@ -17,26 +17,11 @@ Provider = attribute type { xsd:string }? & attribute cluster { xsd:string }? & - # TODO Vespa 8 Remove yca concepts from services.xml syntax - attribute yca-application-id { xsd:string }? & - attribute yca-cache-ttl { xsd:string { pattern = "\d+(\.\d*)?\s*m?s" } }? & - attribute yca-cache-retry-wait { xsd:string { pattern = "\d+(\.\d*)?\s*m?s" } }? & - YcaProxy? & HttpProviderSearcherOptions & Source* } -YcaProxy = - element yca-proxy { - GenericNode? - } - -GenericNode = - attribute host { xsd:string } & - attribute port { xsd:int } - - HttpProviderSearcherOptions = attribute cacheweight { xsd:float { minInclusive = "0" } }? & attribute path { xsd:string }? & diff --git a/config-model/src/main/resources/schema/searchchains.rnc b/config-model/src/main/resources/schema/searchchains.rnc index d4c9e8f4f98..993a2088859 100644 --- a/config-model/src/main/resources/schema/searchchains.rnc +++ b/config-model/src/main/resources/schema/searchchains.rnc @@ -3,40 +3,17 @@ include "federation.rnc" -SearchChains = - element searchchains { - Searcher* & - SearchChainInQrservers* & - GenericConfig* - } - -OuterSearchChains = - element searchchains { - Searcher* & - SearchChainInQrservers* - } - -SearchChainInQrservers = - element searchchain { - GenericSearchChainInQrservers - } | - Provider - -GenericSearchChainInQrservers = +SearchChainInFederation = ComponentId & - SearchChainInheritanceInQrservers & + SearchChainInheritanceInFederation & attribute searchers { text }? & Searcher* & Phase* & GenericConfig* -SearchChainInheritanceInQrservers = +SearchChainInheritanceInFederation = attribute inherits { text }? & - attribute excludes { text }? & - element inherits { - element searchchain { ComponentSpec }* & - element exclude { ComponentSpec }* - }? + attribute excludes { text }? Searcher = RegularSearcher | diff --git a/config-model/src/main/resources/schema/services.rnc b/config-model/src/main/resources/schema/services.rnc index 3a8ffe30563..d3d642b8826 100644 --- a/config-model/src/main/resources/schema/services.rnc +++ b/config-model/src/main/resources/schema/services.rnc @@ -1,28 +1,22 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. include "common.rnc" include "admin.rnc" -include "clients.rnc" include "content.rnc" include "docproc.rnc" include "routing.rnc" include "containercluster.rnc" -include "genericcluster.rnc" start = element services { attribute version { "1.0" }? & attribute application-type { "hosted-infrastructure" }? & element legacy { element v7-geo-positions { xsd:boolean } }? & - GenericCluster* & GenericConfig* & Admin? & - Clients? & Content* & ContainerCluster* & Routing? } -| GenericCluster | Admin -| Clients | Content | ContainerCluster diff --git a/config-model/src/main/resources/schema/version/7.x/.gitignore b/config-model/src/main/resources/schema/version/7.x/.gitignore new file mode 100644 index 00000000000..e8bf39e2289 --- /dev/null +++ b/config-model/src/main/resources/schema/version/7.x/.gitignore @@ -0,0 +1,2 @@ +*.xsd +*.rng diff --git a/config-model/src/main/resources/schema/version/7.x/admin.rnc b/config-model/src/main/resources/schema/version/7.x/admin.rnc new file mode 100644 index 00000000000..2b9b414374c --- /dev/null +++ b/config-model/src/main/resources/schema/version/7.x/admin.rnc @@ -0,0 +1,115 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +Admin = AdminV2 | AdminV3 | AdminV4 + +AdminV2 = + element admin { + attribute version { "2.0" } & + element adminserver { service.attlist } & + GenericConfig* & + LogServer? & + (ConfigServer | ConfigServers)? & + AdminSlobroks? & + AdminMonitoring? & + Metrics? & + ClusterControllers? & + LogForwarding? + } + +AdminV3 = + element admin { + attribute version { "3.0" } & + GenericConfig* & + Nodes + } + +AdminV4 = + element admin { + attribute version { "4.0" } & + AdminV4Slobroks? & + AdminV4LogServers? & + GenericConfig* & + AdminMonitoring? & + Metrics? & + LogForwarding? + } + +AdminV4Slobroks = + element slobroks { + OptionalDedicatedNodes + } + +AdminV4LogServers = + element logservers { + OptionalDedicatedNodes + } + +AdminSlobroks = + element slobroks { + element slobrok { + service.attlist & + attribute index { xsd:nonNegativeInteger }? + }+ + } + +AdminMonitoring = + element monitoring { + attribute interval { xsd:int }?, + attribute systemname { xsd:string }? + } + +ConfigServer = element configserver { + service.attlist +} + +ConfigServers = element configservers { + ConfigServer+ +} + +LogServer = element logserver { + service.attlist +} + +Metrics = element metrics { + element consumer { + attribute id { xsd:Name } & + element metric-set { attribute id { xsd:Name } }* & + element metric { + attribute id { xsd:Name } & + attribute display-name { xsd:Name }? + }* & + Cloudwatch? + }+ +} + +Cloudwatch = element cloudwatch { + attribute region { xsd:Name } & + attribute namespace { xsd:string { pattern = "[\w_\-/#:\.]+" } } & + ( + element credentials { + attribute access-key-name { xsd:Name } & + attribute secret-key-name { xsd:Name } + } + | + element shared-credentials { + attribute file { string } & + attribute profile { xsd:Name }? + } + )? +} + +ClusterControllers = element cluster-controllers { + attribute standalone-zookeeper { xsd:string }? & + element cluster-controller { + service.attlist + }+ +} + +LogForwarding = element logforwarding { + attribute include-admin { xsd:boolean }? & + element splunk { + attribute splunk-home { xsd:string }? & + attribute deployment-server { xsd:string } & + attribute client-name { xsd:string } & + attribute phone-home-interval { xsd:positiveInteger }? + } +} diff --git a/config-model/src/main/resources/schema/clients-v2.rnc b/config-model/src/main/resources/schema/version/7.x/clients-v2.rnc index 5a9353321c8..5a9353321c8 100644 --- a/config-model/src/main/resources/schema/clients-v2.rnc +++ b/config-model/src/main/resources/schema/version/7.x/clients-v2.rnc diff --git a/config-model/src/main/resources/schema/clients.rnc b/config-model/src/main/resources/schema/version/7.x/clients.rnc index 1e1dffc90b9..1e1dffc90b9 100644 --- a/config-model/src/main/resources/schema/clients.rnc +++ b/config-model/src/main/resources/schema/version/7.x/clients.rnc diff --git a/config-model/src/main/resources/schema/version/7.x/common.rnc b/config-model/src/main/resources/schema/version/7.x/common.rnc new file mode 100644 index 00000000000..45183b0657c --- /dev/null +++ b/config-model/src/main/resources/schema/version/7.x/common.rnc @@ -0,0 +1,73 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +service.attlist &= attribute hostalias { xsd:NCName } +service.attlist &= attribute baseport { xsd:unsignedShort }? +service.attlist &= attribute jvmargs { text }? +service.attlist &= attribute jvm-options { text }? +service.attlist &= attribute jvm-gc-options { text }? +# preload is for internal use only +service.attlist &= attribute preload { text }? + +anyElement = element * { + (attribute * { text } + | text + | anyElement)* +} + +# Valid qualified java class name. See http://docs.oracle.com/javase/specs/jls/se8/html/jls-3.html#jls-3.8 +JavaId = xsd:string { pattern = "([a-zA-Z_$][a-zA-Z\d_$]*\.)*[a-zA-Z_$][a-zA-Z\d_$]*" } + +Nodes = element nodes { + attribute count { xsd:positiveInteger | xsd:string } & + attribute flavor { xsd:string }? & + attribute docker-image { xsd:string }? & + Resources? +} + +Resources = element resources { + attribute vcpu { xsd:double { minExclusive = "0.0" } | xsd:string } & + attribute memory { xsd:string } & + attribute disk { xsd:string } & + attribute disk-speed { xsd:string }? & + attribute storage-type { xsd:string }? & + attribute architecture { xsd:string }? +} + +OptionalDedicatedNodes = element nodes { + attribute count { xsd:positiveInteger | xsd:string } & + attribute flavor { xsd:string }? & + attribute required { xsd:boolean }? & + attribute docker-image { xsd:string }? & + attribute dedicated { xsd:boolean }? & + attribute exclusive { xsd:boolean }? & + Resources? +} + +GenericConfig = element config { + attribute name { text }, + attribute namespace { text }?, + attribute version { text }?, + anyElement* +} + +ComponentSpec = + ( attribute id { xsd:Name | JavaId } | attribute idref { xsd:Name } | attribute ident { xsd:Name } ) + +ComponentId = + ComponentSpec + +BundleSpec = + attribute class { xsd:Name | JavaId }? & + attribute bundle { xsd:Name }? + +Component = element component { + ComponentDefinition +} + +ComponentDefinition = + ComponentId & + BundleSpec & + GenericConfig* & + Component* + + + diff --git a/config-model/src/main/resources/schema/version/7.x/container-include.rnc b/config-model/src/main/resources/schema/version/7.x/container-include.rnc new file mode 100644 index 00000000000..8f6a8a3bada --- /dev/null +++ b/config-model/src/main/resources/schema/version/7.x/container-include.rnc @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +include "containercluster.rnc" +include "common.rnc" +include "container.rnc" +include "docproc.rnc" +include "searchchains.rnc" + +start = SearchInContainer | DocprocInContainer | ProcessingInContainer | Components diff --git a/config-model/src/main/resources/schema/version/7.x/container.rnc b/config-model/src/main/resources/schema/version/7.x/container.rnc new file mode 100644 index 00000000000..f7f63829281 --- /dev/null +++ b/config-model/src/main/resources/schema/version/7.x/container.rnc @@ -0,0 +1,52 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Schema for common container options + +include "processing.rnc" + +Handler = element handler { + ComponentDefinition & + ServerBindings & + element clientBinding {text}* +} + +Binding = element binding {text} +ServerBindings = Binding* + +Client = element client { + ComponentDefinition & + element binding {text}* & + element serverBinding {text}* +} + +Server = element server { + ComponentDefinition +} + +AccessControl = element access-control { + attribute domain { xsd:NCName }? & + attribute read { string "true" | string "false" }? & + attribute write { string "true" | string "false" }? & + attribute tls-handshake-client-auth {string "want" | string "need"}? & + element vespa-domain { xsd:NCName }? & + element exclude { + Binding+ + }? +} + +HttpFilterChain = + HttpFilter* & + ChainBaseContent & + ServerBindings + +HttpFilter = element filter { + ComponentDefinition & + FilterConfig? +} + +FilterConfig = element filter-config { + anyElement* +} + +Renderer = element renderer { + ComponentDefinition +} diff --git a/config-model/src/main/resources/schema/version/7.x/containercluster.rnc b/config-model/src/main/resources/schema/version/7.x/containercluster.rnc new file mode 100644 index 00000000000..3fdbff84f6d --- /dev/null +++ b/config-model/src/main/resources/schema/version/7.x/containercluster.rnc @@ -0,0 +1,283 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +ContainerCluster = element container | jdisc { + attribute version { "1.0" } & + attribute id { xsd:NCName }? & + attribute jetty { xsd:boolean }? & + Include* & + ContainerServices & + DocumentBinding* & + Aliases? & + NodesOfContainerCluster? & + ClientAuthorize? +} + +ContainerServices = + SearchInContainer? & + DocprocInContainer? & + ProcessingInContainer? & + ModelEvaluation? & + DocumentApi? & + Components* & + Component* & + Embedder* & + Handler* & + Client* & + Server* & + Http? & + AccessLog* & + SecretStore? & + ZooKeeper? & + GenericConfig* + +# TODO(ogronnesby): Change this configuration syntax +ClientAuthorize = element client-authorize { empty } + +Components = element components { + Include* & + Component* +} + +Aliases = element aliases { + element service-alias { xsd:NCName }* & + element endpoint-alias { xsd:NCName }* +} + +Include = element \include { + attribute dir { text } +} + +Http = element http { + (Filtering & HttpServer+) | + HttpServer+ | + empty +} + +Filtering = element filtering { + attribute strict-mode { xsd:boolean }? & + HttpFilter* & + AccessControl? & + element request-chain { + HttpFilterChain + }* & + element response-chain { + HttpFilterChain + }* +} + +HttpServer = element server { + attribute port { xsd:nonNegativeInteger }? & + attribute required { xsd:boolean }? & + attribute default-request-chain { xsd:string }? & + attribute default-response-chain { xsd:string }? & + ComponentId & + (Ssl | SslProvider)? & + GenericConfig* +} + +AccessLog = element accesslog { + attribute type { string "yapache" | string "vespa" | string "json" | string "disabled" }? & + attribute fileNamePattern { string }? & + attribute compressOnRotation { xsd:boolean }? & + attribute symlinkName { string }? & + attribute compressionType { string "gzip" | string "zstd" }? & + attribute queueSize { xsd:nonNegativeInteger }? & + attribute bufferSize { xsd:nonNegativeInteger }? & + attribute rotationInterval { string }? +} + +SecretStore = element secret-store { + attribute type { string "oath-ckms" | string "cloud" } & + element group { + attribute name { string } & + attribute environment { string "alpha" | string "corp" | string "prod" | string "aws" | string "aws_stage" } + } * & + element store { + attribute id { string } & + element aws-parameter-store { + attribute account { string } & + attribute aws-region { string } + } * + }? +} + +ZooKeeper = element zookeeper { + empty +} + +Embedder = element embedder { + attribute id { string }? & + attribute class { xsd:Name | JavaId }? & + attribute bundle { xsd:Name }? & + attribute def { xsd:Name }? & + anyElement* +} + +ModelEvaluation = element model-evaluation { + element onnx { + element models { + element model { + attribute name { string } & + element intraop-threads { xsd:nonNegativeInteger }? & + element interop-threads { xsd:nonNegativeInteger }? & + element execution-mode { string "sequential" | string "parallel" }? + }* + }? + }? +} + +Ssl = element ssl { + element private-key-file { string } & + element certificate-file { string } & + element ca-certificates-file { string }? & + element client-authentication { string "disabled" | string "want" | string "need" }? & + element cipher-suites { string }? & + element protocols { string }? +} + +SslProvider = element ssl-provider { + BundleSpec +} + +Threadpool = element threadpool { + element max-threads { xsd:nonNegativeInteger } & + element min-threads { xsd:nonNegativeInteger } & + element queue-size { xsd:nonNegativeInteger } +} + +# SEARCH: + +SearchInContainer = element search { + Include* & + ServerBindings? & + Searcher* & + SearchChain* & + Provider* & + Renderer* & + GenericConfig* & + Threadpool? +} + +SearchChain = element chain { + GenericSearchChain +} + +GenericSearchChain = + ComponentId & + SearchChainInheritance & + attribute searchers { text }? & + Searcher* & + Phase* & + GenericConfig* + +SearchChainInheritance = + attribute inherits { text }? & + attribute excludes { text }? & + element inherits { + element chain { ComponentSpec }* & + element exclude { ComponentSpec }* + }? + + +# DOCPROC: + +DocprocInContainer = element document-processing { + Include* & + DocprocClusterAttributes? & + DocumentProcessorV3* & + ChainInDocprocInContainerCluster* & + GenericConfig* +} +ChainInDocprocInContainerCluster = element chain { + DocprocChainV3Contents +} + + + +# PROCESSING: + +ProcessingInContainer = element processing { + Include* & + ServerBindings? & + Processor* & + Chain* & + ProcessingRenderer* & + GenericConfig* +} + + + +# DOCUMENT API: + +DocumentApi = element document-api { + ServerBindings & + GenericConfig* & + element abortondocumenterror { xsd:boolean }? & + element retryenabled { xsd:boolean }? & + element route { text }? & + element maxpendingdocs { xsd:positiveInteger }? & + element maxpendingbytes { xsd:positiveInteger }? & + element retrydelay { xsd:double { minInclusive = "0.0" } }? & + element timeout { xsd:double { minInclusive = "0.0" } }? & + element tracelevel { xsd:positiveInteger }? & + element mbusport { xsd:positiveInteger }? & + DocumentRestApi? & + HttpClientApi? +} + +DocumentRestApi = element rest-api { empty } + +HttpClientApi = element http-client-api { + Threadpool? +} + +# NODES: + +NodesOfContainerCluster = element nodes { + attribute jvmargs { text }? & + attribute jvm-options { text }? & + attribute jvm-gc-options { text }? & + attribute preload { text }? & + attribute allocated-memory { text }? & + attribute cpu-socket-affinity { xsd:boolean }? & + element jvm { + attribute options { text }? & + attribute gc-options { text }? & + attribute allocated-memory { text }? + } ? & + Resources? & + element environment-variables { + anyElement + + } ? & + ( + ( + attribute of { xsd:string } & + attribute required { xsd:boolean }? + ) + | + attribute type { xsd:string } + | + ( + attribute count { xsd:positiveInteger | xsd:string } & + attribute flavor { xsd:string }? & + attribute required { xsd:boolean }? & + attribute exclusive { xsd:boolean }? & + attribute docker-image { xsd:string }? + ) + | + element node { + attribute hostalias { xsd:NCName } & + attribute cpu-socket { xsd:positiveInteger }? & + GenericConfig* + }+ + ) +} + + + +#DOCUMENT BINDINGS: + +DocumentBinding = element document { + attribute type { xsd:NCName } & + attribute class { xsd:NCName } & + attribute bundle { xsd:NCName } +} diff --git a/config-model/src/main/resources/schema/version/7.x/content.rnc b/config-model/src/main/resources/schema/version/7.x/content.rnc new file mode 100644 index 00000000000..a2b1bd908ec --- /dev/null +++ b/config-model/src/main/resources/schema/version/7.x/content.rnc @@ -0,0 +1,395 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace a = "http://relaxng.org/ns/compatibility/annotations/1.0" + +include "container.rnc" +include "searchchains.rnc" + +Redundancy = element redundancy { + attribute reply-after { xsd:nonNegativeInteger }? & + xsd:nonNegativeInteger +} + +DistributionType = element distribution { + attribute type { string "strict" | string "loose" | string "legacy" } +} + +BucketSplitting = element bucket-splitting { + attribute max-documents { xsd:nonNegativeInteger }? & + attribute max-size { xsd:nonNegativeInteger }? & + attribute minimum-bits { xsd:nonNegativeInteger }? +} + +MergeTuning = element merges { + attribute max-per-node { xsd:nonNegativeInteger }? & + attribute max-queue-size { xsd:nonNegativeInteger }? & + attribute max-nodes-per-merge { xsd:nonNegativeInteger { + minInclusive = "2" maxInclusive = "16" } }? +} + +VisitorMaxConcurrent = element max-concurrent { + attribute fixed { xsd:nonNegativeInteger }? & + attribute variable { xsd:nonNegativeInteger }? +} + +VisitorTuning = element visitors { + attribute thread-count { xsd:nonNegativeInteger }? & + attribute max-queue-size { xsd:nonNegativeInteger }? & + VisitorMaxConcurrent? +} + +Maintenance = element maintenance { + attribute start { xsd:string { pattern = "[0-9]{2}:[0-9]{2}" } }, + attribute stop { xsd:string { pattern = "[0-9]{2}:[0-9]{2}" } }, + attribute high { string "monday" | string "tuesday" | string "wednesday" | + string "thursday" | string "friday" | string "saturday" | + string "sunday" } +} + +PersistenceThread = element thread { + ## The lowest priority this thread should handle. + attribute lowest-priority { string "HIGHEST" | string "VERY_HIGH" | string "HIGH_1" | + string "HIGH_2" | string "HIGH_3" | string "NORMAL_1" | string "NORMAL_2" | + string "NORMAL_3" | string "NORMAL_4" | string "NORMAL_5" | string "NORMAL_6" | + string "LOW_1" | string "LOW_2" | string "LOW_3" | string "VERY_LOW" }? & + ## The number of threads of this type to create + attribute count { xsd:integer }? +} + +## Declare which storage threads each disk should have. +PersistenceThreads = element persistence-threads { + ## The number of threads to create + attribute count { xsd:integer }? & + ## All of the below settings are deprecated. + ## Operations with priority worse than this can be blocked + attribute highest-priority-to-block { xsd:string } ? & + ## Operations with priority better than this can block others + attribute lowest-priority-to-block-others { xsd:string } ? & + Thread* +} + +MinNodeRatioPerGroup = element min-node-ratio-per-group { + xsd:double { minInclusive = "0" maxInclusive = "1" } +} + +ClusterControllerTuning = element cluster-controller { + element init-progress-time { xsd:string { pattern = "([0-9\.]+)\s*([a-z]+)?" } }? & + element transition-time { xsd:string { pattern = "([0-9\.]+)\s*([a-z]+)?" } }? & + element max-premature-crashes { xsd:nonNegativeInteger }? & + element stable-state-period { xsd:string { pattern = "([0-9\.]+)\s*([a-z]+)?" } }? & + element min-distributor-up-ratio { xsd:double }? & + element min-storage-up-ratio { xsd:double }? +} + +DispatchTuning = element dispatch { + element max-hits-per-partition { xsd:nonNegativeInteger }? & + element dispatch-policy { string "round-robin" | string "adaptive" | string "random" }? & + element min-group-coverage { xsd:double }? & + element min-active-docs-coverage { xsd:double }? & + element top-k-probability { xsd:double }? & + element use-local-node { string "true" | string "false" }? +} + +ClusterTuning = element tuning { + DispatchTuning? & + DistributionType? & + BucketSplitting? & + MergeTuning? & + VisitorTuning? & + ClusterControllerTuning? & + Maintenance? & + PersistenceThreads? & + MinNodeRatioPerGroup? & + ResourceLimits? +} + +Content = element content { + attribute version { "1.0" } & + attribute id { xsd:NCName }? & + attribute distributor-base-port { xsd:unsignedShort }? & + # Mandatory + Redundancy & + ContentSearch? & + Dispatch? & + ClusterTuning? & + # Can be used for caches and feedbatching. + GenericConfig* & + Engine? & + # Here you can add document definitions that you also want to handle. + # Search might want to know of them in advance. + Documents? & + ContentNodes? & + TopGroup? & + Controllers? + # Contains experimental feature switches + #Experimental? +} + +Controllers = + element controllers { + OptionalDedicatedNodes + } + +ContentSearch = element search { + element query-timeout { xsd:double { minInclusive = "0" } }? & + element visibility-delay { xsd:double { minInclusive = "0" } }? & + SearchCoverage? +} + +SearchCoverage = element coverage { + element minimum { xsd:double { minInclusive = "0" maxInclusive = "1" } }? & + element min-wait-after-coverage-factor { xsd:double { minInclusive = "0" maxInclusive = "1" } }? & + element max-wait-after-coverage-factor { xsd:double { minInclusive = "0" maxInclusive = "1" } }? +} + +Dispatch = element dispatch { + element num-dispatch-groups { xsd:nonNegativeInteger }? & + DispatchGroup* +} + +DispatchGroup = element group { + DispatchNode+ +} + +DispatchNode = element node { + attribute distribution-key { xsd:nonNegativeInteger } +} + +## Specification of what document processing should be done for indexing. +DocumentProcessing = element document-processing { + attribute cluster { text }? & + attribute chain { text }? +} + +## Config for the persistence providers. +Engine = element engine { + (Proton | Dummy) +} + +Proton = element proton { + element flush-on-shutdown { xsd:string }? & + element sync-transactionlog { xsd:string }? & + element visibility-delay { xsd:double { minInclusive = "0.0" } }? & + element query-timeout { xsd:double { minInclusive = "0.0" } }? & + element searchable-copies { xsd:integer { minInclusive = "0" } }? & + ResourceLimits? & + Tuning? +} + +ResourceLimits = element resource-limits { + element disk { xsd:double { minInclusive = "0.0" maxInclusive = "1.0" } }? & + element memory { xsd:double { minInclusive = "0.0" maxInclusive = "1.0" } }? +} + +Dummy = element dummy { + text +} + +Documents = element documents { + attribute selection { xsd:string }? & + attribute garbage-collection { xsd:string }? & + attribute garbage-collection-interval { xsd:nonNegativeInteger }? & + DocumentProcessing? & + + element document { + attribute type { xsd:string } & + attribute selection { xsd:string }? & + attribute mode { string "index" | string "streaming" | string "store-only" } & + attribute global { xsd:boolean }? + }+ +} + +ContentNode = element node { + GenericConfig* & + service.attlist & + attribute distribution-key { xsd:nonNegativeInteger } & + attribute capacity { xsd:double { minExclusive = "0.0" } }? & + attribute mmap-core-limit { xsd:nonNegativeInteger }? & + attribute core-on-oom { xsd:boolean }? & + attribute no-vespamalloc { xsd:string }? & + attribute vespamalloc { xsd:string }? & + attribute vespamalloc-debug { xsd:string }? & + attribute vespamalloc-debug-stacktrace { xsd:string }? & + attribute cpu-socket { xsd:nonNegativeInteger }? +} + +ContentNodes = element nodes { + Resources? & + attribute cpu-socket-affinity { xsd:string }? & + attribute mmap-core-limit { xsd:nonNegativeInteger }? & + attribute core-on-oom { xsd:boolean }? & + attribute no-vespamalloc { xsd:string }? & + attribute vespamalloc { xsd:string }? & + attribute vespamalloc-debug { xsd:string }? & + attribute vespamalloc-debug-stacktrace { xsd:string }? & + ( + ( + attribute count { xsd:positiveInteger | xsd:string } & + attribute flavor { xsd:string }? & + attribute required { xsd:boolean }? & + attribute exclusive { xsd:boolean }? & + attribute docker-image { xsd:string }? & + attribute groups { xsd:positiveInteger | xsd:string }? + ) + | + ContentNode + + ) +} + +TopGroup = element group { + # Neither name nor distribution key makes any sense for the top group. There has to be + # a top group so it never needs referring to, and it's only one group to choose from so + # it has no use of a distribution key. Leaving it allowed to set them for now to not + # break all system tests and backward compatibility. + attribute name { xsd:string }? & + attribute cpu-socket-affinity { xsd:string }? & + attribute mmap-core-limit { xsd:nonNegativeInteger }? & + attribute core-on-oom { xsd:boolean }? & + attribute no-vespamalloc { xsd:string }? & + attribute vespamalloc { xsd:string }? & + attribute vespamalloc-debug { xsd:string }? & + attribute vespamalloc-debug-stacktrace { xsd:string }? & + attribute distribution-key { xsd:nonNegativeInteger }? & + ( + ContentNode + + | + ( + element distribution { + attribute partitions { xsd:string } + } & + Group + + ) + ) +} + +Group = element group { + attribute distribution-key { xsd:nonNegativeInteger } & + attribute name { xsd:string } & + ( + ContentNode + + | + ( + element nodes { + attribute count { xsd:positiveInteger | xsd:string } & + attribute flavor { xsd:string }? & + attribute required { xsd:boolean }? & + attribute exclusive { xsd:boolean }? & + attribute docker-image { xsd:string }? & + attribute groups { xsd:positiveInteger | xsd:string }? + } + ) + | + ( + element distribution { + attribute partitions { xsd:string } + } & + Group + + ) + ) +} + +Tuning = element tuning { + element dispatch { + element max-hits-per-partition { xsd:nonNegativeInteger }? + }? & + element searchnode { + element requestthreads { + element search { xsd:nonNegativeInteger }? & + element persearch { xsd:nonNegativeInteger }? & + element summary { xsd:nonNegativeInteger }? + }? & + element flushstrategy { + element native { + element total { + element maxmemorygain { xsd:nonNegativeInteger }? & + element diskbloatfactor { xsd:double { minInclusive = "0.0" } }? + }? & + element component { + element maxmemorygain { xsd:nonNegativeInteger }? & + element diskbloatfactor { xsd:double { minInclusive = "0.0" } }? & + element maxage { xsd:nonNegativeInteger }? + }? & + element transactionlog { + element maxsize { xsd:nonNegativeInteger }? + }? & + element conservative { + element memory-limit-factor { xsd:double { minInclusive = "0.0" maxInclusive = "1.0" } }? & + element disk-limit-factor { xsd:double { minInclusive = "0.0" maxInclusive = "1.0" } }? + }? + }? + }? & + element resizing { + element initialdocumentcount { xsd:nonNegativeInteger }? & + element amortize-count { xsd:nonNegativeInteger }? + }? & + element index { + element io { + element write { TuningIoOptionsLight }? & + element read { TuningIoOptionsLight }? & + element search { TuningIoOptionsSearch }? + }? & + element warmup { + element time { xsd:double { minInclusive = "0.0" } }? & + element unpack { xsd:boolean }? + }? + }? & + element attribute { + element io { + element write { TuningIoOptionsLight }? + } + }? & + element summary { + element io { + element write { TuningIoOptionsLight }? & + element read { TuningIoOptionsFull }? + }? & + element store { + element cache { + element maxsize { xsd:nonNegativeInteger }? & + element maxsize-percent { xsd:double { minInclusive = "0.0" maxInclusive = "50.0" } }? & + element initialentries { xsd:nonNegativeInteger }? & + TuningCompression? + }? & + element logstore { + element maxfilesize { xsd:nonNegativeInteger }? & + element minfilesizefactor { xsd:double { minInclusive = "0.10" maxInclusive = "1.0" } }? & + element chunk { + element maxsize { xsd:nonNegativeInteger }? & + TuningCompression? + }? + }? + }? + }? & + element initialize { + element threads { xsd:nonNegativeInteger }? + }? & + element feeding { + element concurrency { xsd:double { minInclusive = "0.0" maxInclusive = "1.0" } }? + }? & + element removed-db { + element prune { + element age { xsd:double { minInclusive = "0.0" } }? & + element interval { xsd:double { minInclusive = "60.0" } }? + }? + }? + }? +} + +TuningIoOptionsLight = string "normal" | string "directio" +TuningIoOptionsFull = string "normal" | string "directio" | string "mmap" | string "populate" +TuningIoOptionsSearch = string "mmap" | string "populate" + +TuningCompression = element compression { + element type { string "none" | string "lz4" | string "zstd" }? & + element level { xsd:nonNegativeInteger }? +} + +#Experimental = element experimental { +# Put experimental flags here +#} + +Thread = element thread { + ## The lowest priority this thread should handle. + attribute lowest-priority { xsd:string}? & + ## The number of threads of this type to create + attribute count { xsd:integer }? +} diff --git a/config-model/src/main/resources/schema/version/7.x/deployment.rnc b/config-model/src/main/resources/schema/version/7.x/deployment.rnc new file mode 100644 index 00000000000..3abced8e04a --- /dev/null +++ b/config-model/src/main/resources/schema/version/7.x/deployment.rnc @@ -0,0 +1,144 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# RELAX NG Compact Syntax +# Vespa Deployment file + +start = element deployment { + attribute version { "1.0" } & + attribute major-version { text }? & + attribute athenz-domain { xsd:string }? & + attribute athenz-service { xsd:string }? & + attribute cloud-account { xsd:string }? & + Step +} + +Step = + StepExceptInstance & + Instance* + +StepExceptInstance = + Delay* & + ParallelInstances* & + Upgrade? & + BlockChange* & + Notifications? & + Endpoints? & + Test? & + Staging? & + Prod* + +PrimitiveStep = + Delay* & + Region* & + ProdTest* + +Instance = element instance { + attribute id { xsd:string } & + attribute athenz-service { xsd:string }? & + attribute cloud-account { xsd:string }? & + StepExceptInstance +} + +ParallelSteps = element parallel { + SerialSteps* & + PrimitiveStep* +} + +SerialSteps = element steps { + ParallelSteps* & + PrimitiveStep* +} + +ParallelInstances = element parallel { + Instance* +} + +Upgrade = element upgrade { + attribute policy { xsd:string }? & + attribute revision-target { xsd:string }? & + attribute revision-change { xsd:string }? & + attribute min-risk { xsd:long }? & + attribute max-risk { xsd:long }? & + attribute max-idle-hours { xsd:long }? & + attribute rollout { xsd:string }? +} + +BlockChange = element block-change { + attribute revision { xsd:boolean }? & + attribute version { xsd:boolean }? & + attribute days { xsd:string }? & + attribute hours { xsd:string }? & + attribute from-date { xsd:string }? & + attribute to-date { xsd:string }? & + attribute time-zone { xsd:string }? +} + +Notifications = element notifications { + attribute when { xsd:string }? & + Email* +} + +Email = element email { + attribute address { xsd:string }? & + attribute role { xsd:string }? & + attribute when { xsd:string }? +} + +Test = element test { + attribute athenz-service { xsd:string }? & + attribute tester-flavor { xsd:string }? & + text +} + +Staging = element staging { + attribute athenz-service { xsd:string }? & + attribute tester-flavor { xsd:string }? & + text +} + +Prod = element prod { + attribute global-service-id { text }? & + attribute athenz-service { xsd:string }? & + attribute tester-flavor { xsd:string }? & + Region* & + Delay* & + ProdTest* & + ParallelSteps* +} + +ProdTest = element test { + text +} + +Region = element region { + attribute active { xsd:boolean }? & + attribute athenz-service { xsd:string }? & + attribute cloud-account { xsd:string }? & + text +} + +Delay = element delay { + attribute hours { xsd:long }? & + attribute minutes { xsd:long }? & + attribute seconds { xsd:long }? +} + +EndpointRegion = element region { + text +} + +EndpointInstance = element instance { + attribute weight { xsd:long } & + text +} + +Endpoint = element endpoint { + attribute id { xsd:string }? & + attribute container-id { xsd:string } & + attribute region { xsd:string }? & + EndpointRegion* & + EndpointInstance* +} + +Endpoints = element endpoints { + Endpoint+ +} diff --git a/config-model/src/main/resources/schema/version/7.x/docproc-standalone.rnc b/config-model/src/main/resources/schema/version/7.x/docproc-standalone.rnc new file mode 100644 index 00000000000..caba5327e25 --- /dev/null +++ b/config-model/src/main/resources/schema/version/7.x/docproc-standalone.rnc @@ -0,0 +1,6 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +include "common.rnc" +include "container.rnc" +include "searchchains.rnc" +include "docproc.rnc" +start = DocprocChainsV3 diff --git a/config-model/src/main/resources/schema/version/7.x/docproc.rnc b/config-model/src/main/resources/schema/version/7.x/docproc.rnc new file mode 100644 index 00000000000..1e7e28b2002 --- /dev/null +++ b/config-model/src/main/resources/schema/version/7.x/docproc.rnc @@ -0,0 +1,99 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +DocProc = element docproc { + DocProcV3 +} + +SchemaMapping = element map { + element field { + attribute doctype { text }?, + attribute in-document { text }, + attribute in-processor { text } + }+ +} + + + + +#Version 3 config: + +DocProcV3 = attribute version { "3.0" }, + (ClusterV3* & + OuterDocprocChainsV3? & + GenericConfig* +) + +DocprocClusterAttributes = attribute compressdocuments { xsd:boolean }? & + attribute numnodesperclient { xsd:positiveInteger }? & + attribute preferlocalnode { xsd:boolean }? & + attribute maxmessagesinqueue { xsd:nonNegativeInteger }? & + attribute maxqueuebytesize { xsd:string { minLength = "1" } }? & + attribute maxqueuewait { xsd:positiveInteger }? & + attribute maxconcurrentfactor { xsd:double { minExclusive = "0.0" maxExclusive = "1.0" } }? & + attribute documentexpansionfactor { xsd:double { minExclusive = "0.0" } }? & + attribute containercorememory { xsd:nonNegativeInteger }? + +ClusterV3 = element cluster { + attribute name { xsd:NCName } & + DocprocClusterAttributes? & + + element nodes { + Resources? & + attribute jvmargs { text }? & + attribute preload { text }? & + element node { + GenericConfig* & + service.attlist & + attribute maxmessagesinqueue { xsd:nonNegativeInteger }? & + attribute maxqueuebytesize { xsd:string { minLength = "1" } }? & + attribute maxqueuewait { xsd:positiveInteger }? + }+ + } & + GenericConfig* & + SchemaMapping? & + Component* & + Handler* & + DocprocChainsV3? +} + +DocprocChainsV3 = + element docprocchains { + DocumentProcessorV3* & + DocprocChainV3* & + GenericConfig* + } + +OuterDocprocChainsV3 = + element docprocchains { + DocumentProcessorV3* & + DocprocChainV3* + } + +DocprocChainV3 = + element docprocchain { + DocprocChainV3Contents + } + +DocprocChainV3Contents = attribute name { xsd:NCName }? & + ComponentId & + SchemaMapping? & + DocprocChainInheritance & + attribute documentprocessors { text }? & + DocumentProcessorV3* & + Phase* & + GenericConfig* + + +DocprocChainInheritance = + attribute inherits { text }? & + attribute excludes { text }? & + element inherits { + element docprocchain { ComponentSpec }* & + element exclude { ComponentSpec }* + }? + +DocumentProcessorV3 = + element documentprocessor { + BundleSpec & + SchemaMapping? & + GenericSearcherOrDocumentProcessor + } diff --git a/config-model/src/main/resources/schema/version/7.x/federation.rnc b/config-model/src/main/resources/schema/version/7.x/federation.rnc new file mode 100644 index 00000000000..8e341fa7a9c --- /dev/null +++ b/config-model/src/main/resources/schema/version/7.x/federation.rnc @@ -0,0 +1,74 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Schema for federation configuration inside the searchchains section. + +GenericSource = + GenericSearchChainInQrservers & + FederationOptions? + +Source = + element source { + GenericSource + } + +Provider = + element provider { + GenericSource & + attribute cachesize { xsd:string { pattern = "\d+(\.\d*)?\s*[kmgKMG]?" } }? & + attribute type { xsd:string }? & + attribute cluster { xsd:string }? & + + attribute yca-application-id { xsd:string }? & + attribute yca-cache-ttl { xsd:string { pattern = "\d+(\.\d*)?\s*m?s" } }? & + attribute yca-cache-retry-wait { xsd:string { pattern = "\d+(\.\d*)?\s*m?s" } }? & + YcaProxy? & + HttpProviderSearcherOptions & + + Source* + } + +YcaProxy = + element yca-proxy { + GenericNode? + } + +GenericNode = + attribute host { xsd:string } & + attribute port { xsd:int } + + +HttpProviderSearcherOptions = + attribute cacheweight { xsd:float { minInclusive = "0" } }? & + attribute path { xsd:string }? & + attribute readtimeout { xsd:string { pattern = "\d+(\.\d*)?\s*m?s" } }? & + attribute connectiontimeout { xsd:string { pattern = "\d+(\.\d*)?\s*m?s" } }? & + attribute connectionpooltimeout { xsd:string { pattern = "\d+(\.\d*)?\s*m?s" } }? & + attribute retries { xsd:int }? & + element nodes { + element node { + attribute host { xsd:string } & + attribute port { xsd:int } + }+ + } ? + +FederationOptions = + element federationoptions { + attribute optional { xsd:boolean }? & + attribute timeout { xsd:string { pattern = "\d+(\.\d*)?\s*m?s" } }? & + attribute requestTimeout { xsd:string { pattern = "\d+(\.\d*)?\s*m?s" } }? & + attribute default { xsd:boolean }? + } + +FederationSearcher = + element federation { + GenericSearcherOrDocumentProcessor & + element source { + ComponentSpec & + FederationOptions? + }* & + element target-selector { + ComponentDefinition + }? & + element source-set { + attribute inherits { string "default" } + }? + } diff --git a/config-model/src/main/resources/schema/genericcluster.rnc b/config-model/src/main/resources/schema/version/7.x/genericcluster.rnc index a749a592c09..a749a592c09 100644 --- a/config-model/src/main/resources/schema/genericcluster.rnc +++ b/config-model/src/main/resources/schema/version/7.x/genericcluster.rnc diff --git a/config-model/src/main/resources/schema/genericmodule.rnc b/config-model/src/main/resources/schema/version/7.x/genericmodule.rnc index 5f8ac3f7dda..5f8ac3f7dda 100644 --- a/config-model/src/main/resources/schema/genericmodule.rnc +++ b/config-model/src/main/resources/schema/version/7.x/genericmodule.rnc diff --git a/config-model/src/main/resources/schema/version/7.x/hosts.rnc b/config-model/src/main/resources/schema/version/7.x/hosts.rnc new file mode 100644 index 00000000000..d089b23804e --- /dev/null +++ b/config-model/src/main/resources/schema/version/7.x/hosts.rnc @@ -0,0 +1,10 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# RELAX NG Compact Syntax +# Vespa Hosts file + +element hosts { + element host { + attribute name { text }, + element alias { text }* + }+ +} diff --git a/config-model/src/main/resources/schema/version/7.x/legacygenericmodule.rnc b/config-model/src/main/resources/schema/version/7.x/legacygenericmodule.rnc new file mode 100644 index 00000000000..a54f7fd9afc --- /dev/null +++ b/config-model/src/main/resources/schema/version/7.x/legacygenericmodule.rnc @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Generic, nestable module + +LegacyGenericModule = element module { + attribute name { text } & + GenericConfig* & + LegacyGenericModule* +} diff --git a/config-model/src/main/resources/schema/version/7.x/processing.rnc b/config-model/src/main/resources/schema/version/7.x/processing.rnc new file mode 100644 index 00000000000..a753de70265 --- /dev/null +++ b/config-model/src/main/resources/schema/version/7.x/processing.rnc @@ -0,0 +1,39 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Schema for processing components and chains + +ProcessingRenderer = element renderer { + ComponentDefinition +} + +Processing = + element processing { + Processor* & + Chain* & + ProcessingRenderer* & + GenericConfig* + } + +ChainBaseContent = + ComponentId & + ChainInheritance & + GenericConfig* + +Chain = element chain { + ChainBaseContent & + Processor* & + Phase* + } + +ChainInheritance = + attribute inherits { text }? & + attribute excludes { text }? & + element inherits { + element chain { ComponentSpec }* & + element exclude { ComponentSpec }* + }? + +Processor = + element processor { + BundleSpec & + GenericSearcherOrDocumentProcessor + } diff --git a/vespa-http-client/CMakeLists.txt b/config-model/src/main/resources/schema/version/7.x/routing-standalone.rnc index 3fb22316a28..e95369fd192 100644 --- a/vespa-http-client/CMakeLists.txt +++ b/config-model/src/main/resources/schema/version/7.x/routing-standalone.rnc @@ -1,2 +1,4 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_jar(vespa-http-client-jar-with-dependencies.jar) +include "common.rnc" +include "routing.rnc" +start = Routing diff --git a/config-model/src/main/resources/schema/version/7.x/routing.rnc b/config-model/src/main/resources/schema/version/7.x/routing.rnc new file mode 100644 index 00000000000..5ca033b2fd7 --- /dev/null +++ b/config-model/src/main/resources/schema/version/7.x/routing.rnc @@ -0,0 +1,28 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# RELAX NG compact syntax pattern +# for Vespa MessageBus explicit routing config +Routing = element routing { + attribute version { "1.0" } & + element routingtable { + attribute protocol { string "document" } & + attribute verify { xsd:boolean }? & + element hop { + attribute name { text } & + attribute selector { text } & + attribute ignore-result { xsd:boolean }? & + element recipient { + attribute session { text } + }* + }* & + element route { + attribute name { text } & + attribute hops { text } + }* + }* & + element services { + attribute protocol { string "document" } & + element service { + attribute name { text } + }* + }* +} diff --git a/config-model/src/main/resources/schema/version/7.x/schemas.xml b/config-model/src/main/resources/schema/version/7.x/schemas.xml new file mode 100644 index 00000000000..c3f8e2be448 --- /dev/null +++ b/config-model/src/main/resources/schema/version/7.x/schemas.xml @@ -0,0 +1,7 @@ +<?xml version="1.0"?> +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<locatingRules xmlns="http://thaiopensource.com/ns/locating-rules/1.0"> + <documentElement localName="hosts" uri="hosts.rnc"/> + <documentElement localName="services" uri="services.rnc"/> + <documentElement localName="deployment" uri="deployment.rnc"/> +</locatingRules> diff --git a/config-model/src/main/resources/schema/searchchains-standalone.rnc b/config-model/src/main/resources/schema/version/7.x/searchchains-standalone.rnc index 6725627143d..6725627143d 100644 --- a/config-model/src/main/resources/schema/searchchains-standalone.rnc +++ b/config-model/src/main/resources/schema/version/7.x/searchchains-standalone.rnc diff --git a/config-model/src/main/resources/schema/version/7.x/searchchains.rnc b/config-model/src/main/resources/schema/version/7.x/searchchains.rnc new file mode 100644 index 00000000000..d4c9e8f4f98 --- /dev/null +++ b/config-model/src/main/resources/schema/version/7.x/searchchains.rnc @@ -0,0 +1,72 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#Schema for search chains and searchers inside the searchchains section. + +include "federation.rnc" + +SearchChains = + element searchchains { + Searcher* & + SearchChainInQrservers* & + GenericConfig* + } + +OuterSearchChains = + element searchchains { + Searcher* & + SearchChainInQrservers* + } + +SearchChainInQrservers = + element searchchain { + GenericSearchChainInQrservers + } | + Provider + +GenericSearchChainInQrservers = + ComponentId & + SearchChainInheritanceInQrservers & + attribute searchers { text }? & + Searcher* & + Phase* & + GenericConfig* + +SearchChainInheritanceInQrservers = + attribute inherits { text }? & + attribute excludes { text }? & + element inherits { + element searchchain { ComponentSpec }* & + element exclude { ComponentSpec }* + }? + +Searcher = + RegularSearcher | + FederationSearcher + +RegularSearcher = + element searcher { + BundleSpec & + GenericSearcherOrDocumentProcessor + } + + +GenericSearcherOrDocumentProcessor = + ComponentId & + SearcherOrDocumentProcessorDependencies & + GenericConfig* + +SearcherOrDocumentProcessorDependencies = + Dependencies & + attribute provides { text }? & + element provides { xsd:Name }* + +Dependencies = + attribute before { text }? & + attribute after { text }? & + element before { xsd:Name }* & + element after { xsd:Name }* + +Phase = + element phase { + ComponentId & + Dependencies + } diff --git a/config-model/src/main/resources/schema/version/7.x/services.rnc b/config-model/src/main/resources/schema/version/7.x/services.rnc new file mode 100644 index 00000000000..3a8ffe30563 --- /dev/null +++ b/config-model/src/main/resources/schema/version/7.x/services.rnc @@ -0,0 +1,28 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +include "common.rnc" +include "admin.rnc" +include "clients.rnc" +include "content.rnc" +include "docproc.rnc" +include "routing.rnc" +include "containercluster.rnc" +include "genericcluster.rnc" + +start = element services { + attribute version { "1.0" }? & + attribute application-type { "hosted-infrastructure" }? & + element legacy { element v7-geo-positions { xsd:boolean } }? & + GenericCluster* & + GenericConfig* & + Admin? & + Clients? & + Content* & + ContainerCluster* & + Routing? +} +| GenericCluster +| Admin +| Clients +| Content +| ContainerCluster + diff --git a/config-model/src/main/resources/schema/version/7.x/validation-overrides.rnc b/config-model/src/main/resources/schema/version/7.x/validation-overrides.rnc new file mode 100644 index 00000000000..a0caa10fc60 --- /dev/null +++ b/config-model/src/main/resources/schema/version/7.x/validation-overrides.rnc @@ -0,0 +1,13 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# RELAX NG Compact Syntax +# Vespa validation overrides + +start = element validation-overrides { + Allow* +} + +Allow = element allow { + attribute until { xsd:string } & + attribute comment { xsd:string }? & + text +} diff --git a/config-model/src/test/cfg/routing/invalidstoragepolicy/errors.txt b/config-model/src/test/cfg/routing/invalidstoragepolicy/errors.txt deleted file mode 100644 index 797dee3fd88..00000000000 --- a/config-model/src/test/cfg/routing/invalidstoragepolicy/errors.txt +++ /dev/null @@ -1 +0,0 @@ -Can't use storage policy with only slobrok in 5.0 and hierarchical grouping
\ No newline at end of file diff --git a/config-model/src/test/cfg/routing/invalidstoragepolicy/hosts.xml b/config-model/src/test/cfg/routing/invalidstoragepolicy/hosts.xml deleted file mode 100644 index 54e4487ce77..00000000000 --- a/config-model/src/test/cfg/routing/invalidstoragepolicy/hosts.xml +++ /dev/null @@ -1,7 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<hosts> - <host name="localhost"> - <alias>node1</alias> - </host> -</hosts> diff --git a/config-model/src/test/cfg/routing/invalidstoragepolicy/services.xml b/config-model/src/test/cfg/routing/invalidstoragepolicy/services.xml deleted file mode 100644 index 991ea2751a6..00000000000 --- a/config-model/src/test/cfg/routing/invalidstoragepolicy/services.xml +++ /dev/null @@ -1,11 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<services> - <admin version="2.0"> - <adminserver hostalias="node1" /> - </admin> - - <routing version="1.0"> - <route name="foo" hops="[Storage:cluster=storage;slobroks=donald.duck.example.com:14020"/> - </routing> -</services> diff --git a/config-model/src/test/derived/advanced/advanced.sd b/config-model/src/test/derived/advanced/advanced.sd index 69ea3a529b4..98a403dc44e 100644 --- a/config-model/src/test/derived/advanced/advanced.sd +++ b/config-model/src/test/derived/advanced/advanced.sd @@ -50,7 +50,6 @@ schema advanced { field title type string { indexing: input title_src | index - indexing-rewrite: none rank-type: identity stemming: none alias: headline @@ -65,7 +64,6 @@ schema advanced { case "": "myweb" | index | summary; default: input product_src | index | summary; } - indexing-rewrite: none normalizing: none } @@ -77,7 +75,6 @@ schema advanced { "baz" | index product2; }; } - indexing-rewrite: none normalizing: none } @@ -87,7 +84,6 @@ schema advanced { default: input product3_src | index product3 | summary product3; } normalizing: none - indexing-rewrite: none } field location type position { diff --git a/config-model/src/test/derived/advanced/attributes.cfg b/config-model/src/test/derived/advanced/attributes.cfg index b241d03cdb2..5ebd25b7678 100644 --- a/config-model/src/test/derived/advanced/attributes.cfg +++ b/config-model/src/test/derived/advanced/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,5 +27,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg index f61dd7069a1..75e892fda7c 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "elem_array.weight" attribute[].datatype INT32 @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,5 +57,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/attributeprefetch/attributes.cfg b/config-model/src/test/derived/attributeprefetch/attributes.cfg index 8b49dc1dc23..3168ec24b98 100644 --- a/config-model/src/test/derived/attributeprefetch/attributes.cfg +++ b/config-model/src/test/derived/attributeprefetch/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "multibyte" attribute[].datatype INT8 @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,7 +57,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "wsbyte" attribute[].datatype INT8 @@ -71,7 +67,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -92,7 +87,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "singleint" attribute[].datatype INT32 @@ -103,7 +97,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -124,7 +117,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "multiint" attribute[].datatype INT32 @@ -135,7 +127,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -156,7 +147,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "wsint" attribute[].datatype INT32 @@ -167,7 +157,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -188,7 +177,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "singlelong" attribute[].datatype INT64 @@ -199,7 +187,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -220,7 +207,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "multilong" attribute[].datatype INT64 @@ -231,7 +217,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -252,7 +237,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "wslong" attribute[].datatype INT64 @@ -263,7 +247,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -284,7 +267,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "singlefloat" attribute[].datatype FLOAT @@ -295,7 +277,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -316,7 +297,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "multifloat" attribute[].datatype FLOAT @@ -327,7 +307,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -348,7 +327,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "singledouble" attribute[].datatype DOUBLE @@ -359,7 +337,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -380,7 +357,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "multidouble" attribute[].datatype DOUBLE @@ -391,7 +367,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -412,7 +387,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "singlestring" attribute[].datatype STRING @@ -423,7 +397,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -444,7 +417,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "multistring" attribute[].datatype STRING @@ -455,7 +427,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -476,7 +447,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "wsstring" attribute[].datatype STRING @@ -487,7 +457,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -508,5 +477,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/attributes/attributes.cfg b/config-model/src/test/derived/attributes/attributes.cfg index cf473ad4854..07a3631819d 100644 --- a/config-model/src/test/derived/attributes/attributes.cfg +++ b/config-model/src/test/derived/attributes/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "a2" attribute[].datatype STRING @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,7 +57,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "a3" attribute[].datatype STRING @@ -71,7 +67,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -92,7 +87,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "a5" attribute[].datatype STRING @@ -103,7 +97,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -124,7 +117,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "a6" attribute[].datatype STRING @@ -135,7 +127,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -156,7 +147,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "b1" attribute[].datatype STRING @@ -167,7 +157,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -188,7 +177,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "b2" attribute[].datatype STRING @@ -199,7 +187,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -220,7 +207,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "b3" attribute[].datatype STRING @@ -231,7 +217,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -252,7 +237,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "b4" attribute[].datatype INT32 @@ -263,7 +247,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -284,7 +267,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "b5" attribute[].datatype INT32 @@ -295,7 +277,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -316,7 +297,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "b6" attribute[].datatype INT64 @@ -327,7 +307,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -348,7 +327,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "b7" attribute[].datatype INT32 @@ -359,7 +337,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -380,7 +357,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "a9" attribute[].datatype INT32 @@ -391,7 +367,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -412,7 +387,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "a10" attribute[].datatype INT32 @@ -423,7 +397,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -444,7 +417,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "a11" attribute[].datatype INT32 @@ -455,7 +427,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -476,7 +447,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "a12" attribute[].datatype INT32 @@ -487,7 +457,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -508,7 +477,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "a13" attribute[].datatype TENSOR @@ -519,7 +487,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -540,7 +507,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "a7_arr" attribute[].datatype STRING @@ -551,7 +517,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -572,7 +537,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "a8_arr" attribute[].datatype STRING @@ -583,7 +547,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -604,5 +567,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/attributes/summarymap.cfg b/config-model/src/test/derived/attributes/summarymap.cfg index 78bf22b7109..a96f1e3858d 100644 --- a/config-model/src/test/derived/attributes/summarymap.cfg +++ b/config-model/src/test/derived/attributes/summarymap.cfg @@ -50,6 +50,3 @@ override[].arguments "a11" override[].field "a12" override[].command "attribute" override[].arguments "a12" -override[].field "a13" -override[].command "attribute" -override[].arguments "a13" diff --git a/config-model/src/test/derived/complex/attributes.cfg b/config-model/src/test/derived/complex/attributes.cfg index 86b6b9ebc13..0772b797777 100644 --- a/config-model/src/test/derived/complex/attributes.cfg +++ b/config-model/src/test/derived/complex/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge true attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "fleeting" attribute[].datatype FLOAT @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,7 +57,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "fleeting2" attribute[].datatype FLOAT @@ -71,7 +67,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -92,7 +87,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "foundat" attribute[].datatype INT64 @@ -103,7 +97,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -124,7 +117,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "collapseby" attribute[].datatype INT32 @@ -135,7 +127,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -156,7 +147,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "ts" attribute[].datatype INT64 @@ -167,7 +157,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -188,7 +177,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "combineda" attribute[].datatype INT32 @@ -199,7 +187,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -220,7 +207,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "year_arr" attribute[].datatype INT32 @@ -231,7 +217,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -252,7 +237,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "year_sub" attribute[].datatype INT32 @@ -263,7 +247,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -284,5 +267,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/complex/complex.sd b/config-model/src/test/derived/complex/complex.sd index 3dcea95ed72..3e28801c53a 100644 --- a/config-model/src/test/derived/complex/complex.sd +++ b/config-model/src/test/derived/complex/complex.sd @@ -41,7 +41,6 @@ schema complex { field prefixenabled type string { indexing: index | attribute index: prefix - attribute: huge normalizing: none stemming: shortest } diff --git a/config-model/src/test/derived/function_arguments/attributes.cfg b/config-model/src/test/derived/function_arguments/attributes.cfg index 8320063ab66..101fe901ce2 100644 --- a/config-model/src/test/derived/function_arguments/attributes.cfg +++ b/config-model/src/test/derived/function_arguments/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "t1" attribute[].datatype TENSOR @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,7 +57,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "t2" attribute[].datatype TENSOR @@ -71,7 +67,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -92,5 +87,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/function_arguments_with_expressions/attributes.cfg b/config-model/src/test/derived/function_arguments_with_expressions/attributes.cfg index 8bcad42c882..7280596be5b 100644 --- a/config-model/src/test/derived/function_arguments_with_expressions/attributes.cfg +++ b/config-model/src/test/derived/function_arguments_with_expressions/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "t1" attribute[].datatype TENSOR @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,7 +57,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "t2" attribute[].datatype TENSOR @@ -71,7 +67,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -92,5 +87,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/hnsw_index/attributes.cfg b/config-model/src/test/derived/hnsw_index/attributes.cfg index 403f78be2f7..99a0d93fd35 100644 --- a/config-model/src/test/derived/hnsw_index/attributes.cfg +++ b/config-model/src/test/derived/hnsw_index/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric ANGULAR attribute[].index.hnsw.enabled true attribute[].index.hnsw.maxlinkspernode 32 attribute[].index.hnsw.neighborstoexploreatinsert 300 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing false attribute[].name "t2" attribute[].datatype TENSOR @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,5 +57,4 @@ attribute[].distancemetric GEODEGREES attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg b/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg index bcf1b101caf..70f1f21b465 100644 --- a/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg +++ b/config-model/src/test/derived/imported_fields_inherited_reference/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "ref_from_b" attribute[].datatype REFERENCE @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,7 +57,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "from_a_int_field" attribute[].datatype INT32 @@ -71,7 +67,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -92,7 +87,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "from_b_int_field" attribute[].datatype INT32 @@ -103,7 +97,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -124,5 +117,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/imported_position_field/attributes.cfg b/config-model/src/test/derived/imported_position_field/attributes.cfg index 34219243d14..2d1d1b232b1 100644 --- a/config-model/src/test/derived/imported_position_field/attributes.cfg +++ b/config-model/src/test/derived/imported_position_field/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "my_pos_zcurve" attribute[].datatype INT64 @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,5 +57,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/imported_struct_fields/attributes.cfg b/config-model/src/test/derived/imported_struct_fields/attributes.cfg index c974b46a904..31eb66776d4 100644 --- a/config-model/src/test/derived/imported_struct_fields/attributes.cfg +++ b/config-model/src/test/derived/imported_struct_fields/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "my_elem_array.name" attribute[].datatype STRING @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,7 +57,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "my_elem_array.weight" attribute[].datatype INT32 @@ -71,7 +67,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -92,7 +87,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "my_elem_map.key" attribute[].datatype STRING @@ -103,7 +97,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -124,7 +117,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "my_elem_map.value.name" attribute[].datatype STRING @@ -135,7 +127,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -156,7 +147,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "my_elem_map.value.weight" attribute[].datatype INT32 @@ -167,7 +157,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -188,7 +177,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "my_str_int_map.key" attribute[].datatype STRING @@ -199,7 +187,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -220,7 +207,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "my_str_int_map.value" attribute[].datatype INT32 @@ -231,7 +217,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -252,5 +237,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/importedfields/attributes.cfg b/config-model/src/test/derived/importedfields/attributes.cfg index 12970887510..7f2cb39dc3d 100644 --- a/config-model/src/test/derived/importedfields/attributes.cfg +++ b/config-model/src/test/derived/importedfields/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "b_ref" attribute[].datatype REFERENCE @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,7 +57,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "b_ref_with_summary" attribute[].datatype REFERENCE @@ -71,7 +67,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -92,7 +87,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "my_int_field" attribute[].datatype INT32 @@ -103,7 +97,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -124,7 +117,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "my_string_field" attribute[].datatype STRING @@ -135,7 +127,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -156,7 +147,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "my_int_array_field" attribute[].datatype INT32 @@ -167,7 +157,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -188,7 +177,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "my_int_wset_field" attribute[].datatype INT32 @@ -199,7 +187,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -220,7 +207,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "my_ancient_int_field" attribute[].datatype INT32 @@ -231,7 +217,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -252,5 +237,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/inheritance/attributes.cfg b/config-model/src/test/derived/inheritance/attributes.cfg index 65514adf543..9ab0cb0c67e 100644 --- a/config-model/src/test/derived/inheritance/attributes.cfg +++ b/config-model/src/test/derived/inheritance/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "overridden" attribute[].datatype INT32 @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,7 +57,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "onlymother" attribute[].datatype STRING @@ -71,7 +67,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -92,5 +87,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/inheritfromparent/attributes.cfg b/config-model/src/test/derived/inheritfromparent/attributes.cfg index 59e90691421..27a18a58bfd 100644 --- a/config-model/src/test/derived/inheritfromparent/attributes.cfg +++ b/config-model/src/test/derived/inheritfromparent/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,5 +27,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/map_attribute/attributes.cfg b/config-model/src/test/derived/map_attribute/attributes.cfg index 4f982ba9ec2..2b62418480b 100644 --- a/config-model/src/test/derived/map_attribute/attributes.cfg +++ b/config-model/src/test/derived/map_attribute/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "str_map.value" attribute[].datatype STRING @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,7 +57,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "int_map.key" attribute[].datatype INT32 @@ -71,7 +67,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -92,5 +87,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg index 27c3794271a..216967a93dc 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "str_elem_map.value.name" attribute[].datatype STRING @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,7 +57,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "str_elem_map.value.weight" attribute[].datatype INT32 @@ -71,7 +67,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -92,7 +87,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "int_elem_map.key" attribute[].datatype INT32 @@ -103,7 +97,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -124,7 +117,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "int_elem_map.value.name" attribute[].datatype STRING @@ -135,7 +127,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -156,5 +147,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/multiplesummaries/attributes.cfg b/config-model/src/test/derived/multiplesummaries/attributes.cfg index 44c94c73781..bb323041c0a 100644 --- a/config-model/src/test/derived/multiplesummaries/attributes.cfg +++ b/config-model/src/test/derived/multiplesummaries/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "adynamic" attribute[].datatype STRING @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,7 +57,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "abolded" attribute[].datatype STRING @@ -71,7 +67,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -92,7 +87,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "c" attribute[].datatype STRING @@ -103,7 +97,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -124,7 +117,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "loc_pos_zcurve" attribute[].datatype INT64 @@ -135,7 +127,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -156,5 +147,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/music/attributes.cfg b/config-model/src/test/derived/music/attributes.cfg index 5dea7831ec4..0c37cdb207f 100644 --- a/config-model/src/test/derived/music/attributes.cfg +++ b/config-model/src/test/derived/music/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "pto" attribute[].datatype INT32 @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,7 +57,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "mid" attribute[].datatype INT32 @@ -71,7 +67,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -92,7 +87,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "weight" attribute[].datatype FLOAT @@ -103,7 +97,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -124,7 +117,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "bgnpfrom" attribute[].datatype FLOAT @@ -135,7 +127,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -156,7 +147,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "newestedition" attribute[].datatype INT32 @@ -167,7 +157,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -188,7 +177,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "year" attribute[].datatype INT32 @@ -199,7 +187,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -220,7 +207,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "did" attribute[].datatype INT32 @@ -231,7 +217,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -252,7 +237,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "cbid" attribute[].datatype INT32 @@ -263,7 +247,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -284,7 +267,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "hiphopvalue_arr" attribute[].datatype STRING @@ -295,7 +277,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -316,7 +297,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "metalvalue_arr" attribute[].datatype STRING @@ -327,7 +307,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -348,5 +327,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/newrank/attributes.cfg b/config-model/src/test/derived/newrank/attributes.cfg index 5ba58f3081b..60d174d4b12 100644 --- a/config-model/src/test/derived/newrank/attributes.cfg +++ b/config-model/src/test/derived/newrank/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "pto" attribute[].datatype INT32 @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,7 +57,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "mid" attribute[].datatype INT32 @@ -71,7 +67,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -92,7 +87,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "weight" attribute[].datatype FLOAT @@ -103,7 +97,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -124,7 +117,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "bgnpfrom" attribute[].datatype FLOAT @@ -135,7 +127,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -156,7 +147,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "newestedition" attribute[].datatype INT32 @@ -167,7 +157,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -188,7 +177,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "year" attribute[].datatype INT32 @@ -199,7 +187,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -220,7 +207,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "did" attribute[].datatype INT32 @@ -231,7 +217,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -252,7 +237,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "scorekey" attribute[].datatype INT32 @@ -263,7 +247,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -284,7 +267,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "cbid" attribute[].datatype INT32 @@ -295,7 +277,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -316,5 +297,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/predicate_attribute/attributes.cfg b/config-model/src/test/derived/predicate_attribute/attributes.cfg index 513f4320f58..d5810e78185 100644 --- a/config-model/src/test/derived/predicate_attribute/attributes.cfg +++ b/config-model/src/test/derived/predicate_attribute/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,5 +27,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/prefixexactattribute/attributes.cfg b/config-model/src/test/derived/prefixexactattribute/attributes.cfg index 7051a228ba3..65a0df45fd6 100644 --- a/config-model/src/test/derived/prefixexactattribute/attributes.cfg +++ b/config-model/src/test/derived/prefixexactattribute/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "attributefield2" attribute[].datatype STRING @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,5 +57,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/rankingexpression/summary.cfg b/config-model/src/test/derived/rankingexpression/summary.cfg index b9f89f4a76d..a6ba84ca7a9 100644 --- a/config-model/src/test/derived/rankingexpression/summary.cfg +++ b/config-model/src/test/derived/rankingexpression/summary.cfg @@ -17,7 +17,7 @@ classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "documentid" classes[].fields[].type "longstring" -classes[].id 203924099 +classes[].id 1736696699 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false classes[].fields[].name "year" @@ -38,8 +38,6 @@ classes[].fields[].name "bar3" classes[].fields[].type "integer" classes[].fields[].name "bar4" classes[].fields[].type "integer" -classes[].fields[].name "t1" -classes[].fields[].type "tensor" classes[].fields[].name "rankfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "summaryfeatures" diff --git a/config-model/src/test/derived/rankingexpression/summarymap.cfg b/config-model/src/test/derived/rankingexpression/summarymap.cfg index bd92b36548c..e8d9bd08ca7 100644 --- a/config-model/src/test/derived/rankingexpression/summarymap.cfg +++ b/config-model/src/test/derived/rankingexpression/summarymap.cfg @@ -32,6 +32,3 @@ override[].arguments "bar3" override[].field "bar4" override[].command "attribute" override[].arguments "bar4" -override[].field "t1" -override[].command "attribute" -override[].arguments "t1" diff --git a/config-model/src/test/derived/reference_fields/attributes.cfg b/config-model/src/test/derived/reference_fields/attributes.cfg index e1868700198..a79bf71f65c 100644 --- a/config-model/src/test/derived/reference_fields/attributes.cfg +++ b/config-model/src/test/derived/reference_fields/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "other_ref" attribute[].datatype REFERENCE @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,7 +57,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "yet_another_ref" attribute[].datatype REFERENCE @@ -71,7 +67,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -92,5 +87,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/schemainheritance/attributes.cfg b/config-model/src/test/derived/schemainheritance/attributes.cfg index 26bd10a262c..a6ee394dcf3 100644 --- a/config-model/src/test/derived/schemainheritance/attributes.cfg +++ b/config-model/src/test/derived/schemainheritance/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "child_field" attribute[].datatype STRING @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,7 +57,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "parent_field" attribute[].datatype STRING @@ -71,7 +67,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -92,7 +87,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "parent_imported" attribute[].datatype STRING @@ -103,7 +97,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -124,7 +117,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "child_imported" attribute[].datatype STRING @@ -135,7 +127,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -156,5 +147,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/sorting/attributes.cfg b/config-model/src/test/derived/sorting/attributes.cfg index 2bb7078a37d..f2074be891f 100644 --- a/config-model/src/test/derived/sorting/attributes.cfg +++ b/config-model/src/test/derived/sorting/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "syntaxcheck2a" attribute[].datatype STRING @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending false @@ -60,7 +57,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "syntaxcheck3a" attribute[].datatype STRING @@ -71,7 +67,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -92,7 +87,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "syntaxcheck4a" attribute[].datatype STRING @@ -103,7 +97,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending false @@ -124,7 +117,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "syntaxcheck5a" attribute[].datatype STRING @@ -135,7 +127,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -156,7 +147,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "syntaxcheck1b" attribute[].datatype STRING @@ -167,7 +157,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -188,7 +177,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "syntaxcheck2b" attribute[].datatype STRING @@ -199,7 +187,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending false @@ -220,7 +207,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "syntaxcheck3b" attribute[].datatype STRING @@ -231,7 +217,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -252,7 +237,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "syntaxcheck4b" attribute[].datatype STRING @@ -263,7 +247,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending false @@ -284,7 +267,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "syntaxcheck5b" attribute[].datatype STRING @@ -295,7 +277,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -316,7 +297,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "infieldonly" attribute[].datatype STRING @@ -327,7 +307,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending false @@ -348,5 +327,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/structandfieldset/attributes.cfg b/config-model/src/test/derived/structandfieldset/attributes.cfg index b441857c9c7..42557c2de3a 100644 --- a/config-model/src/test/derived/structandfieldset/attributes.cfg +++ b/config-model/src/test/derived/structandfieldset/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "people.first_name" attribute[].datatype STRING @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,7 +57,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "people.last_name" attribute[].datatype STRING @@ -71,7 +67,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -92,5 +87,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/tensor/attributes.cfg b/config-model/src/test/derived/tensor/attributes.cfg index 8e28044a724..4cfb2eb97df 100644 --- a/config-model/src/test/derived/tensor/attributes.cfg +++ b/config-model/src/test/derived/tensor/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "f3" attribute[].datatype TENSOR @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,7 +57,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "f4" attribute[].datatype TENSOR @@ -71,7 +67,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -92,7 +87,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "f5" attribute[].datatype TENSOR @@ -103,7 +97,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -124,7 +117,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "f6" attribute[].datatype FLOAT @@ -135,7 +127,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -156,5 +147,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/derived/tensor/summary.cfg b/config-model/src/test/derived/tensor/summary.cfg index badb7c37829..a5b38684e52 100644 --- a/config-model/src/test/derived/tensor/summary.cfg +++ b/config-model/src/test/derived/tensor/summary.cfg @@ -17,17 +17,9 @@ classes[].fields[].name "summaryfeatures" classes[].fields[].type "featuredata" classes[].fields[].name "documentid" classes[].fields[].type "longstring" -classes[].id 1476352352 +classes[].id 219619290 classes[].name "attributeprefetch" classes[].omitsummaryfeatures false -classes[].fields[].name "f2" -classes[].fields[].type "tensor" -classes[].fields[].name "f3" -classes[].fields[].type "tensor" -classes[].fields[].name "f4" -classes[].fields[].type "tensor" -classes[].fields[].name "f5" -classes[].fields[].type "tensor" classes[].fields[].name "f6" classes[].fields[].type "float" classes[].fields[].name "rankfeatures" diff --git a/config-model/src/test/derived/types/attributes.cfg b/config-model/src/test/derived/types/attributes.cfg index 37b56481506..fbef851b4af 100644 --- a/config-model/src/test/derived/types/attributes.cfg +++ b/config-model/src/test/derived/types/attributes.cfg @@ -7,7 +7,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -28,7 +27,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "along" attribute[].datatype INT64 @@ -39,7 +37,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -60,7 +57,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "abool" attribute[].datatype BOOL @@ -71,7 +67,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -92,7 +87,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "ashortfloat" attribute[].datatype FLOAT16 @@ -103,7 +97,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -124,7 +117,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "arrayfield" attribute[].datatype INT32 @@ -135,7 +127,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -156,7 +147,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "setfield" attribute[].datatype STRING @@ -167,7 +157,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -188,7 +177,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "setfield2" attribute[].datatype STRING @@ -199,7 +187,6 @@ attribute[].match UNCASED attribute[].removeifzero true attribute[].createifnonexistent true attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -220,7 +207,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "setfield3" attribute[].datatype STRING @@ -231,7 +217,6 @@ attribute[].match UNCASED attribute[].removeifzero true attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -252,7 +237,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "setfield4" attribute[].datatype STRING @@ -263,7 +247,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent true attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -284,7 +267,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "tagfield" attribute[].datatype STRING @@ -295,7 +277,6 @@ attribute[].match UNCASED attribute[].removeifzero true attribute[].createifnonexistent true attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -316,7 +297,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "juletre" attribute[].datatype INT64 @@ -327,7 +307,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch true -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -348,7 +327,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "album1" attribute[].datatype STRING @@ -359,7 +337,6 @@ attribute[].match UNCASED attribute[].removeifzero true attribute[].createifnonexistent true attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -380,7 +357,6 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true attribute[].name "other" attribute[].datatype INT64 @@ -391,7 +367,6 @@ attribute[].match UNCASED attribute[].removeifzero false attribute[].createifnonexistent false attribute[].fastsearch false -attribute[].huge false attribute[].paged false attribute[].ismutable false attribute[].sortascending true @@ -412,5 +387,4 @@ attribute[].distancemetric EUCLIDEAN attribute[].index.hnsw.enabled false attribute[].index.hnsw.maxlinkspernode 16 attribute[].index.hnsw.neighborstoexploreatinsert 200 -attribute[].index.hnsw.distancemetric EUCLIDEAN attribute[].index.hnsw.multithreadedindexing true diff --git a/config-model/src/test/examples/attributesettings.sd b/config-model/src/test/examples/attributesettings.sd index e7bb2062227..c87aad5b59b 100644 --- a/config-model/src/test/examples/attributesettings.sd +++ b/config-model/src/test/examples/attributesettings.sd @@ -5,7 +5,6 @@ search attributesettings { field f1 type long { indexing: attribute - attribute: huge } field f2 type long { diff --git a/config-model/src/test/examples/indexrewrite.sd b/config-model/src/test/examples/indexrewrite.sd index ee6fe21a15c..1fd9c3a2b79 100644 --- a/config-model/src/test/examples/indexrewrite.sd +++ b/config-model/src/test/examples/indexrewrite.sd @@ -8,7 +8,6 @@ search indexrewrite { field title type string { indexing: input title_src | lowercase | normalize | tokenize | index - indexing-rewrite: none rank-type: about stemming: none alias: headline diff --git a/config-model/src/test/examples/multiplesummaries.sd b/config-model/src/test/examples/multiplesummaries.sd index 83e817718aa..07ab3c5a104 100644 --- a/config-model/src/test/examples/multiplesummaries.sd +++ b/config-model/src/test/examples/multiplesummaries.sd @@ -28,11 +28,6 @@ search multiplesummaries { summary field3 type array<int> { } - # TODO disallow duplicates - summary field1 type weightedset<string> { - source: field1 - } - } } diff --git a/config-model/src/test/java/com/yahoo/config/model/producer/AbstractConfigProducerTest.java b/config-model/src/test/java/com/yahoo/config/model/producer/AbstractConfigProducerTest.java index 5d62abcc4df..04ae4c76f27 100644 --- a/config-model/src/test/java/com/yahoo/config/model/producer/AbstractConfigProducerTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/producer/AbstractConfigProducerTest.java @@ -25,7 +25,7 @@ public class AbstractConfigProducerTest { producer.getConfig(builder); LogdConfig config = new LogdConfig(builder); assertEquals("bar", config.logserver().host()); - assertEquals(1338, config.logserver().port()); + assertEquals(1338, config.logserver().rpcport()); } @Test @@ -38,7 +38,7 @@ public class AbstractConfigProducerTest { producer.getConfig(builder); LogdConfig config = new LogdConfig(builder); assertEquals("foo", config.logserver().host()); - assertEquals(1337, config.logserver().port()); + assertEquals(1337, config.logserver().rpcport()); } private static class MockLogdProducer extends AbstractConfigProducer implements LogdConfig.Producer { @@ -49,7 +49,7 @@ public class AbstractConfigProducerTest { @Override public void getConfig(LogdConfig.Builder builder) { - builder.logserver(new LogdConfig.Logserver.Builder().host("bar").port(1338)); + builder.logserver(new LogdConfig.Logserver.Builder().host("bar").rpcport(1338)); } } @@ -67,7 +67,7 @@ public class AbstractConfigProducerTest { @Override public void getConfig(LogdConfig.Builder builder) { - builder.logserver(new LogdConfig.Logserver.Builder().host("foo").port(1337)); + builder.logserver(new LogdConfig.Logserver.Builder().host("foo").rpcport(1337)); } } } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index f164a1045b6..5e56efc4460 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -249,108 +249,101 @@ public class ModelProvisioningTest { @Test public void testCombinedCluster() { - var containerElements = Set.of("jdisc", "container"); - for (var containerElement : containerElements) { - String xmlWithNodes = - "<?xml version='1.0' encoding='utf-8' ?>" + - "<services>" + - " <" + containerElement + " version='1.0' id='container1'>" + - " <search/>" + - " <nodes of='content1'/>" + - " </" + containerElement + ">" + - " <content version='1.0' id='content1'>" + - " <redundancy>2</redundancy>" + - " <documents>" + - " <document type='type1' mode='index'/>" + - " </documents>" + - " <nodes count='2'>" + - " <resources vcpu='1' memory='3Gb' disk='9Gb'/>" + - " </nodes>" + - " </content>" + - "</services>"; - VespaModelTester tester = new VespaModelTester(); - tester.addHosts(5); - VespaModel model = tester.createModel(xmlWithNodes, true); - assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size()); - assertEquals("Nodes in container1", 2, model.getContainerClusters().get("container1").getContainers().size()); - assertEquals("Heap size is lowered with combined clusters", - 18, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); - assertEquals("Memory for proton is lowered to account for the jvm heap", - (long)((3 - reservedMemoryGb) * (Math.pow(1024, 3)) * (1 - 0.18)), protonMemorySize(model.getContentClusters().get("content1"))); - assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); - assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model); - } + String xmlWithNodes = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <container version='1.0' id='container1'>" + + " <search/>" + + " <nodes of='content1'/>" + + " </container>" + + " <content version='1.0' id='content1'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='2'>" + + " <resources vcpu='1' memory='3Gb' disk='9Gb'/>" + + " </nodes>" + + " </content>" + + "</services>"; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(5); + VespaModel model = tester.createModel(xmlWithNodes, true); + assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size()); + assertEquals("Nodes in container1", 2, model.getContainerClusters().get("container1").getContainers().size()); + assertEquals("Heap size is lowered with combined clusters", + 18, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); + assertEquals("Memory for proton is lowered to account for the jvm heap", + (long) ((3 - reservedMemoryGb) * (Math.pow(1024, 3)) * (1 - 0.18)), protonMemorySize(model.getContentClusters() + .get("content1"))); + assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); + assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model); } @Test public void testCombinedClusterWithJvmHeapSizeOverride() { - var containerElements = Set.of("jdisc", "container"); - for (var containerElement : containerElements) { - String xmlWithNodes = - "<?xml version='1.0' encoding='utf-8' ?>" + - "<services>" + - " <" + containerElement + " version='1.0' id='container1'>" + - " <search/>" + - " <nodes of='content1'>" + - " <jvm allocated-memory=\"30%\"/>" + - " </nodes>" + - " </" + containerElement + ">" + - " <content version='1.0' id='content1'>" + - " <redundancy>2</redundancy>" + - " <documents>" + - " <document type='type1' mode='index'/>" + - " </documents>" + - " <nodes count='2'>" + - " <resources vcpu='1' memory='3Gb' disk='9Gb'/>" + - " </nodes>" + - " </content>" + - "</services>"; - VespaModelTester tester = new VespaModelTester(); - tester.addHosts(5); - VespaModel model = tester.createModel(xmlWithNodes, true); - assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size()); - assertEquals("Nodes in container1", 2, model.getContainerClusters().get("container1").getContainers().size()); - assertEquals("Heap size is lowered with combined clusters", - 30, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); - assertEquals("Memory for proton is lowered to account for the jvm heap", - (long)((3 - reservedMemoryGb) * (Math.pow(1024, 3)) * (1 - 0.30)), protonMemorySize(model.getContentClusters().get("content1"))); - assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); - assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model); - } + String xmlWithNodes = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <container version='1.0' id='container1'>" + + " <search/>" + + " <nodes of='content1'>" + + " <jvm allocated-memory=\"30%\"/>" + + " </nodes>" + + " </container>" + + " <content version='1.0' id='content1'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='2'>" + + " <resources vcpu='1' memory='3Gb' disk='9Gb'/>" + + " </nodes>" + + " </content>" + + "</services>"; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(5); + VespaModel model = tester.createModel(xmlWithNodes, true); + assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size()); + assertEquals("Nodes in container1", 2, model.getContainerClusters().get("container1").getContainers().size()); + assertEquals("Heap size is lowered with combined clusters", + 30, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); + assertEquals("Memory for proton is lowered to account for the jvm heap", + (long) ((3 - reservedMemoryGb) * (Math.pow(1024, 3)) * (1 - 0.30)), protonMemorySize(model.getContentClusters() + .get("content1"))); + assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); + assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model); } /** For comparison with the above */ @Test public void testNonCombinedCluster() { - var containerElements = Set.of("jdisc", "container"); - for (var containerElement : containerElements) { - String xmlWithNodes = - "<?xml version='1.0' encoding='utf-8' ?>" + - "<services>" + - " <" + containerElement + " version='1.0' id='container1'>" + - " <search/>" + - " <nodes count='2'/>" + - " </" + containerElement + ">" + - " <content version='1.0' id='content1'>" + - " <redundancy>2</redundancy>" + - " <documents>" + - " <document type='type1' mode='index'/>" + - " </documents>" + - " <nodes count='2'>" + - " <resources vcpu='1' memory='3Gb' disk='9Gb'/>" + - " </nodes>" + - " </content>" + - "</services>"; - VespaModelTester tester = new VespaModelTester(); - tester.addHosts(7); - VespaModel model = tester.createModel(xmlWithNodes, true); - assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size()); - assertEquals("Nodes in container1", 2, model.getContainerClusters().get("container1").getContainers().size()); - assertEquals("Heap size is normal", - 70, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); - assertEquals("Memory for proton is normal", - (long)((3 - reservedMemoryGb) * (Math.pow(1024, 3))), protonMemorySize(model.getContentClusters().get("content1"))); - } + String xmlWithNodes = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <container version='1.0' id='container1'>" + + " <search/>" + + " <nodes count='2'/>" + + " </container>" + + " <content version='1.0' id='content1'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='2'>" + + " <resources vcpu='1' memory='3Gb' disk='9Gb'/>" + + " </nodes>" + + " </content>" + + "</services>"; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(7); + VespaModel model = tester.createModel(xmlWithNodes, true); + assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size()); + assertEquals("Nodes in container1", 2, model.getContainerClusters().get("container1").getContainers().size()); + assertEquals("Heap size is normal", + 70, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); + assertEquals("Memory for proton is normal", + (long) ((3 - reservedMemoryGb) * (Math.pow(1024, 3))), protonMemorySize(model.getContentClusters().get("content1"))); } @Test @@ -463,35 +456,31 @@ public class ModelProvisioningTest { @Test public void testCombinedClusterWithZooKeeperFails() { - var containerElements = Set.of("jdisc", "container"); - for (var containerElement : containerElements) { - String xmlWithNodes = - "<?xml version='1.0' encoding='utf-8' ?>" + - "<services>" + - " <" + containerElement + " version='1.0' id='container1'>" + - " <search/>" + - " <nodes of='content1'/>" + - " <zookeeper />" + - " </" + containerElement + ">" + - " <content version='1.0' id='content1'>" + - " <redundancy>2</redundancy>" + - " <documents>" + - " <document type='type1' mode='index'/>" + - " </documents>" + - " <nodes count='2'>" + - " <resources vcpu='1' memory='3Gb' disk='9Gb'/>" + - " </nodes>" + - " </content>" + - "</services>"; - VespaModelTester tester = new VespaModelTester(); - tester.addHosts(2); - try { - tester.createModel(xmlWithNodes, true); - fail("ZooKeeper should not be allowed on combined clusters"); - } - catch (IllegalArgumentException e) { - assertEquals("A combined cluster cannot run ZooKeeper", e.getMessage()); - } + String xmlWithNodes = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <container version='1.0' id='container1'>" + + " <search/>" + + " <nodes of='content1'/>" + + " <zookeeper />" + + " </container>" + + " <content version='1.0' id='content1'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='2'>" + + " <resources vcpu='1' memory='3Gb' disk='9Gb'/>" + + " </nodes>" + + " </content>" + + "</services>"; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(2); + try { + tester.createModel(xmlWithNodes, true); + fail("ZooKeeper should not be allowed on combined clusters"); + } catch (IllegalArgumentException e) { + assertEquals("A combined cluster cannot run ZooKeeper", e.getMessage()); } } @@ -1107,16 +1096,13 @@ public class ModelProvisioningTest { int numberOfHosts = 3; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, false, "node-1-3-50-03"); - assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); - - ContentCluster cluster = model.getContentClusters().get("bar"); - assertEquals(2, cluster.redundancy().effectiveInitialRedundancy()); - assertEquals(2, cluster.redundancy().effectiveFinalRedundancy()); - assertEquals(2, cluster.redundancy().effectiveReadyCopies()); - assertEquals("1|*", cluster.getRootGroup().getPartitions().get()); - assertEquals(0, cluster.getRootGroup().getNodes().size()); - assertEquals(2, cluster.getRootGroup().getSubgroups().size()); + try { + VespaModel model = tester.createModel(services, false, "node-1-3-50-03"); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("Cluster 'bar' specifies redundancy 2, but it cannot be higher than the minimum nodes per group, which is 1", Exceptions.toMessageString(e)); + } } @Test @@ -1549,22 +1535,6 @@ public class ModelProvisioningTest { } @Test - public void testJvmArgs() { - String services = - "<?xml version='1.0' encoding='utf-8' ?>\n" + - "<container version='1.0'>" + - " <search/>" + - " <nodes jvmargs='-DfooOption=xyz' count='3'/>" + - "</container>"; - int numberOfHosts = 3; - VespaModelTester tester = new VespaModelTester(); - tester.addHosts(numberOfHosts); - VespaModel model = tester.createModel(services, true); - assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); - assertEquals("-DfooOption=xyz", model.getContainerClusters().get("container").getContainers().get(0).getAssignedJvmOptions()); - } - - @Test public void testJvmOptions() { String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + @@ -1583,30 +1553,6 @@ public class ModelProvisioningTest { } @Test - public void testFailWhenBothJvmOptionsAndJvmArgs() { - String services = - "<?xml version='1.0' encoding='utf-8' ?>\n" + - "<container version='1.0'>" + - " <search/>" + - " <nodes jvm-options='xyz' jvmargs='abc' count='3'/>" + - "</container>"; - int numberOfHosts = 3; - VespaModelTester tester = new VespaModelTester(); - tester.addHosts(numberOfHosts); - try { - tester.createModel(services, true); - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - assertEquals("You have specified both deprecated jvm-options='xyz' and deprecated jvmargs='abc'. " + - "'jvm-options' and 'jvmargs' are deprecated and will be removed in Vespa 8. " + - "Please merge 'jvmargs' into 'options' or 'gc-options' in 'jvm' element. " + - "See https://docs.vespa.ai/en/reference/services-container.html#jvm", - e.getMessage()); - } - } - - @Test public void testUsingHostaliasWithProvisioner() { String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + diff --git a/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java b/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java index 4b0d4ee7a9a..6a8645195a5 100644 --- a/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java @@ -16,12 +16,12 @@ public class SDFieldTestCase extends AbstractSchemaTestCase { @Test public void testIdSettingConflict() { SDDocumentType doc = new SDDocumentType("testdoc"); - doc.addField("one", DataType.STRING, false, 60); + doc.addField("one", DataType.STRING, 60); - doc.addField("two", DataType.STRING, false, 61); + doc.addField("two", DataType.STRING, 61); try { - doc.addField("three", DataType.STRING, false, 60); + doc.addField("three", DataType.STRING, 60); fail("Allowed to set duplicate id"); } catch (IllegalArgumentException e) { @@ -33,7 +33,7 @@ public class SDFieldTestCase extends AbstractSchemaTestCase { public void testSettingReservedId() { SDDocumentType doc = new SDDocumentType("testdoc"); try { - doc.addField("one", DataType.STRING, false, 127); + doc.addField("one", DataType.STRING, 127); fail("Allowed to set reserved id"); } catch (IllegalArgumentException e) { @@ -41,7 +41,7 @@ public class SDFieldTestCase extends AbstractSchemaTestCase { } try { - doc.addField("one", DataType.STRING, false, 100); + doc.addField("one", DataType.STRING, 100); fail("Allowed to set reserved id"); } catch (IllegalArgumentException e) { @@ -49,7 +49,7 @@ public class SDFieldTestCase extends AbstractSchemaTestCase { } try { - doc.addField("one", DataType.STRING, false, -1); + doc.addField("one", DataType.STRING, -1); fail("Allowed to set reserved id"); } catch (IllegalArgumentException e) { diff --git a/config-model/src/test/java/com/yahoo/schema/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/schema/AttributeSettingsTestCase.java index bbe63f95787..885cc385dfa 100644 --- a/config-model/src/test/java/com/yahoo/schema/AttributeSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/AttributeSettingsTestCase.java @@ -37,7 +37,6 @@ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { Attribute a1 = f1.getAttributes().get(f1.getName()); assertEquals(Attribute.Type.LONG, a1.getType()); assertEquals(Attribute.CollectionType.SINGLE, a1.getCollectionType()); - assertTrue(a1.isHuge()); assertFalse(a1.isFastSearch()); assertFalse(a1.isFastAccess()); assertFalse(a1.isRemoveIfZero()); @@ -48,7 +47,6 @@ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { Attribute a2 = f2.getAttributes().get(f2.getName()); assertEquals(Attribute.Type.LONG, a2.getType()); assertEquals(Attribute.CollectionType.SINGLE, a2.getCollectionType()); - assertFalse(a2.isHuge()); assertTrue(a2.isFastSearch()); assertFalse(a2.isFastAccess()); assertFalse(a2.isRemoveIfZero()); @@ -61,7 +59,6 @@ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { Attribute a3 = f3.getAttributes().get(f3.getName()); assertEquals(Attribute.Type.LONG, a3.getType()); assertEquals(Attribute.CollectionType.SINGLE, a3.getCollectionType()); - assertFalse(a3.isHuge()); assertFalse(a3.isFastSearch()); assertFalse(a3.isFastAccess()); assertFalse(a3.isRemoveIfZero()); @@ -306,7 +303,6 @@ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { single.setEnableBitVectors(true); single.setEnableOnlyBitVector(true); single.setFastSearch(true); - single.setHuge(true); single.setPaged(true); single.setFastAccess(true); single.setPosition(true); @@ -329,7 +325,6 @@ public class AttributeSettingsTestCase extends AbstractSchemaTestCase { assertTrue(array.isEnabledBitVectors()); assertTrue(array.isEnabledOnlyBitVector()); assertTrue(array.isFastSearch()); - assertTrue(array.isHuge()); assertTrue(array.isPaged()); assertTrue(array.isFastAccess()); assertTrue(array.isPosition()); diff --git a/config-model/src/test/java/com/yahoo/schema/RankingExpressionInliningTestCase.java b/config-model/src/test/java/com/yahoo/schema/RankingExpressionInliningTestCase.java index 0695e20d780..79c803e007c 100644 --- a/config-model/src/test/java/com/yahoo/schema/RankingExpressionInliningTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/RankingExpressionInliningTestCase.java @@ -19,6 +19,7 @@ import java.util.logging.Level; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author bratseth @@ -191,36 +192,38 @@ public class RankingExpressionInliningTestCase extends AbstractSchemaTestCase { } @Test - public void testFunctionInliningWithReplacement() throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - MockDeployLogger deployLogger = new MockDeployLogger(); - ApplicationBuilder builder = new ApplicationBuilder(MockApplicationPackage.createEmpty(), - new MockFileRegistry(), - deployLogger, - new TestProperties(), - rankProfileRegistry, - new QueryProfileRegistry()); - builder.addSchema( - "search test {\n" + - " document test { }\n" + - " rank-profile test {\n" + - " first-phase {\n" + - " expression: foo\n" + - " }\n" + - " function foo(x) {\n" + - " expression: x + x\n" + - " }\n" + - " function inline foo() {\n" + // replaces previous "foo" during parsing - " expression: foo(2)\n" + - " }\n" + - " }\n" + - "}\n"); - builder.build(true); - Schema s = builder.getSchema(); - RankProfile test = rankProfileRegistry.get(s, "test").compile(new QueryProfileRegistry(), new ImportedMlModels()); - assertEquals("foo(2)", test.getFirstPhaseRanking().getRoot().toString()); - assertTrue("Does not contain expected warning", - deployLogger.contains("Function 'foo' is defined twice in rank profile 'test'")); + public void testFunctionRedefinitionIsIllegal() throws ParseException { + try { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + MockDeployLogger deployLogger = new MockDeployLogger(); + ApplicationBuilder builder = new ApplicationBuilder(MockApplicationPackage.createEmpty(), + new MockFileRegistry(), + deployLogger, + new TestProperties(), + rankProfileRegistry, + new QueryProfileRegistry()); + builder.addSchema( + "search test {\n" + + " document test { }\n" + + " rank-profile test {\n" + + " first-phase {\n" + + " expression: foo\n" + + " }\n" + + " function foo(x) {\n" + + " expression: x + x\n" + + " }\n" + + " function inline foo() {\n" + + " expression: foo(2)\n" + + " }\n" + + " }\n" + + "}\n"); + builder.build(true); + fail("Expected failure"); + } + catch (IllegalArgumentException e) { + // success + assertEquals("Function 'foo' is defined twice in rank profile 'test'", e.getMessage()); + } } /** diff --git a/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java b/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java index e93dd0e0a8f..929c407215a 100644 --- a/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java @@ -18,7 +18,6 @@ import org.junit.Test; import java.io.IOException; import java.util.Iterator; -import static com.google.common.collect.testing.Helpers.assertEmpty; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -68,7 +67,6 @@ public class SchemaImporterTestCase extends AbstractSchemaTestCase { assertEquals(0, field.getAttributes().size()); assertNull(field.getStemming()); assertTrue(field.getNormalizing().doRemoveAccents()); - assertTrue(field.isHeader()); // Second field field = (SDField) document.getField("description"); @@ -108,13 +106,11 @@ public class SchemaImporterTestCase extends AbstractSchemaTestCase { field = schema.getConcreteField("categories"); assertEquals("{ input categories_src | lowercase | normalize | tokenize normalize stem:\"BEST\" | index categories; }", field.getIndexingScript().toString()); - assertTrue(field.isHeader()); // Eight field field= schema.getConcreteField("categoriesagain"); assertEquals("{ input categoriesagain_src | lowercase | normalize | tokenize normalize stem:\"BEST\" | index categoriesagain; }", field.getIndexingScript().toString()); - assertTrue(field.isHeader()); // Ninth field field= schema.getConcreteField("exactemento"); @@ -129,7 +125,6 @@ public class SchemaImporterTestCase extends AbstractSchemaTestCase { assertEquals("category_arr", attribute.getName()); assertEquals(Attribute.Type.STRING, attribute.getType()); assertEquals(Attribute.CollectionType.ARRAY, attribute.getCollectionType()); - assertTrue(field.isHeader()); // Eleventh field field = schema.getConcreteField("measurement_arr"); @@ -142,7 +137,7 @@ public class SchemaImporterTestCase extends AbstractSchemaTestCase { // Rank Profiles RankProfile profile = rankProfileRegistry.get(schema, "default"); assertNotNull(profile); - assertEmpty(profile.inheritedNames()); + assertTrue(profile.inheritedNames().isEmpty()); assertNull(profile.getDeclaredRankSetting("measurement", RankProfile.RankSetting.Type.RANKTYPE)); assertEquals(RankType.EMPTY, profile.getRankSetting("measurement", RankProfile.RankSetting.Type.RANKTYPE).getValue()); diff --git a/config-model/src/test/java/com/yahoo/schema/derived/NearestNeighborTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/NearestNeighborTestCase.java index baee7bec2a2..2f3ad7cfc69 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/NearestNeighborTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/NearestNeighborTestCase.java @@ -4,7 +4,6 @@ package com.yahoo.schema.derived; import com.yahoo.component.ComponentId; import com.yahoo.search.Query; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; -import com.yahoo.search.query.profile.config.QueryProfileConfigurer; import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; @@ -22,8 +21,7 @@ public class NearestNeighborTestCase extends AbstractExportingTestCase { ComponentId.resetGlobalCountersForTests(); DerivedConfiguration c = assertCorrectDeriving("nearestneighbor"); - CompiledQueryProfileRegistry queryProfiles = - QueryProfileConfigurer.createFromConfig(new QueryProfiles(c.getQueryProfiles(), (level, message) -> {}).getConfig()).compile(); + CompiledQueryProfileRegistry queryProfiles = CompiledQueryProfileRegistry.fromConfig(new QueryProfiles(c.getQueryProfiles(), (level, message) -> {}).getConfig()); Query q = new Query("?ranking.features.query(q_vec)=[1,2,3,4,5,6]", // length is 6, not 5 queryProfiles.getComponent("default")); fail("This should fail when q_vec is parsed as a tensor"); diff --git a/config-model/src/test/java/com/yahoo/schema/derived/NeuralNetTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/NeuralNetTestCase.java index 6e584099331..56248c02a51 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/NeuralNetTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/NeuralNetTestCase.java @@ -4,7 +4,6 @@ package com.yahoo.schema.derived; import com.yahoo.search.Query; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; -import com.yahoo.search.query.profile.config.QueryProfileConfigurer; import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; @@ -22,8 +21,7 @@ public class NeuralNetTestCase extends AbstractExportingTestCase { ComponentId.resetGlobalCountersForTests(); DerivedConfiguration c = assertCorrectDeriving("neuralnet"); // Verify that query profiles end up correct when passed through the same intermediate forms as a full system - CompiledQueryProfileRegistry queryProfiles = - QueryProfileConfigurer.createFromConfig(new QueryProfiles(c.getQueryProfiles(), (level, message) -> {}).getConfig()).compile(); + CompiledQueryProfileRegistry queryProfiles = CompiledQueryProfileRegistry.fromConfig(new QueryProfiles(c.getQueryProfiles(), (level, message) -> {}).getConfig()); assertNeuralNetQuery(c, queryProfiles.getComponent("default")); } diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java index 341d3ef7d43..73713319cfe 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java @@ -28,7 +28,6 @@ public class SummaryTestCase extends AbstractSchemaTestCase { public void deriveRawAsBase64() throws ParseException { String sd = joinLines( "schema s {", - " raw-as-base64-in-summary", " document s {", " field raw_field type raw {", " indexing: summary", @@ -44,6 +43,7 @@ public class SummaryTestCase extends AbstractSchemaTestCase { public void deriveRawAsLegacy() throws ParseException { String sd = joinLines( "schema s {", + " raw-as-base64-in-summary: false", " document s {", " field raw_field type raw {", " indexing: summary", diff --git a/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java b/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java index d4e4f1dbb88..77d1b55019d 100644 --- a/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java @@ -175,7 +175,6 @@ public class SchemaParserTestCase { checkFileParses("src/test/derived/map_attribute/test.sd"); checkFileParses("src/test/derived/map_of_struct_attribute/test.sd"); checkFileParses("src/test/derived/mlr/mlr.sd"); - checkFileParses("src/test/derived/multiplesummaries/multiplesummaries.sd"); checkFileParses("src/test/derived/music3/music3.sd"); checkFileParses("src/test/derived/music/music.sd"); checkFileParses("src/test/derived/namecollision/collision.sd"); diff --git a/config-model/src/test/java/com/yahoo/schema/processing/PositionTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/PositionTestCase.java index 6f0facf9541..e4f32fff41d 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/PositionTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/PositionTestCase.java @@ -113,8 +113,8 @@ public class PositionTestCase { PositionDataType.getZCurveFieldName(fieldName), (isArray ? DataType.getArray(PositionDataType.INSTANCE) : PositionDataType.INSTANCE), SummaryTransform.GEOPOS); - assertNull(schema.getSummaryField(PositionDataType.getDistanceSummaryFieldName(fieldName))); - assertNull(schema.getSummaryField(PositionDataType.getPositionSummaryFieldName(fieldName))); + assertNull(schema.getSummaryField(AdjustPositionSummaryFields.getDistanceSummaryFieldName(fieldName))); + assertNull(schema.getSummaryField(AdjustPositionSummaryFields.getPositionSummaryFieldName(fieldName))); } private static void assertSummaryField(Schema schema, String fieldName, String sourceName, DataType dataType, diff --git a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxModelTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxModelTestCase.java index 713e11fd608..1280895bfc0 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxModelTestCase.java @@ -139,7 +139,7 @@ public class RankingExpressionWithOnnxModelTestCase { assertEquals("vespa.rank.firstphase", config.rankprofile(2).fef().property(2).name()); assertEquals("rankingExpression(firstphase)", config.rankprofile(2).fef().property(2).value()); assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(2).fef().property(3).name()); - assertEquals("onnxModel(my_model).out{d0:1}", config.rankprofile(2).fef().property(3).value()); + assertEquals("onnx(my_model).out{d0:1}", config.rankprofile(2).fef().property(3).value()); assertEquals("test_generated_model_config", config.rankprofile(3).name()); assertEquals("rankingExpression(my_function).rankingScript", config.rankprofile(3).fef().property(0).name()); @@ -149,25 +149,25 @@ public class RankingExpressionWithOnnxModelTestCase { assertEquals("vespa.rank.firstphase", config.rankprofile(3).fef().property(8).name()); assertEquals("rankingExpression(firstphase)", config.rankprofile(3).fef().property(8).value()); assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(3).fef().property(9).name()); - assertEquals("onnxModel(files_model_onnx).path_to_output_1{d0:1}", config.rankprofile(3).fef().property(9).value()); + assertEquals("onnx(files_model_onnx).path_to_output_1{d0:1}", config.rankprofile(3).fef().property(9).value()); assertEquals("test_summary_features", config.rankprofile(4).name()); assertEquals("rankingExpression(another_function).rankingScript", config.rankprofile(4).fef().property(0).name()); assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(4).fef().property(3).name()); assertEquals("1", config.rankprofile(4).fef().property(3).value()); assertEquals("vespa.summary.feature", config.rankprofile(4).fef().property(4).name()); - assertEquals("onnxModel(files_summary_model_onnx).path_to_output_2", config.rankprofile(4).fef().property(4).value()); + assertEquals("onnx(another_model).out", config.rankprofile(4).fef().property(4).value()); assertEquals("vespa.summary.feature", config.rankprofile(4).fef().property(5).name()); - assertEquals("onnxModel(another_model).out", config.rankprofile(4).fef().property(5).value()); + assertEquals("onnx(files_summary_model_onnx).path_to_output_2", config.rankprofile(4).fef().property(5).value()); assertEquals("test_dynamic_model", config.rankprofile(5).name()); assertEquals("rankingExpression(my_function).rankingScript", config.rankprofile(5).fef().property(0).name()); assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(5).fef().property(3).name()); - assertEquals("onnxModel(dynamic_model).my_output{d0:0, d1:1}", config.rankprofile(5).fef().property(3).value()); + assertEquals("onnx(dynamic_model).my_output{d0:0, d1:1}", config.rankprofile(5).fef().property(3).value()); assertEquals("test_dynamic_model_2", config.rankprofile(6).name()); assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(6).fef().property(5).name()); - assertEquals("onnxModel(dynamic_model).my_output{d0:0, d1:2}", config.rankprofile(6).fef().property(5).value()); + assertEquals("onnx(dynamic_model).my_output{d0:0, d1:2}", config.rankprofile(6).fef().property(5).value()); assertEquals("test_dynamic_model_with_transformer_tokens", config.rankprofile(7).name()); assertEquals("rankingExpression(my_function).rankingScript", config.rankprofile(7).fef().property(1).name()); @@ -176,7 +176,7 @@ public class RankingExpressionWithOnnxModelTestCase { assertEquals("test_unbound_model", config.rankprofile(8).name()); assertEquals("rankingExpression(my_function).rankingScript", config.rankprofile(8).fef().property(0).name()); assertEquals("rankingExpression(firstphase).rankingScript", config.rankprofile(8).fef().property(3).name()); - assertEquals("onnxModel(unbound_model).my_output{d0:0, d1:1}", config.rankprofile(8).fef().property(3).value()); + assertEquals("onnx(unbound_model).my_output{d0:0, d1:1}", config.rankprofile(8).fef().property(3).value()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java index 4d8dd3dacfd..ea075dc1129 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java @@ -34,9 +34,9 @@ public class HostResourceTest { } @Test - public void host_witrh_membership() { - HostResource host = hostResourceWithMemberships(ClusterMembership.from(clusterSpec(container, "jdisc"), 0)); - assertClusterMembership(host, container, "jdisc"); + public void host_with_membership() { + HostResource host = hostResourceWithMemberships(ClusterMembership.from(clusterSpec(container, "container"), 0)); + assertClusterMembership(host, container, "container"); } private void assertClusterMembership(HostResource host, ClusterSpec.Type type, String id) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java index e7dc86ccfea..eee4d29cc6e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java @@ -28,7 +28,7 @@ import java.util.stream.IntStream; import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; -import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER; +import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -132,13 +132,13 @@ public class DedicatedAdminV4Test { // 4 slobroks, 2 per cluster where possible assertHostContainsServices(model, "hosts/myhost0", "slobrok", "logd", "logserver", - METRICS_PROXY_CONTAINER.serviceName, QRSERVER.serviceName); + METRICS_PROXY_CONTAINER.serviceName, CONTAINER.serviceName); assertHostContainsServices(model, "hosts/myhost1", "slobrok", "logd", - METRICS_PROXY_CONTAINER.serviceName, QRSERVER.serviceName); + METRICS_PROXY_CONTAINER.serviceName, CONTAINER.serviceName); assertHostContainsServices(model, "hosts/myhost2", "slobrok", "logd", - METRICS_PROXY_CONTAINER.serviceName, QRSERVER.serviceName); + METRICS_PROXY_CONTAINER.serviceName, CONTAINER.serviceName); assertHostContainsServices(model, "hosts/myhost3", "slobrok", "logd", - METRICS_PROXY_CONTAINER.serviceName, QRSERVER.serviceName); + METRICS_PROXY_CONTAINER.serviceName, CONTAINER.serviceName); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java index c3aa387da54..8d6f05f40e7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java @@ -116,14 +116,6 @@ public class AttributeChangeValidatorTest { } @Test - public void changing_huge_require_restart() throws Exception { - new Fixture("field f1 type string { indexing: attribute }", - "field f1 type string { indexing: attribute \n attribute: huge }"). - assertValidation(newRestartAction(ClusterSpec.Id.from("test"), - "Field 'f1' changed: add attribute 'huge'")); - } - - @Test public void changing_dense_posting_list_threshold_require_restart() throws Exception { new Fixture( "field f1 type predicate { indexing: attribute \n index { arity: 8 \n dense-posting-list-threshold: 0.2 } }", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/UserConfigBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/UserConfigBuilderTest.java index 114038b884e..55c10a28a01 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/UserConfigBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/UserConfigBuilderTest.java @@ -6,7 +6,6 @@ import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.config.model.deploy.ConfigDefinitionStore; import com.yahoo.config.model.producer.UserConfigRepo; -import com.yahoo.test.ArraytypesConfig; import com.yahoo.test.SimpletypesConfig; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayload; @@ -15,7 +14,6 @@ import com.yahoo.vespa.configdefinition.SpecialtokensConfig; import org.junit.Test; import org.w3c.dom.Document; import org.w3c.dom.Element; - import java.io.Reader; import java.io.StringReader; import java.util.Optional; @@ -53,32 +51,21 @@ public class UserConfigBuilderTest { } @Test - public void require_that_arrays_config_is_resolved() { - Element configRoot = getDocument("<config name=\"test.arraytypes\">" + - " <intarr operation=\"append\">13</intarr>" + - " <intarr operation=\"append\">10</intarr>" + - " <intarr operation=\"append\">1337</intarr>" + - "</config>"); - UserConfigRepo map = UserConfigBuilder.build(configRoot, configDefinitionStore, new BaseDeployLogger()); - assertFalse(map.isEmpty()); - ConfigDefinitionKey key = new ConfigDefinitionKey("arraytypes", "test"); - assertNotNull(map.get(key)); - ArraytypesConfig config = createConfig(ArraytypesConfig.class, map.get(key)); - assertEquals(3, config.intarr().size()); - assertEquals(13, config.intarr(0)); - assertEquals(10, config.intarr(1)); - assertEquals(1337, config.intarr(2)); - } - - @Test public void require_that_arrays_of_structs_are_resolved() { Element configRoot = getDocument( " <config name='vespa.configdefinition.specialtokens'>" + - " <tokenlist operation='append'>" + - " <name>default</name>" + - " <tokens operation='append'>" + - " <token>dvd+-r</token>" + - " </tokens>" + + " <tokenlist>" + + " <item>" + + " <name>default</name>" + + " <tokens>" + + " <item>" + + " <token>dvd+-r</token>" + + " </item>" + + " <item>" + + " <token>c++</token>" + + " </item>" + + " </tokens>" + + " </item>" + " </tokenlist>" + " </config>" ); @@ -92,9 +79,10 @@ public class UserConfigBuilderTest { assertNotNull(map.get(key)); SpecialtokensConfig config = createConfig(SpecialtokensConfig.class, map.get(key)); assertEquals(1, config.tokenlist().size()); - assertEquals("default", config.tokenlist().get(0).name()); - assertEquals(1, config.tokenlist().get(0).tokens().size()); - assertEquals("dvd+-r", config.tokenlist().get(0).tokens().get(0).token()); + SpecialtokensConfig.Tokenlist tokenlist = config.tokenlist().get(0); + assertEquals("default", tokenlist.name()); + assertEquals(2, tokenlist.tokens().size()); + assertEquals("dvd+-r", tokenlist.tokens().get(0).token()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 3468d81b748..a3360278505 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -166,7 +166,7 @@ public class ContentBuilderTest extends DomBuilderTest { assertEquals("clu/storage/0", c.getRootGroup().getNodes().get(0).getConfigId()); // Due to reuse. assertEquals(1, c.getRoot().hostSystem().getHosts().size()); HostResource h = c.getRoot().hostSystem().getHost("mockhost"); - String [] expectedServices = {"configserver", "logserver", "logd", "container-clustercontroller", "metricsproxy-container", "slobrok", "configproxy","config-sentinel", "qrserver", "storagenode", "searchnode", "distributor", "transactionlogserver"}; + String [] expectedServices = {"configserver", "logserver", "logd", "container-clustercontroller", "metricsproxy-container", "slobrok", "configproxy","config-sentinel", "container", "storagenode", "searchnode", "distributor", "transactionlogserver"}; assertServices(h, expectedServices); assertEquals("clu/storage/0", h.getService("storagenode").getConfigId()); assertEquals("clu/search/cluster.clu/0", h.getService("searchnode").getConfigId()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java index a20ce425ac0..c0db0ac9e09 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilderTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.config.ConfigurationRuntimeException; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.codegen.DefParser; import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.slime.JsonFormat; @@ -14,13 +13,11 @@ import com.yahoo.vespa.config.ConfigPayloadBuilder; import org.junit.Test; import org.w3c.dom.Document; import org.w3c.dom.Element; - import java.io.ByteArrayOutputStream; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.Reader; import java.io.StringReader; -import java.util.logging.Level; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -33,10 +30,6 @@ import static org.junit.Assert.fail; */ public class DomConfigPayloadBuilderTest { - private static final DeployLogger logger = (level, message) -> { - if (level.intValue() > Level.INFO.intValue()) System.err.println(message); - }; - @Test public void testFunctionTest_DefaultValues() throws FileNotFoundException { Element configRoot = getDocument(new FileReader("src/test/cfg/admin/userconfigs/functiontest-defaultvalues.xml")); @@ -122,17 +115,6 @@ public class DomConfigPayloadBuilderTest { } @Test - public void append_to_leaf_array() { - // Simulate user config from vespa-services.xml - Reader xmlConfig = new StringReader("<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + - "<config name=\"a.function-test\">" + - " <intarr operation=\"append\">1</intarr>" + - " <intarr operation=\"append\">2</intarr>" + - "</config> "); - assertPayload("{\"intarr\":[\"1\",\"2\"]}", getDocument(xmlConfig)); - } - - @Test public void camel_case_via_dashes() { Reader xmlConfig = new StringReader("<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + "<config name=\"test.function-test\">" + @@ -146,7 +128,7 @@ public class DomConfigPayloadBuilderTest { public void testFailWrongTagName() { Element configRoot = getDocument(new StringReader("<configs name=\"foo\"/>")); try { - new DomConfigPayloadBuilder(null, logger).build(configRoot); + new DomConfigPayloadBuilder(null).build(configRoot); fail("Expected exception for wrong tag name."); } catch (ConfigurationRuntimeException e) { assertEquals("The root element must be 'config', but was 'configs'.", e.getMessage()); @@ -158,7 +140,7 @@ public class DomConfigPayloadBuilderTest { public void testFailNoNameAttribute() { Element configRoot = getDocument(new StringReader("<config/>")); try { - new DomConfigPayloadBuilder(null, logger).build(configRoot); + new DomConfigPayloadBuilder(null).build(configRoot); fail("Expected exception for mismatch between def-name and xml name attribute."); } catch (ConfigurationRuntimeException e) { assertEquals("The 'config' element must have a 'name' attribute that matches the name of the config definition.", e.getMessage()); @@ -240,7 +222,7 @@ public class DomConfigPayloadBuilderTest { "<config name=\"test.arraytypes\" version=\"1\">" + " <item>13</item>" + "</config>"); - new DomConfigPayloadBuilder(null, logger).build(configRoot); + new DomConfigPayloadBuilder(null).build(configRoot); } @Test(expected=ConfigurationRuntimeException.class) @@ -252,13 +234,12 @@ public class DomConfigPayloadBuilderTest { DefParser defParser = new DefParser("simpletypes", new FileReader("src/test/resources/configdefinitions/test.simpletypes.def")); ConfigDefinition def = ConfigDefinitionBuilder.createConfigDefinition(defParser.getTree()); - ConfigPayloadBuilder unused = new DomConfigPayloadBuilder(def, logger).build(configRoot); + ConfigPayloadBuilder unused = new DomConfigPayloadBuilder(def).build(configRoot); } - private void assertPayload(String expected, Element configRoot) { ConfigPayload payload = ConfigPayload.fromBuilder( - new DomConfigPayloadBuilder(null, logger).build(configRoot)); + new DomConfigPayloadBuilder(null).build(configRoot)); try { ByteArrayOutputStream a = new ByteArrayOutputStream(); new JsonFormat(true).encode(a, payload.getSlime()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilderTest.java index 7af81b3b9b1..1a5a7bbe544 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilderTest.java @@ -83,8 +83,8 @@ public class VespaDomBuilderTest { @Test public void testGetElement() { - Element e = Xml.getElement(new StringReader("<searchchain><foo>sdf</foo></searchchain>")); - assertEquals(e.getTagName(), "searchchain"); + Element e = Xml.getElement(new StringReader("<chain><foo>sdf</foo></chain>")); + assertEquals(e.getTagName(), "chain"); assertEquals(XML.getChild(e, "foo").getTagName(), "foo"); assertEquals(XML.getValue(XML.getChild(e, "foo")), "sdf"); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSchemaChainsBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSchemaChainsBuilderTest.java index 6006b52cf02..9fb7b58a43c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSchemaChainsBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSchemaChainsBuilderTest.java @@ -43,7 +43,7 @@ public class DomSchemaChainsBuilderTest extends DomBuilderTest { private SearchChains searchChains; private static final Element element = parse( - "<searchchains>", + "<search>", " <searcher id='searcher:1'/>", " <provider id='provider:1' inherits='parentChain1 parentChain2' excludes='ExcludedSearcher1 ExcludedSearcher2'", @@ -59,15 +59,15 @@ public class DomSchemaChainsBuilderTest extends DomBuilderTest { " </provider>", - " <searchchain id='default'>", + " <chain id='default'>", " <federation id='federationSearcher'>", " <source id='mysource'>", " <federationoptions optional='false' />", " </source>", " </federation>", - " </searchchain>", + " </chain>", - "</searchchains>"); + "</search>"); @Before @@ -78,17 +78,17 @@ public class DomSchemaChainsBuilderTest extends DomBuilderTest { @Test public void referToFederationAsSearcher() { final Element element = parse( - "<searchchains>", + "<search>", " <federation id='federationSearcher'>", " <source id='mysource'>", " <federationoptions optional='false' />", " </source>", " </federation>", - " <searchchain id='default'>", + " <chain id='default'>", " <searcher id='federationSearcher'/>", - " </searchchain>", - "</searchchains>"); + " </chain>", + "</search>"); try { MockRoot root = new MockRoot(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/http/StrictFilteringTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/http/StrictFilteringTest.java index f703f39295f..ad81c9a5a4d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/http/StrictFilteringTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/http/StrictFilteringTest.java @@ -8,7 +8,7 @@ import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; import org.junit.Test; import org.w3c.dom.Element; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; /** * @author bjorncs @@ -16,11 +16,11 @@ import static org.junit.Assert.assertTrue; public class StrictFilteringTest extends DomBuilderTest { @Test - public void default_request_and_response_filters_in_services_xml_are_listen_in_server_config() { + public void strict_filtering_enabled_if_specified_in_services() { Element xml = parse( "<container version='1.0'>", " <http>", - " <filtering strict-mode=\"true\">", + " <filtering strict-mode='true'>", " <request-chain id='request-chain-with-binding'>", " <filter id='my-filter' class='MyFilter'/>", " <binding>http://*/my-chain-binding</binding>", @@ -29,9 +29,42 @@ public class StrictFilteringTest extends DomBuilderTest { " <server id='server1' port='8000' />", " </http>", "</container>"); - buildContainerCluster(xml); + assertStrictFiltering(true, xml); + } + + @Test + public void strict_filtering_enabled_by_default_if_filter_present() { + Element xml = parse( + "<container version='1.0'>", + " <http>", + " <filtering>", + " <request-chain id='request-chain'>", + " <filter id='my-filter' class='MyFilter'/>", + " </request-chain>", + " </filtering>", + " <server id='server1' port='8000' />", + " </http>", + "</container>"); + assertStrictFiltering(true, xml); + } + + @Test + public void strict_filtering_disabled_if_no_filter() { + Element xml = parse( + "<container version='1.0'>", + " <http>", + " <filtering>", + " </filtering>", + " <server id='server1' port='8000' />", + " </http>", + "</container>"); + assertStrictFiltering(false, xml); + } + + private void assertStrictFiltering(boolean expected, Element services) { + buildContainerCluster(services); ServerConfig config = root.getConfig(ServerConfig.class, "container/http/jdisc-jetty/server1"); - assertTrue(config.strictFiltering()); + assertEquals(expected, config.strictFiltering()); } private void buildContainerCluster(Element containerElem) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java index d74f271f6ef..20345615753 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java @@ -21,15 +21,10 @@ import static org.junit.Assert.assertEquals; */ public class SchemaChainsTest extends SchemaChainsTestBase { - private ChainsConfig chainsConfig; private ClusterConfig clusterConfig; @Before public void subscribe() { - ChainsConfig.Builder chainsBuilder = new ChainsConfig.Builder(); - chainsBuilder = (ChainsConfig.Builder)root.getConfig(chainsBuilder, "searchchains"); - chainsConfig = new ChainsConfig(chainsBuilder); - ClusterConfig.Builder clusterBuilder = new ClusterConfig.Builder(); clusterBuilder = (ClusterConfig.Builder)root.getConfig(clusterBuilder, "searchchains/chain/cluster2/component/" + ClusterSearcher.class.getName()); clusterConfig = new ClusterConfig(clusterBuilder); @@ -39,7 +34,7 @@ public class SchemaChainsTest extends SchemaChainsTestBase { @Override Element servicesXml() { return parse( - "<searchchains>", + "<search>", " <searcher id='searcher:1' classId='classId1' />", " <provider id='provider:1' inherits='parentChain1 parentChain2' excludes='ExcludedSearcher1 ExcludedSearcher2'", @@ -67,20 +62,7 @@ public class SchemaChainsTest extends SchemaChainsTestBase { " <queryType>PROGRAMMATIC</queryType>", " </config>", " </provider>", - - " <searchchain id='default:99'>", - " <federation id='federation:98' provides='provide_federation' before='p1 p2' after='s1 s2'>", - " <source id='source:1'>", - " <federationoptions optional='false' />", - " </source>", - " </federation>", - " </searchchain>", - - " <searchchain id='parentChain1' />", - " <searchchain id='parentChain2' />", - " <searchchain id='parentChain3' />", - " <searchchain id='parentChain4' />", - "</searchchains>"); + "</search>"); } private SearchChains getSearchChains() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java index e73eb905566..03641e95f84 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java @@ -34,7 +34,7 @@ import static org.junit.Assert.assertTrue; */ public class QueryProfilesTestCase { - private final static String root="src/test/java/com/yahoo/vespa/model/container/search/test/"; + private final static String root = "src/test/java/com/yahoo/vespa/model/container/search/test/"; @Test public void testVariantReference() { @@ -212,10 +212,6 @@ public class QueryProfilesTestCase { protected void assertConfig(String correctFileName, QueryProfileRegistry check) throws IOException { assertSerializedConfigFileEquals(root + "/" + correctFileName, com.yahoo.text.StringUtilities.implodeMultiline(com.yahoo.config.ConfigInstance.serialize(new QueryProfiles(check, new SilentDeployLogger()).getConfig()))); - - // Also assert that the correct config config can actually be read as a config source - QueryProfileConfigurer configurer = new QueryProfileConfigurer("file:" + root + "empty.cfg"); - configurer.shutdown(); } private static class SilentDeployLogger implements DeployLogger { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java index f6a5bc14ab5..d676dc29c94 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java @@ -62,31 +62,6 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { assertEquals("Wrong domain.", "my-tenant-domain", accessControl.domain); } - @Test - public void read_is_disabled_and_write_is_enabled_by_default() { - Http http = createModelAndGetHttp( - " <http>", - " <filtering>", - " <access-control domain='my-tenant-domain'/>", - " </filtering>", - " </http>"); - - assertFalse("Wrong default value for read.", http.getAccessControl().get().readEnabled); - assertTrue("Wrong default value for write.", http.getAccessControl().get().writeEnabled); - } - - @Test - public void read_and_write_can_be_overridden() { - Http http = createModelAndGetHttp( - " <http>", - " <filtering>", - " <access-control domain='my-tenant-domain' read='true' write='false'/>", - " </filtering>", - " </http>"); - - assertTrue("Given read value not honoured.", http.getAccessControl().get().readEnabled); - assertFalse("Given write value not honoured.", http.getAccessControl().get().writeEnabled); - } @Test public void access_control_excluded_filter_chain_has_all_bindings_from_excluded_handlers() { @@ -176,8 +151,6 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { Optional<AccessControl> maybeAccessControl = http.getAccessControl(); assertTrue(maybeAccessControl.isPresent()); AccessControl accessControl = maybeAccessControl.get(); - assertFalse(accessControl.writeEnabled); - assertFalse(accessControl.readEnabled); assertEquals(AccessControl.ClientAuthentication.need, accessControl.clientAuthentication); assertEquals("my-tenant-domain", accessControl.domain); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ApplicationBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ApplicationBuilderTest.java index 3490568770c..ca59e053a89 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ApplicationBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ApplicationBuilderTest.java @@ -16,7 +16,7 @@ import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.Test; import org.w3c.dom.Element; -import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER; +import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; import static com.yahoo.test.Matchers.hasItemWithMethod; import static com.yahoo.vespa.model.container.search.ContainerSearch.QUERY_PROFILE_REGISTRY_CLASS; import static org.hamcrest.MatcherAssert.assertThat; @@ -120,7 +120,7 @@ public class ApplicationBuilderTest extends ContainerModelBuilderTestBase { public void cluster_with_only_search_gets_qrserver_as_service_name() { createClusterWithOnlyDefaultChains(); ApplicationContainerCluster cluster = (ApplicationContainerCluster)root.getChildren().get("default"); - assertEquals(QRSERVER.serviceName, cluster.getContainers().get(0).getServiceName()); + assertEquals(CONTAINER.serviceName, cluster.getContainers().get(0).getServiceName()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 701878f98ab..d91dba0572f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -102,6 +102,7 @@ import static org.junit.Assert.fail; * @author gjoranv */ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { + @Rule public TemporaryFolder applicationFolder = new TemporaryFolder(); @@ -114,22 +115,6 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test - public void deprecated_jdisc_tag_is_allowed() { - Element clusterElem = DomBuilderTest.parse( - "<jdisc version='1.0'>", - nodesXml, - "</jdisc>" ); - TestLogger logger = new TestLogger(); - createModel(root, logger, clusterElem); - AbstractService container = (AbstractService)root.getProducer("jdisc/container.0"); - assertNotNull(container); - - assertFalse(logger.msgs.isEmpty()); - assertEquals(Level.WARNING, logger.msgs.get(0).getFirst()); - assertEquals("'jdisc' is deprecated as tag name. Use 'container' instead.", logger.msgs.get(0).getSecond()); - } - - @Test public void default_port_is_4080() { Element clusterElem = DomBuilderTest.parse( "<container version='1.0'>", @@ -290,10 +275,8 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { public void handler_bindings_are_included_in_discBindings_config() { createClusterWithJDiscHandler(); String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); - assertThat(discBindingsConfig, containsString("{discHandler}")); assertThat(discBindingsConfig, containsString(".serverBindings[0] \"http://*/binding0\"")); assertThat(discBindingsConfig, containsString(".serverBindings[1] \"http://*/binding1\"")); - assertThat(discBindingsConfig, containsString(".clientBindings[0] \"http://*/clientBinding\"")); } @Test @@ -308,7 +291,6 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { " <handler id='discHandler'>", " <binding>http://*/binding0</binding>", " <binding>http://*/binding1</binding>", - " <clientBinding>http://*/clientBinding</clientBinding>", " </handler>", "</container>"); @@ -334,35 +316,6 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test - public void clientProvider_bindings_are_included_in_discBindings_config() { - createModelWithClientProvider(); - String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString(); - assertThat(discBindingsConfig, containsString("{discClient}")); - assertThat(discBindingsConfig, containsString(".clientBindings[0] \"http://*/binding0\"")); - assertThat(discBindingsConfig, containsString(".clientBindings[1] \"http://*/binding1\"")); - assertThat(discBindingsConfig, containsString(".serverBindings[0] \"http://*/serverBinding\"")); - } - - @Test - public void clientProviders_are_included_in_components_config() { - createModelWithClientProvider(); - assertThat(componentsConfig().toString(), containsString(".id \"discClient\"")); - } - - private void createModelWithClientProvider() { - Element clusterElem = DomBuilderTest.parse( - "<container id='default' version='1.0'>" + - " <client id='discClient'>" + - " <binding>http://*/binding0</binding>" + - " <binding>http://*/binding1</binding>" + - " <serverBinding>http://*/serverBinding</serverBinding>" + - " </client>" + - "</container>" ); - - createModel(root, clusterElem); - } - - @Test public void serverProviders_are_included_in_components_config() { Element clusterElem = DomBuilderTest.parse( "<container id='default' version='1.0'>" + @@ -448,17 +401,11 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { " <handler id='myHandler'>", " <component id='injected' />", " </handler>", - " <client id='myClient'>", // remember, a client is also a request handler - " <component id='injected' />", - " </client>", "</container>"); createModel(root, clusterElem); Component<?,?> handler = getContainerComponent("default", "myHandler"); assertThat(handler.getInjectedComponentIds(), hasItem("injected@myHandler")); - - Component<?,?> client = getContainerComponent("default", "myClient"); - assertThat(client.getInjectedComponentIds(), hasItem("injected@myClient")); } @Test @@ -505,53 +452,6 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test - public void http_aliases_are_stored_on_cluster_and_on_service_properties() { - Element clusterElem = DomBuilderTest.parse( - "<container id='default' version='1.0'>", - " <aliases>", - " <service-alias>service1</service-alias>", - " <service-alias>service2</service-alias>", - " <endpoint-alias>foo1.bar1.com</endpoint-alias>", - " <endpoint-alias>foo2.bar2.com</endpoint-alias>", - " </aliases>", - " <nodes>", - " <node hostalias='host1' />", - " </nodes>", - "</container>"); - - createModel(root, clusterElem); - assertEquals(getContainerCluster("default").serviceAliases().get(0), "service1"); - assertEquals(getContainerCluster("default").endpointAliases().get(0), "foo1.bar1.com"); - assertEquals(getContainerCluster("default").serviceAliases().get(1), "service2"); - assertEquals(getContainerCluster("default").endpointAliases().get(1), "foo2.bar2.com"); - - assertEquals(getContainerCluster("default").getContainers().get(0).getServicePropertyString("servicealiases"), "service1,service2"); - assertEquals(getContainerCluster("default").getContainers().get(0).getServicePropertyString("endpointaliases"), "foo1.bar1.com,foo2.bar2.com"); - } - - @Test - public void http_aliases_are_only_honored_in_prod_environment() { - Element clusterElem = DomBuilderTest.parse( - "<container id='default' version='1.0'>", - " <aliases>", - " <service-alias>service1</service-alias>", - " <endpoint-alias>foo1.bar1.com</endpoint-alias>", - " </aliases>", - " <nodes>", - " <node hostalias='host1' />", - " </nodes>", - "</container>"); - - DeployState deployState = new DeployState.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east-1"))).build(); - createModel(root, deployState, null, clusterElem); - assertEquals(0, getContainerCluster("default").serviceAliases().size()); - assertEquals(0, getContainerCluster("default").endpointAliases().size()); - - assertNull(getContainerCluster("default").getContainers().get(0).getServicePropertyString("servicealiases")); - assertNull(getContainerCluster("default").getContainers().get(0).getServicePropertyString("endpointaliases")); - } - - @Test public void endpoints_are_added_to_containers() throws IOException, SAXException { final var servicesXml = joinLines("", "<container id='comics-search' version='1.0'>", @@ -1087,6 +987,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { createModel(root, deployState, null, DomBuilderTest.parse(containerService)); assertFalse(logger.msgs.isEmpty()); + System.out.println(logger.msgs); assertEquals(Level.WARNING, logger.msgs.get(0).getFirst()); assertEquals(Level.WARNING, logger.msgs.get(1).getFirst()); assertEquals("Element 'prod' contains attribute 'global-service-id' deprecated since major version 7. See https://cloud.vespa.ai/en/reference/routing#deprecated-syntax", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java index 49c1783c187..6cd0fa18c3d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java @@ -126,7 +126,7 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas @Test public void ssl_element_generates_connector_config_and_injects_provider_component() { Element clusterElem = DomBuilderTest.parse( - "<container id='default' version='1.0' jetty='true'>", + "<container id='default' version='1.0'>", " <http>", " <server port='9000' id='minimal'>", " <ssl>", @@ -198,7 +198,7 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas @Test public void verify_tht_ssl_provider_configuration_configures_correct_config() { Element clusterElem = DomBuilderTest.parse( - "<container id='default' version='1.0' jetty='true'>", + "<container id='default' version='1.0'>", " <http>", " <server port='9000' id='ssl'>", " <ssl-provider class='com.yahoo.CustomSslProvider' bundle='mybundle'/>", @@ -222,7 +222,7 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas @Test public void verify_that_container_factory_sees_same_config(){ Element clusterElem = DomBuilderTest.parse( - "<container id='default' version='1.0' jetty='true'>", + "<container id='default' version='1.0'>", " <http>", " <server port='9000' id='ssl'>", " <ssl>", @@ -243,7 +243,7 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas @Test public void verify_that_container_setup_additional_tls4443(){ Element clusterElem = DomBuilderTest.parse( - "<container id='default' version='1.0' jetty='true'>", + "<container id='default' version='1.0'>", " <http>", " <server port='8080' id='ssl'>", " <ssl>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java index b2c29b88e38..146e383078e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.model.container.ContainerCluster; import org.junit.Test; import org.w3c.dom.Element; import org.xml.sax.SAXException; - import java.io.IOException; import java.util.Arrays; import java.util.Collections; @@ -23,7 +22,6 @@ import java.util.List; import java.util.logging.Level; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -57,14 +55,6 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { assertEquals(45, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory()); assertEquals("-XX:SoftRefLRUPolicyMSPerMB=2500", model.getContainerClusters().values().iterator().next().getContainers().get(0).getJvmOptions()); } - @Test - public void detect_conflicting_jvmgcoptions_in_jvmargs() { - assertFalse(ContainerModelBuilder.incompatibleGCOptions("")); - assertFalse(ContainerModelBuilder.incompatibleGCOptions("UseG1GC")); - assertTrue(ContainerModelBuilder.incompatibleGCOptions("-XX:+UseG1GC")); - assertTrue(ContainerModelBuilder.incompatibleGCOptions("abc -XX:+UseParNewGC xyz")); - assertTrue(ContainerModelBuilder.incompatibleGCOptions("-XX:CMSInitiatingOccupancyFraction=19")); - } @Test public void honours_jvm_gc_options() { @@ -83,14 +73,9 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { } private static void verifyIgnoreJvmGCOptions(boolean isHosted) throws IOException, SAXException { - verifyIgnoreJvmGCOptionsIfJvmArgs("jvmargs", ContainerCluster.G1GC, isHosted); - verifyIgnoreJvmGCOptionsIfJvmArgs( "jvm-options", "-XX:+UseG1GC", isHosted); - - } - private static void verifyIgnoreJvmGCOptionsIfJvmArgs(String jvmOptionsName, String expectedGC, boolean isHosted) throws IOException, SAXException { String servicesXml = "<container version='1.0'>" + - " <nodes jvm-gc-options='-XX:+UseG1GC' " + jvmOptionsName + "='-XX:+UseParNewGC'>" + + " <nodes jvm-gc-options='-XX:+UseG1GC' jvm-options='-XX:+UseParNewGC'>" + " <node hostalias='mockhost'/>" + " </nodes>" + "</container>"; @@ -105,11 +90,11 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); model.getConfig(qrStartBuilder, "container/container.0"); QrStartConfig qrStartConfig = new QrStartConfig(qrStartBuilder); - assertEquals(expectedGC, qrStartConfig.jvm().gcopts()); + assertEquals("-XX:+UseG1GC", qrStartConfig.jvm().gcopts()); } @Test - public void ignores_jvmgcoptions_on_conflicting_jvmargs() throws IOException, SAXException { + public void ignores_jvmgcoptions_on_conflicting_jvmoptions() throws IOException, SAXException { verifyIgnoreJvmGCOptions(false); verifyIgnoreJvmGCOptions(true); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 10a2feaba5b..8c263e21a21 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -744,18 +744,18 @@ public class ContentClusterTest extends ContentBaseTest { MetricsmanagerConfig config = new MetricsmanagerConfig(builder); String expected = - "[vds.filestor.alldisks.allthreads.put.sum\n" + - "vds.filestor.alldisks.allthreads.get.sum\n" + - "vds.filestor.alldisks.allthreads.remove.sum\n" + - "vds.filestor.alldisks.allthreads.update.sum\n" + + "[vds.filestor.allthreads.put\n" + + "vds.filestor.allthreads.get\n" + + "vds.filestor.allthreads.remove\n" + + "vds.filestor.allthreads.update\n" + "vds.datastored.alldisks.docs\n" + "vds.datastored.alldisks.bytes\n" + - "vds.filestor.alldisks.queuesize\n" + - "vds.filestor.alldisks.averagequeuewait.sum\n" + + "vds.filestor.queuesize\n" + + "vds.filestor.averagequeuewait\n" + "vds.visitor.cv_queuewaittime\n" + "vds.visitor.allthreads.averagequeuewait\n" + "vds.visitor.allthreads.averagevisitorlifetime\n" + - "vds.visitor.allthreads.created.sum]"; + "vds.visitor.allthreads.created]"; String actual = getConsumer("log", config).addedmetrics().toString().replaceAll(", ", "\n"); assertEquals(expected, actual); assertEquals("[logdefault]", getConsumer("log", config).tags().toString()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java index d9dbd6eeca7..a39cbc60a89 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java @@ -45,14 +45,6 @@ public class IndexedTest extends ContentBaseTest { " <admin version='2.0'>" + " <adminserver hostalias='node0'/>" + " </admin>" + - " <config name='vespa.configdefinition.specialtokens'>" + - " <tokenlist operation='append'>" + - " <name>default</name>" + - " <tokens operation='append'>" + - " <token>dvd+-r</token>" + - " </tokens>" + - " </tokenlist>" + - " </config>" + " <container version='1.0'>" + " <search/>" + " <nodes>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java index 8c81ea28b4c..aa68dad83cd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java @@ -101,7 +101,6 @@ public class ClusterTest { assertEquals(8, config.numJrtTransportThreads()); assertEquals(100.0, config.minSearchCoverage(), DELTA); assertEquals(97.0, config.minActivedocsPercentage(), DELTA); - assertEquals(100.0, config.minGroupCoverage(), DELTA); assertEquals(0.9999, config.topKProbability(), DELTA); assertEquals(3, config.node().size()); assertEquals(0, config.node(0).key()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/generic/GenericServicesModelTest.java b/config-model/src/test/java/com/yahoo/vespa/model/generic/GenericServicesModelTest.java deleted file mode 100644 index 7b7fc83131e..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/generic/GenericServicesModelTest.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.generic; - -import com.yahoo.config.model.ConfigModelContext; -import com.yahoo.config.model.builder.xml.ConfigModelId; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.config.model.test.MockRoot; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.generic.service.ServiceCluster; -import org.junit.Test; -import org.xml.sax.SAXException; - -import java.io.IOException; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; - -/** - * @author Ulf Lilleengen - * @since 5.1 - */ -public class GenericServicesModelTest { - - @Test - public void test_generic_services_builder() { - GenericServicesBuilder builder = new GenericServicesBuilder(); - assertThat(builder.handlesElements().size(), is(1)); - assertThat(builder.handlesElements().get(0), is(ConfigModelId.fromName("service"))); - } - - @Test - public void test_generic_services_model() { - MockRoot root = new MockRoot(); - GenericServicesModel model = new GenericServicesModel(ConfigModelContext.create(root.getDeployState(), null, null, root, "foo")); - assertThat(model.serviceClusters().size(), is(0)); - model.addCluster(new ServiceCluster(root, "mycluster", "/bin/foo")); - assertThat(model.serviceClusters().size(), is(1)); - assertThat(model.serviceClusters().get(0).getName(), is("mycluster")); - } - - @Test - public void test_generic_services_parsing() throws IOException, SAXException { - final String hosts = - "<hosts>" + - "<host name=\"localhost\">" + - " <alias>mockhost</alias>" + - " </host> " + - "</hosts>"; - String services = "<services version=\"1.0\">" - + "<service id=\"me\" name=\"foo\" command=\"/bin/bar\" version=\"1.0\">" - + "<node hostalias=\"mockhost\" />" - + "</service>" - + "</services>"; - VespaModel model = new VespaModel(new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).build()); - GenericServicesModel gsModel = (GenericServicesModel) model.configModelRepo().get("me"); - assertThat(gsModel.serviceClusters().size(), is(1)); - assertThat(gsModel.serviceClusters().get(0).getName(), is("foo")); - assertThat(gsModel.serviceClusters().get(0).services().size(), is(1)); - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/generic/GenericServicesTest.java b/config-model/src/test/java/com/yahoo/vespa/model/generic/GenericServicesTest.java deleted file mode 100644 index 7429c2cc113..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/generic/GenericServicesTest.java +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.generic; - -import com.yahoo.cloud.config.SentinelConfig; -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.vespa.model.VespaModel; -import org.junit.BeforeClass; -import org.junit.Test; -import org.xml.sax.SAXException; - -import java.io.File; -import java.io.IOException; -import java.util.Iterator; - -import static org.junit.Assert.*; - -/** - * Tests that generic services result in correct sentinel config settings - * - * @author vegardh - */ -public class GenericServicesTest { - - private static VespaModel model; - - @BeforeClass - public static void getModel() throws IOException, SAXException { - String appDir = "src/test/cfg/application/app_genericservices"; - ApplicationPackage app = FilesApplicationPackage.fromFile(new File(appDir)); - model = new VespaModel(app); - } - - @Test - public void testServicesSentinelConfig() { - String sentinelConfigId1="hosts/bogusname1/sentinel"; - String sentinelConfigId2="hosts/bogusname2/sentinel"; - String sentinelConfigId3="hosts/bogusname3/sentinel"; - String sentinelConfigId4="hosts/bogusname4/sentinel"; - SentinelConfig sentinel1 = model.getConfig(SentinelConfig.class, sentinelConfigId1); - SentinelConfig sentinel2 = model.getConfig(SentinelConfig.class, sentinelConfigId2); - SentinelConfig sentinel3 = model.getConfig(SentinelConfig.class, sentinelConfigId3); - SentinelConfig sentinel4 = model.getConfig(SentinelConfig.class, sentinelConfigId4); - - assertServiceExists(sentinel1, "myservice", "mycmd1.sh", "myservice/0"); - assertServiceExists(sentinel2, "myservice", "mycmd1.sh", "myservice/1"); - assertServiceExists(sentinel3, "myservice", "mycmd1.sh", "myservice/2"); - assertServiceExists(sentinel3, "myservice2", "mycmd1.sh", "myservice/3"); - assertServiceExists(sentinel3, "myotherservice", "/home/vespa/bin/mycmd2.sh --ytest $FOO_BAR", "myotherservice/0"); - assertServiceExists(sentinel4, "myotherservice", "/home/vespa/bin/mycmd2.sh --ytest $FOO_BAR", "myotherservice/1"); - } - - private void assertServiceExists(SentinelConfig sentinel, String serviceName, String cmd, String configId) { - boolean matches = false; - Iterator<SentinelConfig.Service> it = sentinel.service().iterator(); - while (!matches && it.hasNext()) { - SentinelConfig.Service service = it.next(); - matches = - service.name().equals(serviceName) && - service.id().equals(configId) && - service.command().equals(cmd); - } - assertTrue(matches); - } - - @Test - public void testServicesModel() { - // Testing that this model can be constructed only for now - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java index c60817704cd..694b908478d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java @@ -170,7 +170,7 @@ public class ModelEvaluationTest { } private final String profile = - "rankingExpression(output).rankingScript: onnxModel(small_constants_and_functions).output\n" + + "rankingExpression(output).rankingScript: onnx(small_constants_and_functions).output\n" + "rankingExpression(output).type: tensor<float>(d0[3])\n"; private RankProfilesConfig.Rankprofile.Fef findProfile(String name, RankProfilesConfig config) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java index 3de8cfe540f..2ae1399e9d1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/ModelAmendingTestCase.java @@ -82,7 +82,7 @@ public class ModelAmendingTestCase { assertFalse(host + " is amended", host.getHost().getChildrenByTypeRecursive(AmendedService.class).isEmpty()); } - // Check that jdisc clusters are amended + // Check that container clusters are amended assertEquals(2, model.getContainerClusters().size()); assertNotNull(model.getContainerClusters().get("test1").getComponentsMap().get(new ComponentId("com.yahoo.MyAmendedComponent"))); assertNotNull(model.getContainerClusters().get("test2").getComponentsMap().get(new ComponentId("com.yahoo.MyAmendedComponent"))); @@ -129,7 +129,7 @@ public class ModelAmendingTestCase { assertFalse(host + " is amended", host.getHost().getChildrenByTypeRecursive(AmendedService.class).isEmpty()); } - // Check that jdisc clusters are amended + // Check that container clusters are amended assertEquals(2, model.getContainerClusters().size()); assertNotNull(model.getContainerClusters().get("test1").getComponentsMap().get(new ComponentId("com.yahoo.MyAmendedComponent"))); assertNotNull(model.getContainerClusters().get("test2").getComponentsMap().get(new ComponentId("com.yahoo.MyAmendedComponent"))); @@ -211,7 +211,7 @@ public class ModelAmendingTestCase { @Override public void doBuild(ContainerModelAmender model, Element spec, ConfigModelContext modelContext) { - if (built) return; // the same instance will be called once per jdisc cluster + if (built) return; // the same instance will be called once per container cluster for (ContainerModel containerModel : model.containerModels) amend(containerModel.getCluster()); built = true; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index 41f9b4c7b55..ed0cedd4e87 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -192,14 +192,19 @@ public class VespaModelTester { VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(hosts, services, generateSchemas("type1")); ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg; - provisioner = hosted ? new ProvisionerAdapter(new InMemoryProvisioner(hostsByResources, - failOnOutOfCapacity, - useMaxResources, - alwaysReturnOneNode, - false, - startIndexForClusters, - retiredHostNames)) : - new SingleNodeProvisioner(); + if (hosted) { + InMemoryProvisioner provisioner = new InMemoryProvisioner(hostsByResources, + failOnOutOfCapacity, + useMaxResources, + alwaysReturnOneNode, + false, + startIndexForClusters, + retiredHostNames); + provisioner.setEnvironment(zone.environment()); + this.provisioner = new ProvisionerAdapter(provisioner); + } else { + provisioner = new SingleNodeProvisioner(); + } TestProperties properties = new TestProperties() .setMultitenant(hosted) // Note: system tests are multitenant but not hosted diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 9ccf50de906..4fb81963152 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -60,11 +60,10 @@ <http> <filtering strict-mode="true"> - <access-control domain="my.athens-domain" read="true"> + <access-control> <exclude> <binding>http//*/foo/*</binding> </exclude> - <vespa-domain>vespa.vespa.cd</vespa-domain> </access-control> <filter id="com.yahoo.YcaFilter" bundle="mybundle"> @@ -121,9 +120,9 @@ </http> <accesslog type='json' - fileNamePattern='logs/vespa/qrs/access-json.%Y%m%d%H%M%S' compressOnRotation='true' compressionType='zstd' queueSize='13' bufferSize='65536'/> + fileNamePattern='logs/vespa/access/access-json.%Y%m%d%H%M%S' compressOnRotation='true' compressionType='zstd' queueSize='13' bufferSize='65536'/> <accesslog type='vespa' - fileNamePattern='logs/vespa/qrs/access-vespa.%Y%m%d%H%M%S' /> + fileNamePattern='logs/vespa/access/access-vespa.%Y%m%d%H%M%S' /> <document-api> <binding>http://*/document-api/</binding> @@ -148,8 +147,7 @@ <searcher id='inner-searcher' /> </chain> - <provider id='yca-provider' type='vespa' yca-application-id='my-app'> - <yca-proxy host='myhost' port='80'/> + <provider id='yca-provider' type='vespa'> <nodes> <node host='sourcehost' port='12'/> </nodes> @@ -195,17 +193,8 @@ <handler id="jdisc-handler"> <binding>http://*:*/HelloWorld</binding> <binding>http://*:*/Status</binding> - <clientBinding>http://*:*/foo</clientBinding> - <clientBinding>http://*:*/bar</clientBinding> </handler> - <client id="client-provider"> - <binding>http://*:*/HelloWorld</binding> - <binding>http://*:*/Status</binding> - <serverBinding>http://*:*/foo</serverBinding> - <serverBinding>http://*:*/bar</serverBinding> - </client> - <server id="server-provider"> </server> diff --git a/config-model/src/test/schema-test-files/standalone-container.xml b/config-model/src/test/schema-test-files/standalone-container.xml index cc34ae43712..67793df880b 100644 --- a/config-model/src/test/schema-test-files/standalone-container.xml +++ b/config-model/src/test/schema-test-files/standalone-container.xml @@ -55,8 +55,7 @@ <searcher id='inner-searcher' /> </chain> - <provider id='yca-provider' type='vespa' yca-application-id='my-app'> - <yca-proxy host='myhost' port='80'/> + <provider id='yca-provider' type='vespa'> <nodes> <node host='sourcehost' port='12'/> </nodes> @@ -94,18 +93,8 @@ <handler id="jdisc-handler"> <binding>http://*:*/HelloWorld</binding> <binding>http://*:*/Status</binding> - <clientBinding>http://*:*/foo</clientBinding> - <clientBinding>http://*:*/bar</clientBinding> </handler> - <client id="client-provider"> - <binding>http://*:*/HelloWorld</binding> - <binding>http://*:*/Status</binding> - <serverBinding>http://*:*/foo</serverBinding> - <serverBinding>http://*:*/bar</serverBinding> - </client> - <server id="server-provider" /> - </container> diff --git a/config-provisioning/pom.xml b/config-provisioning/pom.xml index c7616b2187a..119931465bf 100644 --- a/config-provisioning/pom.xml +++ b/config-provisioning/pom.xml @@ -5,7 +5,7 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> diff --git a/config-proxy/pom.xml b/config-proxy/pom.xml index 95cbac62dda..4272cf6617e 100644 --- a/config-proxy/pom.xml +++ b/config-proxy/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-proxy</artifactId> <packaging>jar</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <dependency> <groupId>junit</groupId> diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java index a559f3025ed..879b2b80273 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java @@ -34,7 +34,6 @@ import static com.yahoo.vespa.config.proxy.Mode.ModeName.DEFAULT; * * @author hmusum */ -@SuppressWarnings("removal") // TODO Vespa 8: remove (com.yahoo.log.event) public class ProxyServer implements Runnable { private static final DaemonThreadFactory threadFactory = new DaemonThreadFactory("ProxyServer"); diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java index 8f844fdd3ee..02c5aeb33c7 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java @@ -47,7 +47,6 @@ class RpcConfigSourceClient implements ConfigSourceClient, Runnable { private final Supervisor supervisor = new Supervisor(new Transport("config-source-client")); private final ResponseHandler responseHandler; - @SuppressWarnings("removal") // TODO Vespa 8: remove private final ConfigSourceSet configSourceSet; private final Object subscribersLock = new Object(); private final Map<ConfigCacheKey, Subscriber> subscribers = new ConcurrentHashMap<>(); @@ -62,7 +61,6 @@ class RpcConfigSourceClient implements ConfigSourceClient, Runnable { Executors.newScheduledThreadPool(1, new DaemonThreadFactory("delayed responses")); private final ScheduledFuture<?> delayedResponsesFuture; - @SuppressWarnings("removal") // TODO Vespa 8: remove RpcConfigSourceClient(ResponseHandler responseHandler, ConfigSourceSet configSourceSet) { this.responseHandler = responseHandler; this.configSourceSet = configSourceSet; diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java index 8e471aa1f5d..68570722117 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java @@ -26,7 +26,6 @@ public class FileDistributionAndUrlDownload { private final ScheduledExecutorService cleanupExecutor = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("file references and downloads cleanup")); - @SuppressWarnings("removal") // TODO Vespa 8: remove public FileDistributionAndUrlDownload(Supervisor supervisor, ConfigSourceSet source) { fileDistributionRpcServer = new FileDistributionRpcServer(supervisor, createDownloader(supervisor, source)); urlDownloadRpcServer = new UrlDownloadRpcServer(supervisor); @@ -45,7 +44,6 @@ public class FileDistributionAndUrlDownload { } } - @SuppressWarnings("removal") // TODO Vespa 8: remove private FileDownloader createDownloader(Supervisor supervisor, ConfigSourceSet source) { return new FileDownloader(new FileDistributionConnectionPool(source, supervisor), supervisor, diff --git a/config-proxy/src/main/sh/vespa-config-ctl.sh b/config-proxy/src/main/sh/vespa-config-ctl.sh index 8f2e6a44563..aa36e09c995 100755 --- a/config-proxy/src/main/sh/vespa-config-ctl.sh +++ b/config-proxy/src/main/sh/vespa-config-ctl.sh @@ -109,7 +109,12 @@ case $1 in nohup nice sbin/vespa-retention-enforcer > ${LOGDIR}/vre-start.log 2>&1 </dev/null & configsources=`bin/vespa-print-default configservers_rpc` userargs=$VESPA_CONFIGPROXY_JVMARGS - jvmopts="-Xms32M -Xmx128M -XX:+PreserveFramePointer -XX:+UseTransparentHugePages -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=448 -XX:MaxJavaStackTraceDepth=1000 -XX:-OmitStackTraceInFastThrow" + # Note that the hardcoded memory Xmx=128 migh be overriden in VESPA_CONFIGPROXY_JVMARGS, + # hence rendering get_jvm_hugepage_settings incorrect, but it is better than not. + # TODO Better way of extracting heap size arguments. + heap_min=32 + heap_max=128 + jvmopts="-Xms${heap_min}M -Xmx${heap_max}M -XX:+PreserveFramePointer $(get_jvm_hugepage_settings $heap_max) -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=448 -XX:MaxJavaStackTraceDepth=1000 -XX:-OmitStackTraceInFastThrow" VESPA_SERVICE_NAME=configproxy export VESPA_SERVICE_NAME diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java index 2105bd72975..f743c0ed231 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java @@ -256,7 +256,6 @@ public class ConfigProxyRpcServerTest { assertEquals("success", req.returnValues().get(0).asString()); } - @SuppressWarnings("removal") // TODO Vespa 8: remove private static ProxyServer createTestServer(ConfigSourceSet source) { return new ProxyServer(null, source, new RpcConfigSourceClient(new ResponseHandler(), source)); } @@ -265,7 +264,6 @@ public class ConfigProxyRpcServerTest { private static final Spec SPEC = new Spec(0); - @SuppressWarnings("removal") // TODO Vespa 8: remove private final ProxyServer proxyServer = createTestServer(new ConfigSourceSet(configSourceAddress)); private final Supervisor supervisor = new Supervisor(new Transport()); private final ConfigProxyRpcServer rpcServer = new ConfigProxyRpcServer(proxyServer, supervisor, SPEC); diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSource.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSource.java index 2b3624fe2b8..0377e20330a 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSource.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSource.java @@ -15,7 +15,6 @@ import java.util.Set; * * @author hmusum */ -@SuppressWarnings("removal") // TODO Vespa 8: remove class MockConfigSource extends ConfigSourceSet { private final HashMap<ConfigKey<?>, RawConfig> backing = new HashMap<>(); diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java index 1bcd1562a85..0e25e62b925 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java @@ -216,7 +216,6 @@ public class ProxyServerTest { assertEquals(ProxyServer.DEFAULT_PROXY_CONFIG_SOURCES, properties.configSources[0]); } - @SuppressWarnings("removal") // TODO Vespa 8: remove private static ProxyServer createTestServer(ConfigSourceSet source, ConfigSourceClient configSourceClient) { return new ProxyServer(null, source, configSourceClient); } diff --git a/config/abi-spec.json b/config/abi-spec.json index e94749cfba8..9e26dfeeb6e 100644 --- a/config/abi-spec.json +++ b/config/abi-spec.json @@ -1,312 +1 @@ -{ - "com.yahoo.config.subscription.CfgConfigPayloadBuilder": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public com.yahoo.vespa.config.ConfigPayload deserialize(java.util.List)", - "public com.yahoo.vespa.config.ConfigPayloadBuilder deserializeToBuilder(java.util.List)" - ], - "fields": [] - }, - "com.yahoo.config.subscription.ConfigDebug": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public static void logDebug(java.util.logging.Logger, long, com.yahoo.vespa.config.ConfigKey, java.lang.String)", - "public static void logDebug(java.util.logging.Logger, com.yahoo.config.ConfigInstance$Builder, java.lang.String, java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.config.subscription.ConfigGetter": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.Class)", - "public void <init>(com.yahoo.config.subscription.ConfigSource, java.lang.Class)", - "public synchronized com.yahoo.config.ConfigInstance getConfig(java.lang.String)", - "public static com.yahoo.config.ConfigInstance getConfig(java.lang.Class, java.lang.String)", - "public static com.yahoo.config.ConfigInstance getConfig(java.lang.Class, java.lang.String, com.yahoo.config.subscription.ConfigSource)" - ], - "fields": [] - }, - "com.yahoo.config.subscription.ConfigHandle": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "protected void <init>(com.yahoo.config.subscription.impl.ConfigSubscription)", - "public boolean isChanged()", - "public com.yahoo.config.ConfigInstance getConfig()", - "public java.lang.String toString()" - ], - "fields": [] - }, - "com.yahoo.config.subscription.ConfigInstanceSerializer": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.Serializer" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.slime.Slime)", - "public void <init>(com.yahoo.slime.Slime, com.yahoo.slime.Cursor)", - "public com.yahoo.config.Serializer createInner(java.lang.String)", - "public com.yahoo.config.Serializer createArray(java.lang.String)", - "public com.yahoo.config.Serializer createInner()", - "public com.yahoo.config.Serializer createMap(java.lang.String)", - "public void serialize(java.lang.String, boolean)", - "public void serialize(java.lang.String, double)", - "public void serialize(java.lang.String, int)", - "public void serialize(java.lang.String, long)", - "public void serialize(java.lang.String, java.lang.String)", - "public void serialize(boolean)", - "public void serialize(double)", - "public void serialize(long)", - "public void serialize(int)", - "public void serialize(java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.config.subscription.ConfigInstanceUtil": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public static void setValues(com.yahoo.config.ConfigBuilder, com.yahoo.config.ConfigBuilder)", - "public static com.yahoo.config.ConfigInstance getNewInstance(java.lang.Class, java.lang.String, com.yahoo.vespa.config.ConfigPayload)", - "public static java.lang.Object getField(com.yahoo.config.ConfigBuilder, java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.config.subscription.ConfigInterruptedException": { - "superClass": "java.lang.RuntimeException", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.Throwable)" - ], - "fields": [] - }, - "com.yahoo.config.subscription.ConfigSet": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.subscription.ConfigSource" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void addBuilder(java.lang.String, com.yahoo.config.ConfigInstance$Builder)", - "public com.yahoo.config.ConfigInstance$Builder get(com.yahoo.vespa.config.ConfigKey)", - "public boolean contains(com.yahoo.vespa.config.ConfigKey)", - "public java.lang.String toString()" - ], - "fields": [] - }, - "com.yahoo.config.subscription.ConfigSource": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [], - "fields": [] - }, - "com.yahoo.config.subscription.ConfigSourceSet": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.subscription.ConfigSource" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(java.util.List)", - "public void <init>(java.lang.String[])", - "public void <init>(java.lang.String)", - "public java.util.Set getSources()", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public java.lang.String toString()", - "public static com.yahoo.config.subscription.ConfigSourceSet createDefault()" - ], - "fields": [] - }, - "com.yahoo.config.subscription.ConfigSubscriber$SingleSubscriber": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void configure(com.yahoo.config.ConfigInstance)" - ], - "fields": [] - }, - "com.yahoo.config.subscription.ConfigSubscriber$State": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.config.subscription.ConfigSubscriber$State[] values()", - "public static com.yahoo.config.subscription.ConfigSubscriber$State valueOf(java.lang.String)" - ], - "fields": [ - "public static final enum com.yahoo.config.subscription.ConfigSubscriber$State OPEN", - "public static final enum com.yahoo.config.subscription.ConfigSubscriber$State FROZEN", - "public static final enum com.yahoo.config.subscription.ConfigSubscriber$State CLOSED" - ] - }, - "com.yahoo.config.subscription.ConfigSubscriber": { - "superClass": "java.lang.Object", - "interfaces": [ - "java.lang.AutoCloseable" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.config.subscription.ConfigSource)", - "public com.yahoo.config.subscription.ConfigHandle subscribe(java.lang.Class, java.lang.String)", - "public com.yahoo.config.subscription.ConfigHandle subscribe(java.lang.Class, java.lang.String, long)", - "protected void checkStateBeforeSubscribe()", - "protected void subscribeAndHandleErrors(com.yahoo.config.subscription.impl.ConfigSubscription, com.yahoo.vespa.config.ConfigKey, com.yahoo.config.subscription.ConfigHandle, com.yahoo.vespa.config.TimingValues)", - "public boolean nextConfig(boolean)", - "public boolean nextConfig()", - "public boolean nextConfig(long, boolean)", - "public boolean nextConfig(long)", - "public boolean nextGeneration(boolean)", - "public boolean nextGeneration()", - "public boolean nextGeneration(long, boolean)", - "public boolean nextGeneration(long)", - "protected void throwIfExceptionSet(com.yahoo.config.subscription.impl.ConfigSubscription)", - "public void close()", - "public java.lang.String toString()", - "public java.lang.Thread startConfigThread(java.lang.Runnable)", - "protected com.yahoo.config.subscription.ConfigSubscriber$State state()", - "public void reload(long)", - "public com.yahoo.config.subscription.ConfigSource getSource()", - "public boolean isClosed()", - "public com.yahoo.config.subscription.ConfigHandle subscribe(com.yahoo.config.subscription.ConfigSubscriber$SingleSubscriber, java.lang.Class, java.lang.String)", - "public long getGeneration()", - "protected void finalize()" - ], - "fields": [ - "protected final java.util.List subscriptionHandles" - ] - }, - "com.yahoo.config.subscription.ConfigURI": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public java.lang.String getConfigId()", - "public com.yahoo.config.subscription.ConfigSource getSource()", - "public static com.yahoo.config.subscription.ConfigURI createFromId(java.lang.String)", - "public static com.yahoo.config.subscription.ConfigURI createFromIdAndSource(java.lang.String, com.yahoo.config.subscription.ConfigSource)" - ], - "fields": [] - }, - "com.yahoo.config.subscription.DirSource": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.subscription.ConfigSource" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.io.File)", - "public java.io.File getDir()" - ], - "fields": [] - }, - "com.yahoo.config.subscription.FileSource": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.subscription.ConfigSource" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.io.File)", - "public java.io.File getFile()" - ], - "fields": [] - }, - "com.yahoo.config.subscription.JarSource": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.subscription.ConfigSource" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.util.jar.JarFile, java.lang.String)", - "public java.util.jar.JarFile getJarFile()", - "public java.lang.String getPath()" - ], - "fields": [] - }, - "com.yahoo.config.subscription.RawSource": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.subscription.ConfigSource" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String)" - ], - "fields": [ - "public final java.lang.String payload" - ] - }, - "com.yahoo.config.subscription.SubscriberClosedException": { - "superClass": "java.lang.RuntimeException", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()" - ], - "fields": [] - } -}
\ No newline at end of file +{}
\ No newline at end of file diff --git a/config/pom.xml b/config/pom.xml index 546e65c048c..f787a7b0b49 100755 --- a/config/pom.xml +++ b/config/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <!-- provided scope --> <dependency> @@ -73,6 +73,10 @@ <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-core</artifactId> </dependency> + <dependency> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> + </dependency> <!-- test scope --> <dependency> @@ -186,14 +190,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <!-- TODO Vespa 8: remove configuration. - Included to allow 'removal' warnings for classes in its own module --> - <configuration> - <compilerArgs> - <arg>-Xlint:all,-serial,-try,-processing,-removal</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> diff --git a/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java index 7d3fdad43c7..1a110fe8e6c 100644 --- a/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java +++ b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java @@ -17,9 +17,7 @@ import static java.util.logging.Level.FINEST; * Deserializes config payload (cfg format) to a ConfigPayload. * * @author hmusum - * @deprecated Will be removed in Vespa 8. Only for internal use. */ -@Deprecated(forRemoval = true, since = "7") public class CfgConfigPayloadBuilder { private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(CfgConfigPayloadBuilder.class.getName()); diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java b/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java index 2fcbdce7030..85aa7a05e17 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java @@ -9,7 +9,6 @@ import java.util.logging.Logger; import static java.util.logging.Level.INFO; // Debug class that provides useful helper routines -@Deprecated(forRemoval = true, since = "7") public class ConfigDebug { public static void logDebug(Logger logger, long timestamp, ConfigKey<?> key, String logmessage) { if (key.getConfigId().matches(".*container.?\\d+.*") || key.getConfigId().matches(".*doc.api.*")) { diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigGetter.java b/config/src/main/java/com/yahoo/config/subscription/ConfigGetter.java index 305ecbd6483..aabfd211fac 100755 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigGetter.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigGetter.java @@ -17,7 +17,6 @@ import com.yahoo.config.ConfigInstance; public class ConfigGetter<T extends ConfigInstance> { private final Class<T> clazz; - private final ConfigSource source; /** * Creates a ConfigGetter for class <code>clazz</code> @@ -25,22 +24,7 @@ public class ConfigGetter<T extends ConfigInstance> { * @param clazz a config class */ public ConfigGetter(Class<T> clazz) { - this(null, clazz); - } - - /** - * Creates a ConfigGetter for class <code>clazz</code> with the specified - * {@link ConfigSource}. - * - * @param source a {@link ConfigSource} - * @param clazz a config class - * - * @deprecated Config should always be injected via the component class constructor. For unit tests, use config builders. - */ - @Deprecated(forRemoval = true, since = "7") - public ConfigGetter(ConfigSource source, Class<T> clazz) { this.clazz = clazz; - this.source = source; } /** @@ -50,8 +34,7 @@ public class ConfigGetter<T extends ConfigInstance> { * @return an instance of a config class */ public synchronized T getConfig(String configId) { - try (ConfigSubscriber subscriber = - source == null ? new ConfigSubscriber() : new ConfigSubscriber(source)) { + try (ConfigSubscriber subscriber = new ConfigSubscriber()) { ConfigHandle<T> handle = subscriber.subscribe(clazz, configId); subscriber.nextConfig(true); return handle.getConfig(); @@ -70,19 +53,4 @@ public class ConfigGetter<T extends ConfigInstance> { return getter.getConfig(configId); } - /** - * Creates a ConfigGetter instance and returns an instance of the config class <code>c</code>. - * - * @param c a config class - * @param configId a config id to use when getting the config - * @param source a {@link ConfigSource} - * @return an instance of a config class - * - * @deprecated Config should always be injected via the component class constructor. For unit tests, use config builders. - */ - @Deprecated(forRemoval = true, since = "7") - public static <T extends ConfigInstance> T getConfig(Class<T> c, String configId, ConfigSource source) { - ConfigGetter<T> getter = new ConfigGetter<>(source, c); - return getter.getConfig(configId); - } } diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigHandle.java b/config/src/main/java/com/yahoo/config/subscription/ConfigHandle.java index 6b832205211..4ae6122becd 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigHandle.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigHandle.java @@ -6,13 +6,11 @@ import com.yahoo.config.subscription.impl.ConfigSubscription; /** * A config handle represents one config in the context of one active subscription on a {@link ConfigSubscriber}. - * It will contain meta data of the subscription of that particular config, as well as access to the {@link com.yahoo.config.ConfigInstance} itself. + * It will contain metadata of the subscription of that particular config, as well as access to the {@link com.yahoo.config.ConfigInstance} itself. * * @param <T> the type of the config * @author vegardh - * @deprecated Will be removed in Vespa 8. Only for internal use. */ -@Deprecated(forRemoval = true, since = "7") public class ConfigHandle<T extends ConfigInstance> { private final ConfigSubscription<T> sub; diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java index d8062a9f95b..e035991abf4 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java @@ -9,9 +9,7 @@ import com.yahoo.slime.Slime; * Implements a config instance serializer, serializing a config instance to a slime object. * * @author Ulf Lilleengen - * @deprecated Will be removed in Vespa 8. Only for internal use. */ -@Deprecated(forRemoval = true, since = "7") public class ConfigInstanceSerializer implements Serializer { private final Slime slime; private final Cursor root; diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java index 5831753eacb..61b421b30e0 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java @@ -1,22 +1,19 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription; -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; - import com.yahoo.config.ConfigBuilder; import com.yahoo.config.ConfigInstance; import com.yahoo.config.ConfigurationRuntimeException; -import com.yahoo.yolean.Exceptions; -import com.yahoo.vespa.config.*; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.ConfigTransformer; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; /** * @author gjoranv - * @deprecated Will be removed in Vespa 8. Only for internal use. */ -@Deprecated(forRemoval = true, since = "7") public class ConfigInstanceUtil { /** diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java index 050b7a81fa2..a7c2c2d9127 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java @@ -4,10 +4,8 @@ package com.yahoo.config.subscription; /** * This exception is thrown when any blocking call within the Config API is interrupted. * @author Ulf Lilleengen - * @deprecated Will be removed in Vespa 8. Only for internal use. */ @SuppressWarnings("serial") -@Deprecated(forRemoval = true, since = "7") public class ConfigInterruptedException extends RuntimeException { public ConfigInterruptedException(Throwable cause) { super(cause); diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java index 36d9047aa87..1b86db6b299 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java @@ -1,19 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - import com.yahoo.config.ConfigInstance; import com.yahoo.vespa.config.ConfigKey; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; /** * Config source as a programmatically built set of {@link com.yahoo.config.ConfigInstance}s * * @author Vegard Havdal - * @deprecated Will be removed in Vespa 8. Only for internal use. */ -@Deprecated(forRemoval = true, since = "7") public class ConfigSet implements ConfigSource { private final Map<ConfigKey<?>, ConfigInstance.Builder> configs = new ConcurrentHashMap<>(); diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java index b8abeb48c3d..5814a004216 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java @@ -5,9 +5,7 @@ package com.yahoo.config.subscription; * A type of source of config * * @author Vegard Havdal - * @deprecated Will be removed in Vespa 8. Only for internal use. */ -@Deprecated(forRemoval = true, since = "7") public interface ConfigSource { } diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java index 37f91605a54..81aeda52b54 100755 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java @@ -18,9 +18,7 @@ import static java.util.logging.Level.INFO; * upper/lower-casing and whitespaces. * * @author gjoranv - * @deprecated Will be removed in Vespa 8. Only for internal use. */ -@Deprecated(forRemoval = true, since = "7") public class ConfigSourceSet implements ConfigSource { private static final Logger log = Logger.getLogger(ConfigSourceSet.class.getName()); diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java index ee6ce37036c..5d5ad1f1edc 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java @@ -24,13 +24,11 @@ import static java.util.stream.Collectors.toList; * Used for subscribing to one or more configs. Can optionally be given a {@link ConfigSource} for the configs * that will be used when {@link #subscribe(Class, String)} is called. * - * {@link #subscribe(Class, String)} on the configs needed, call {@link #nextConfig(long)} and get the config from the + * {@link #subscribe(Class, String)} on the configs needed, call {@link #nextConfig(boolean)} and get the config from the * {@link ConfigHandle} which {@link #subscribe(Class, String)} returned. * * @author Vegard Havdal - * @deprecated Will be removed in Vespa 8. Only for internal use. */ -@Deprecated(forRemoval = true, since = "7") public class ConfigSubscriber implements AutoCloseable { private static final Logger log = Logger.getLogger(ConfigSubscriber.class.getName()); @@ -81,7 +79,7 @@ public class ConfigSubscriber implements AutoCloseable { /** * Subscribes on the given type of {@link ConfigInstance} with the given config id. * - * The method blocks until the first config is ready to be fetched with {@link #nextConfig()}. + * The method blocks until the first config is ready to be fetched with {@link #nextConfig(boolean)}. * * @param configClass The class, typically generated from a def-file using config-class-plugin * @param configId Identifies the service in vespa-services.xml, or null if you are using a local {@link ConfigSource} which does not use config id. @@ -96,7 +94,7 @@ public class ConfigSubscriber implements AutoCloseable { /** * Subscribes on the given type of {@link ConfigInstance} with the given config id and subscribe timeout. * - * The method blocks until the first config is ready to be fetched with {@link #nextConfig()}. + * The method blocks until the first config is ready to be fetched with {@link #nextConfig(boolean)}. * * @param configClass The class, typically generated from a def-file using config-class-plugin * @param configId Identifies the service in vespa-services.xml, or possibly raw:, file:, dir: or jar: type config which addresses config locally. @@ -162,11 +160,6 @@ public class ConfigSubscriber implements AutoCloseable { return nextConfig(TimingValues.defaultNextConfigTimeout, isInitializing); } - @Deprecated // TODO: Remove on Vespa 8 - public boolean nextConfig() { - return nextConfig(false); - } - /** * Use this for waiting for a new config that has changed, with the given timeout. * @@ -194,11 +187,6 @@ public class ConfigSubscriber implements AutoCloseable { return acquireSnapshot(timeoutMillis, true, isInitializing); } - @Deprecated // TODO: Remove on Vespa 8 - public boolean nextConfig(long timeoutMillis) { - return nextConfig(timeoutMillis, false); - } - /** * Use this for waiting for a new config generation. * @@ -226,11 +214,6 @@ public class ConfigSubscriber implements AutoCloseable { return nextGeneration(TimingValues.defaultNextConfigTimeout, isInitializing); } - @Deprecated // TODO: Remove on Vespa 8 - public boolean nextGeneration() { - return nextGeneration(false); - } - /** * Use this for waiting for a new config generation, with the given timeout * @@ -258,11 +241,6 @@ public class ConfigSubscriber implements AutoCloseable { return acquireSnapshot(timeoutMillis, false, isInitializing); } - @Deprecated // TODO: Remove on Vespa 8 - public boolean nextGeneration(long timeoutMillis) { - return nextGeneration(timeoutMillis, false); - } - /** * Acquire a snapshot of all configs with the same generation within a timeout. * @@ -464,7 +442,7 @@ public class ConfigSubscriber implements AutoCloseable { ConfigHandle<T> handle = subscribe(configClass, configId); - if ( ! nextConfig()) + if ( ! nextConfig(false)) throw new ConfigurationRuntimeException("Initial config of " + configClass.getName() + " failed"); singleSubscriber.configure(handle.getConfig()); @@ -473,7 +451,7 @@ public class ConfigSubscriber implements AutoCloseable { boolean hasNewConfig = false; try { - hasNewConfig = nextConfig(); + hasNewConfig = nextConfig(false); } catch (Exception e) { log.log(SEVERE, "Exception on receiving config. Ignoring this change.", e); diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigURI.java b/config/src/main/java/com/yahoo/config/subscription/ConfigURI.java index 46db3c6c83a..f0257457902 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigURI.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigURI.java @@ -1,18 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription; -import java.io.File; - import com.yahoo.config.subscription.impl.JRTConfigRequester; +import java.io.File; /** * A Config URI is a class that can be used to encapsulate a config source and a config id into one * object to simplify parameter passing. * * @author Ulf Lilleengen - * @deprecated Will be removed in Vespa 8. Only for internal use. */ -@Deprecated(forRemoval = true, since = "7") public class ConfigURI { private String configId; diff --git a/config/src/main/java/com/yahoo/config/subscription/DirSource.java b/config/src/main/java/com/yahoo/config/subscription/DirSource.java index 48d52bd49f3..fbd65ccf597 100644 --- a/config/src/main/java/com/yahoo/config/subscription/DirSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/DirSource.java @@ -6,9 +6,7 @@ import java.io.File; /** * Source specifying config from a local directory * @author Vegard Havdal - * @deprecated Will be removed in Vespa 8. Only for internal use. */ -@Deprecated(forRemoval = true, since = "7") public class DirSource implements ConfigSource { private final File dir; diff --git a/config/src/main/java/com/yahoo/config/subscription/FileSource.java b/config/src/main/java/com/yahoo/config/subscription/FileSource.java index bbad7286b4c..2178c64cca3 100644 --- a/config/src/main/java/com/yahoo/config/subscription/FileSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/FileSource.java @@ -7,9 +7,7 @@ import java.io.File; * Source specifying config from one local file * * @author Vegard Havdal - * @deprecated Will be removed in Vespa 8. Only for internal use. */ -@Deprecated(forRemoval = true, since = "7") public class FileSource implements ConfigSource { private final File file; diff --git a/config/src/main/java/com/yahoo/config/subscription/JarSource.java b/config/src/main/java/com/yahoo/config/subscription/JarSource.java index f39ee5e9262..c106823b52d 100644 --- a/config/src/main/java/com/yahoo/config/subscription/JarSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/JarSource.java @@ -6,9 +6,7 @@ import java.util.jar.JarFile; /** * Source specifying config as a jar file entry * @author Vegard Havdal - * @deprecated Will be removed in Vespa 8. Only for internal use. */ -@Deprecated(forRemoval = true, since = "7") public class JarSource implements ConfigSource { private final String path; private final JarFile jarFile; diff --git a/config/src/main/java/com/yahoo/config/subscription/RawSource.java b/config/src/main/java/com/yahoo/config/subscription/RawSource.java index 28b249c6ece..8b73d2058f5 100644 --- a/config/src/main/java/com/yahoo/config/subscription/RawSource.java +++ b/config/src/main/java/com/yahoo/config/subscription/RawSource.java @@ -5,9 +5,7 @@ package com.yahoo.config.subscription; * Source specifying raw config, where payload is given programmatically * * @author Vegard Havdal - * @deprecated Will be removed in Vespa 8. Only for internal use. */ -@Deprecated(forRemoval = true, since = "7") public class RawSource implements ConfigSource { public final String payload; diff --git a/config/src/main/java/com/yahoo/config/subscription/SubscriberClosedException.java b/config/src/main/java/com/yahoo/config/subscription/SubscriberClosedException.java index f7051ab1b38..1e6ebdbd752 100644 --- a/config/src/main/java/com/yahoo/config/subscription/SubscriberClosedException.java +++ b/config/src/main/java/com/yahoo/config/subscription/SubscriberClosedException.java @@ -5,7 +5,5 @@ package com.yahoo.config.subscription; * Thrown when {@link ConfigSubscriber} is closed * * @author bjorncs - * @deprecated Will be removed in Vespa 8. Only for internal use. */ -@Deprecated(forRemoval = true, since = "7") public class SubscriberClosedException extends RuntimeException {} diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java index 23c38812d64..8580b4584e2 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java @@ -132,12 +132,10 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc * @param jrtRequest a config request * @return an instance of a config class (subclass of ConfigInstance) */ - @SuppressWarnings("deprecation") private T toConfigInstance(JRTClientConfigRequest jrtRequest) { Payload payload = jrtRequest.getNewPayload(); ConfigPayload configPayload = ConfigPayload.fromUtf8Array(payload.withCompression(CompressionType.UNCOMPRESSED).getData()); T configInstance = configPayload.toInstance(configClass, jrtRequest.getConfigKey().getConfigId()); - configInstance.setConfigMd5(jrtRequest.getNewChecksums().getForType(MD5).asString()); // Note: Sets configmd5 in ConfigInstance return configInstance; } diff --git a/config/src/main/java/com/yahoo/config/subscription/package-info.java b/config/src/main/java/com/yahoo/config/subscription/package-info.java index 6798809dee2..bda7649d184 100644 --- a/config/src/main/java/com/yahoo/config/subscription/package-info.java +++ b/config/src/main/java/com/yahoo/config/subscription/package-info.java @@ -1,10 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage -@PublicApi /** * Classes for subscribing to Vespa config. */ package com.yahoo.config.subscription; -import com.yahoo.api.annotations.PublicApi; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigDefinition.java b/config/src/main/java/com/yahoo/vespa/config/ConfigDefinition.java index c60f4ad9478..96046e64c1a 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigDefinition.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigDefinition.java @@ -54,13 +54,6 @@ public class ConfigDefinition { private static final Double DOUBLE_MIN = -1e308d; private static final Double DOUBLE_MAX = 1e308d; - // TODO: Remove in Vespa 8 - @Deprecated - public ConfigDefinition(String name, String version, String namespace) { - this.name = name; - this.namespace = namespace; - } - public ConfigDefinition(String name, String namespace) { this.name = name; this.namespace = namespace; diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java b/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java index b4d3a886b10..16a3ef3371d 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java @@ -192,13 +192,6 @@ public class ConfigFileFormat implements SlimeFormat, ObjectTraverser { } @Override - @Deprecated(since = "7", forRemoval = true) - @SuppressWarnings("removal") - public void decode(InputStream is, Slime slime) throws IOException { - throw new UnsupportedOperationException("decode is not supported"); - } - - @Override public void field(String fieldName, Inspector inspector) { try { Node parent = nodeStack.peek(); diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java index 68cadb33ecd..fb116729640 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigGetterTest.java @@ -2,11 +2,8 @@ package com.yahoo.config.subscription; import com.yahoo.foo.AppConfig; - import org.junit.Test; -import java.io.File; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -16,7 +13,7 @@ import static org.junit.Assert.assertTrue; * @author gjoranv */ public class ConfigGetterTest { - private ConfigSourceSet sourceSet = new ConfigSourceSet("config-getter-test"); + private final ConfigSourceSet sourceSet = new ConfigSourceSet("config-getter-test"); @Test public void testGetConfig() { @@ -41,13 +38,6 @@ public class ConfigGetterTest { } @Test - public void testGetFromRawSource() { - ConfigGetter<AppConfig> getter = new ConfigGetter<>(new RawSource("message \"one\""), AppConfig.class); - AppConfig config = getter.getConfig("test"); - assertEquals("one", config.message()); - } - - @Test public void testGetTwice() { ConfigGetter<AppConfig> getter = new ConfigGetter<>(AppConfig.class); AppConfig config = getter.getConfig("raw:message \"one\""); @@ -63,13 +53,6 @@ public class ConfigGetterTest { verifyFooValues(config); } - @Test - public void testGetFromFileSource() { - ConfigGetter<AppConfig> getter = new ConfigGetter<>(new FileSource(new File("src/test/resources/configs/foo/app.cfg")), AppConfig.class); - AppConfig config = getter.getConfig("test"); - verifyFooValues(config); - } - @Test public void testGetFromDir() { ConfigGetter<AppConfig> getter = new ConfigGetter<>(AppConfig.class); @@ -77,12 +60,6 @@ public class ConfigGetterTest { verifyFooValues(config); } - @Test - public void testGetFromDirSource() { - AppConfig config = ConfigGetter.getConfig(AppConfig.class, "test", new DirSource(new File("src/test/resources/configs/foo/"))); - verifyFooValues(config); - } - private void verifyFooValues(AppConfig config) { assertEquals("msg1", config.message()); assertEquals(3, config.times()); diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSourceTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSourceTest.java deleted file mode 100644 index e7baab407f6..00000000000 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigSourceTest.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.subscription; - -import com.yahoo.foo.SimpletypesConfig; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.IOException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -/** - * @author Ulf Lilleengen - */ -public class ConfigSourceTest { - - @Test(expected = IllegalArgumentException.class) - public void require_that_FileSource_throws_exception_on_invalid_file() { - new FileSource(new File("invalid")); - } - - @Test(expected = IllegalArgumentException.class) - public void require_that_DirSource_throws_exception_on_invalid_dir() { - new DirSource(new File("invalid")); - } - - @Rule - public TemporaryFolder tmpDir = new TemporaryFolder(); - - @Test - public void require_that_DirSource_throws_exception_on_missing_file() throws IOException { - File folder = tmpDir.newFolder(); - DirSource dirSource = new DirSource(folder); - try { - ConfigGetter.getConfig(SimpletypesConfig.class, "dir:" + tmpDir, dirSource); - fail(); - } catch (IllegalArgumentException e) { - assertEquals("Could not find a config file for '" + SimpletypesConfig.getDefName() + "' in '" + folder + "/'", - e.getMessage()); - } - } - -} diff --git a/config_test/pom.xml b/config_test/pom.xml index b87ac3aed7c..dc35b53510b 100644 --- a/config_test/pom.xml +++ b/config_test/pom.xml @@ -6,7 +6,7 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>config_test</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <dependency> <groupId>junit</groupId> diff --git a/configdefinitions/pom.xml b/configdefinitions/pom.xml index 0f4f1ddbad3..47cc106c789 100644 --- a/configdefinitions/pom.xml +++ b/configdefinitions/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>configdefinitions</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>configdefinitions</name> <dependencies> <dependency> diff --git a/configdefinitions/src/vespa/attributes.def b/configdefinitions/src/vespa/attributes.def index 676c890d1a7..38cc763704c 100644 --- a/configdefinitions/src/vespa/attributes.def +++ b/configdefinitions/src/vespa/attributes.def @@ -10,7 +10,6 @@ attribute[].match enum { CASED, UNCASED } default=UNCASED attribute[].removeifzero bool default=false attribute[].createifnonexistent bool default=false attribute[].fastsearch bool default=false -attribute[].huge bool default=false attribute[].paged bool default=false # An attribute marked mutable can be updated by a query. attribute[].ismutable bool default=false @@ -46,7 +45,5 @@ attribute[].distancemetric enum { EUCLIDEAN, ANGULAR, GEODEGREES, INNERPRODUCT, attribute[].index.hnsw.enabled bool default=false attribute[].index.hnsw.maxlinkspernode int default=16 attribute[].index.hnsw.neighborstoexploreatinsert int default=200 -# Deprecated: Remove on Vespa 8 or before when possible. -attribute[].index.hnsw.distancemetric enum { EUCLIDEAN, ANGULAR, GEODEGREES, HAMMING } default=EUCLIDEAN # Whether multi-threaded indexing is enabled for this hnsw index. attribute[].index.hnsw.multithreadedindexing bool default=true diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index 64ca1e522f5..da383ecb6f5 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -66,8 +66,5 @@ maxDurationOfBootstrap long default=7200 # Code uses backoff, so wait time will increase for every iteration sleepTimeWhenRedeployingFails long default=15 -# Unused, remove in Vespa 8 -throwIfActiveSessionCannotBeLoaded bool default=true - canReturnEmptySentinelConfig bool default=false serverNodeType enum {config, controller} default=config diff --git a/configdefinitions/src/vespa/dispatch.def b/configdefinitions/src/vespa/dispatch.def index 1fe3c93ca0e..37b49ecaeb9 100644 --- a/configdefinitions/src/vespa/dispatch.def +++ b/configdefinitions/src/vespa/dispatch.def @@ -7,12 +7,6 @@ namespace=vespa.config.search # for that group to be included in queries minActivedocsPercentage double default=97.0 -# Not used. TODO: Remove on Vespa 8 -minGroupCoverage double default=100 - -# Not used. TODO: Remove on Vespa 8 -maxNodesDownPerGroup int default=0 - # Distribution policy for group selection distributionPolicy enum { ROUNDROBIN, ADAPTIVE } default=ADAPTIVE diff --git a/configgen/pom.xml b/configgen/pom.xml index ad8e1672dad..11d6a195bc6 100644 --- a/configgen/pom.xml +++ b/configgen/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>configgen</artifactId> <packaging>jar</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>configgen</name> <description>Config java code generation from config definition files for Java Vespa components.</description> <dependencies> diff --git a/configgen/src/main/java/com/yahoo/config/codegen/DefParser.java b/configgen/src/main/java/com/yahoo/config/codegen/DefParser.java index 34b07b7d098..2ed0bc62d0b 100644 --- a/configgen/src/main/java/com/yahoo/config/codegen/DefParser.java +++ b/configgen/src/main/java/com/yahoo/config/codegen/DefParser.java @@ -18,7 +18,7 @@ public class DefParser { public static final String DEFAULT_PACKAGE_PREFIX = "com.yahoo."; static final Pattern commentPattern = Pattern.compile("^\\s*#+\\s*(.*?)\\s*$"); - // TODO: Version is not used anymore, remove in Vespa 8 + // TODO: Version is ignored, remove in Vespa 9 public static final Pattern versionPattern = Pattern.compile("^(version\\s*=\\s*)([0-9][0-9-]*)$"); // Namespace/package must start with a letter, since Java (Java language Spec, section 3.8) and C++ identifiers cannot start with a digit public static final Pattern namespacePattern = getNamespacePattern("namespace"); @@ -127,7 +127,7 @@ public class DefParser { } Matcher versionMatch = versionPattern.matcher(line); if (versionMatch.matches()) { - // Do nothing, versions are not used + System.err.println("Warning: In config definition '" + name + "': version is deprecated and ignored, please remove, support will be removed in Vespa 9"); return; } Matcher namespaceMatcher = namespacePattern.matcher(line); diff --git a/configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java b/configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java index f3816adb260..2560b218e54 100644 --- a/configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java +++ b/configgen/src/main/java/com/yahoo/config/codegen/JavaClassBuilder.java @@ -86,15 +86,13 @@ public class JavaClassBuilder implements ClassBuilder { " public final static String CONFIG_DEF_MD5 = \"" + root.getMd5() + "\";\n" + // " public final static String CONFIG_DEF_NAME = \"" + root.getName() + "\";\n" + // " public final static String CONFIG_DEF_NAMESPACE = \"" + root.getNamespace() + "\";\n" + // - " public final static String CONFIG_DEF_VERSION = \"\";\n" + // TODO: Remove in Vespa 8 " public final static String[] CONFIG_DEF_SCHEMA = {\n" + // "" + indentCode(INDENTATION + INDENTATION, getDefSchema()) + "\n" + // " };\n" + // "\n" + // " public static String getDefMd5() { return CONFIG_DEF_MD5; }\n" + // " public static String getDefName() { return CONFIG_DEF_NAME; }\n" + // - " public static String getDefNamespace() { return CONFIG_DEF_NAMESPACE; }\n" + // - " public static String getDefVersion() { return CONFIG_DEF_VERSION; }"; + " public static String getDefNamespace() { return CONFIG_DEF_NAMESPACE; }"; } private String getDefSchema() { diff --git a/configgen/src/test/java/com/yahoo/config/codegen/DefParserNamespaceTest.java b/configgen/src/test/java/com/yahoo/config/codegen/DefParserNamespaceTest.java index 7921a3aecbe..2eb86546e51 100644 --- a/configgen/src/test/java/com/yahoo/config/codegen/DefParserNamespaceTest.java +++ b/configgen/src/test/java/com/yahoo/config/codegen/DefParserNamespaceTest.java @@ -19,7 +19,7 @@ public class DefParserNamespaceTest { @Test public void namespace_is_set_on_root_node() { - DefParser parser = createParser("version=1\nnamespace=myproject.config\n"); + DefParser parser = createParser("namespace=myproject.config\n"); CNode root = parser.getTree(); assertEquals("myproject.config", root.getNamespace()); } @@ -34,18 +34,17 @@ public class DefParserNamespaceTest { @Test(expected = CodegenRuntimeException.class) public void uppercase_chars_are_not_allowed() { - createParser("version=1\nnamespace=Foo\n").getTree(); + createParser("namespace=Foo\n").getTree(); } @Test(expected = CodegenRuntimeException.class) public void explicit_com_yahoo_prefix_is_not_allowed() { - createParser("version=1\n" + - "namespace=com.yahoo.myproject.config\n").getTree(); + createParser("namespace=com.yahoo.myproject.config\n").getTree(); } @Test public void spaces_are_allowed_around_equals_sign() { - DefParser parser = createParser("version=1\nnamespace = myproject.config\n"); + DefParser parser = createParser("namespace = myproject.config\n"); CNode root = parser.getTree(); assertEquals("myproject.config", root.getNamespace()); } @@ -62,10 +61,10 @@ public class DefParserNamespaceTest { @Test public void namespace_alters_def_md5() { - DefParser parser = createParser("version=1\n"); + DefParser parser = createParser(""); CNode root = parser.getTree(); - parser = createParser("version=1\nnamespace=myproject.config\n"); + parser = createParser("namespace=myproject.config\n"); CNode namespaceRoot = parser.getTree(); assertNotEquals(root.defMd5, namespaceRoot.defMd5); diff --git a/configgen/src/test/java/com/yahoo/config/codegen/DefParserTest.java b/configgen/src/test/java/com/yahoo/config/codegen/DefParserTest.java index e4d9f6cb3bd..915edce1440 100644 --- a/configgen/src/test/java/com/yahoo/config/codegen/DefParserTest.java +++ b/configgen/src/test/java/com/yahoo/config/codegen/DefParserTest.java @@ -1,19 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.codegen; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import org.junit.Test; import org.junit.Ignore; - +import org.junit.Test; import java.io.File; import java.io.FileReader; import java.io.IOException; import java.io.StringReader; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + /** * Unit tests for DefParser. * @@ -78,45 +77,35 @@ public class DefParserTest { @Test public void testMd5Sum2() { - String def = "version=1\na string\n"; + String def = "a string\n"; CNode root = new DefParser("testMd5Sum2", new StringReader(def)).getTree(); assertEquals("a5e5fdbb2b27e56ba7d5e60e335c598b", root.defMd5); } - @Test - public void testInvalidType() { - String line = "a sting"; - assertLineFails(line, "Could not create sting a"); - } - - // Note: Version is not used anymore, so will always be empty + // TODO: Version is not used anymore, remove test in Vespa 9 @Test public void testValidVersions() { try { - testExpectedVersion("version=8", ""); - testExpectedVersion("version=8-1", ""); - testExpectedVersion("version =8", ""); - testExpectedVersion("version = 8", ""); - testExpectedVersion("version = 8 ", ""); - testExpectedVersion("version =\t8", ""); + parse("version=8"); + parse("version=8-1"); + parse("version =8"); + parse("version = 8"); + parse("version = 8 "); + parse("version =\t8"); } catch (Exception e) { e.printStackTrace(); fail(); } } - private void testExpectedVersion(String versionLine, String expectedVersion) { - InnerCNode root = createParser(versionLine).getTree(); - assertEquals(expectedVersion, root.defVersion); + private void parse(String versionLine) { + InnerCNode ignored = createParser(versionLine).getTree(); } @Test - public void version_is_not_mandatory() { - try { - createParser("a string\n").parse(); - } catch (Exception e) { - fail("Should not get an exception here"); - } + public void testInvalidType() { + String line = "a sting"; + assertLineFails(line, "Could not create sting a"); } static DefParser createParser(String def) { @@ -124,24 +113,6 @@ public class DefParserTest { } @Test - public void testInvalidVersion() { - Class<?> exceptionClass = DefParser.DefParserException.class; - testInvalidVersion("version=a\n", exceptionClass, - "Error when parsing line 1: version=a\nversion=a"); - testInvalidVersion("version = a\n", exceptionClass, - "Error when parsing line 1: version = a\n a"); - } - - private void testInvalidVersion(String versionLine, Class<?> exceptionClass, String exceptionMessage) { - try { - createParser(versionLine).parse(); - fail("Didn't find expected exception of type " + exceptionClass); - } catch (Exception e) { - assertExceptionAndMessage(e, exceptionClass, exceptionMessage); - } - } - - @Test public void verify_fail_on_default_for_file() { assertLineFails("f file default=\"file1.txt\"", "Invalid default value"); @@ -150,7 +121,7 @@ public class DefParserTest { @Test(expected = CodegenRuntimeException.class) @Ignore("Not implemented yet") public void testInvalidEnum() { - DefParser parser = createParser("version=1\nanEnum enum {A, B, A}\n"); + DefParser parser = createParser("anEnum enum {A, B, A}\n"); //parser.validateDef(def); } @@ -191,7 +162,7 @@ public class DefParserTest { @Ignore //TODO: finish this! The numeric leaf nodes must contain their range. @Test public void testRanges() { - StringBuilder sb = new StringBuilder("version=1\n"); + StringBuilder sb = new StringBuilder(); sb.append("i int range=[0,10]"); sb.append("l long range=[-1e20,0]"); sb.append("d double range=[0,1]"); @@ -213,7 +184,7 @@ public class DefParserTest { fail("Didn't find expected exception of type " + exceptionClass); } catch (Exception e) { assertExceptionAndMessage(e, exceptionClass, - "Error when parsing line 4: " + duplicateLine + "b is already defined"); + "Error when parsing line 3: " + duplicateLine + "b is already defined"); } } @@ -292,7 +263,6 @@ public class DefParserTest { static StringBuilder createDefTemplate() { StringBuilder sb = new StringBuilder(); - sb.append("version=8\n"); // Add a comment line to check that we get correct line number with comments sb.append("# comment\n"); @@ -312,7 +282,7 @@ public class DefParserTest { fail("Didn't find expected exception of type " + exceptionClass); } catch (Exception e) { assertExceptionAndMessage(e, exceptionClass, - "Error when parsing line 3: " + line + "\n" + message); + "Error when parsing line 2: " + line + "\n" + message); } } diff --git a/configgen/src/test/java/com/yahoo/config/codegen/JavaClassBuilderTest.java b/configgen/src/test/java/com/yahoo/config/codegen/JavaClassBuilderTest.java index d69617ec5da..e6e86b2c2a3 100644 --- a/configgen/src/test/java/com/yahoo/config/codegen/JavaClassBuilderTest.java +++ b/configgen/src/test/java/com/yahoo/config/codegen/JavaClassBuilderTest.java @@ -29,7 +29,7 @@ public class JavaClassBuilderTest { @Ignore @Test public void visual_inspection_of_generated_class() { - final String testDefinition = "version=1\n" + // + final String testDefinition = "namespace=test\n" + // "p path\n" + // "pathArr[] path\n" + // @@ -74,7 +74,7 @@ public class JavaClassBuilderTest { @Test public void testCreateUniqueSymbol() { - final String testDefinition = "version=1\n" + // + final String testDefinition = "namespace=test\n" + // "m int\n" + // "n int\n"; diff --git a/configgen/src/test/java/com/yahoo/config/codegen/NormalizedDefinitionTest.java b/configgen/src/test/java/com/yahoo/config/codegen/NormalizedDefinitionTest.java index 411ac0fe58a..d57210dd9ed 100644 --- a/configgen/src/test/java/com/yahoo/config/codegen/NormalizedDefinitionTest.java +++ b/configgen/src/test/java/com/yahoo/config/codegen/NormalizedDefinitionTest.java @@ -23,7 +23,6 @@ public class NormalizedDefinitionTest { @Test public void testNormalizingFromReader() { String def = - "version=1\n" + "aString string \n" + "anInt int #comment \n" + "aStringCommentCharacterAfter string default=\"ab\" #foo\n" + @@ -42,13 +41,12 @@ public class NormalizedDefinitionTest { } assertNotNull(out); - assertEquals(6, out.size()); - assertEquals("version=1\n", out.get(0)); - assertEquals("aString string\n", out.get(1)); - assertEquals("anInt int\n", out.get(2)); - assertEquals("aStringCommentCharacterAfter string default=\"ab\"\n", out.get(3)); - assertEquals("aStringWithCommentCharacter string default=\"a#b\"\n", out.get(4)); - assertEquals("aStringWithEscapedQuote string default=\"a\"b\"\n", out.get(5)); + assertEquals(5, out.size()); + assertEquals("aString string\n", out.get(0)); + assertEquals("anInt int\n", out.get(1)); + assertEquals("aStringCommentCharacterAfter string default=\"ab\"\n", out.get(2)); + assertEquals("aStringWithCommentCharacter string default=\"a#b\"\n", out.get(3)); + assertEquals("aStringWithEscapedQuote string default=\"a\"b\"\n", out.get(4)); reader.close(); } diff --git a/configgen/src/test/resources/allfeatures.reference b/configgen/src/test/resources/allfeatures.reference index 21f49d21413..6ebcfc4ba12 100644 --- a/configgen/src/test/resources/allfeatures.reference +++ b/configgen/src/test/resources/allfeatures.reference @@ -38,7 +38,6 @@ public final class AllfeaturesConfig extends ConfigInstance { public final static String CONFIG_DEF_MD5 = "f901bdc5c96e7005130399c63f247823"; public final static String CONFIG_DEF_NAME = "allfeatures"; public final static String CONFIG_DEF_NAMESPACE = "configgen"; - public final static String CONFIG_DEF_VERSION = ""; public final static String[] CONFIG_DEF_SCHEMA = { "namespace=configgen", "boolVal bool", @@ -92,7 +91,6 @@ public final class AllfeaturesConfig extends ConfigInstance { public static String getDefMd5() { return CONFIG_DEF_MD5; } public static String getDefName() { return CONFIG_DEF_NAME; } public static String getDefNamespace() { return CONFIG_DEF_NAMESPACE; } - public static String getDefVersion() { return CONFIG_DEF_VERSION; } public interface Producer extends ConfigInstance.Producer { void getConfig(Builder builder); diff --git a/configserver-flags/pom.xml b/configserver-flags/pom.xml index 5d32daae833..43263885371 100644 --- a/configserver-flags/pom.xml +++ b/configserver-flags/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>configserver-flags</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <description>Config Server Flags.</description> @@ -26,6 +26,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>zkfacade</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/configserver/pom.xml b/configserver/pom.xml index adb2541fb6f..d66dad8bb9d 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -5,21 +5,22 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>configserver</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <!-- BEGIN Jersey deps. - TODO: remove after Vespa 8 is released, and provision-controller does not import any Jersey related packages. - Also, remove all package-info.java files for jersey/jackson packages. --> + TODO: Vespa > 8, and provision-controller does not import any Jersey related packages: + Remove, and remove all package-info.java files for jersey/jackson packages. --> <dependency> <groupId>com.fasterxml.jackson.jaxrs</groupId> <artifactId>jackson-jaxrs-json-provider</artifactId> + <version>${jackson2.version}</version> <exclusions> <exclusion> <!-- Conflicts with javax.activation:javax.activation-api:1.2.0, which is "exported" via jdisc_core. --> @@ -41,10 +42,12 @@ <dependency> <groupId>org.glassfish.jersey.core</groupId> <artifactId>jersey-client</artifactId> + <version>2.25</version> </dependency> <dependency> <groupId>org.glassfish.jersey.core</groupId> <artifactId>jersey-server</artifactId> + <version>2.25</version> <exclusions> <exclusion> <groupId>org.glassfish.jersey.media</groupId> @@ -55,10 +58,12 @@ <dependency> <groupId>org.glassfish.jersey.ext</groupId> <artifactId>jersey-proxy-client</artifactId> + <version>2.25</version> </dependency> <dependency> <groupId>org.glassfish.jersey.media</groupId> <artifactId>jersey-media-json-jackson</artifactId> + <version>2.25</version> <exclusions> <!-- Prevent embedding deps provided by jdisc --> <exclusion> @@ -74,6 +79,7 @@ <dependency> <groupId>org.glassfish.jersey.media</groupId> <artifactId>jersey-media-multipart</artifactId> + <version>2.25</version> </dependency> <!-- END Jersey deps --> @@ -191,11 +197,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava-testlib</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>zkfacade</artifactId> <version>${project.version}</version> @@ -278,6 +279,10 @@ <artifactId>http-utils</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> + </dependency> </dependencies> <build> @@ -287,7 +292,7 @@ <artifactId>bundle-plugin</artifactId> <extensions>true</extensions> <configuration> - <!-- TODO Vespa 8: remove importPackage when the jackson-jaxrs-json-provider bundle is no longer installed in jdisc --> + <!-- TODO: Vespa > 8: remove importPackage when the jackson-jaxrs-json-provider bundle is no longer installed in jdisc --> <!-- added to ensure using the same class as orchestrator, core-dump-reporter, provision-controller and controller-clients --> <importPackage>com.fasterxml.jackson.jaxrs.json;version="[2.12.6,3)"</importPackage> </configuration> diff --git a/configserver/src/main/java/com/fasterxml/jackson/jaxrs/json/package-info.java b/configserver/src/main/java/com/fasterxml/jackson/jaxrs/json/package-info.java index 4ea1e169f0f..167995c832d 100644 --- a/configserver/src/main/java/com/fasterxml/jackson/jaxrs/json/package-info.java +++ b/configserver/src/main/java/com/fasterxml/jackson/jaxrs/json/package-info.java @@ -1,4 +1,4 @@ -@ExportPackage(version = @Version(major = 2, minor = 12, micro = 6)) +@ExportPackage(version = @Version(major = 2, minor = 13, micro = 3)) package com.fasterxml.jackson.jaxrs.json; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java index 443ab47e786..1ac52213323 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java @@ -72,7 +72,6 @@ public class CompressedApplicationInputStream implements AutoCloseable { public File decompress(File dir) throws IOException { decompressInto(dir.toPath()); - dir = findActualApplicationDir(dir); return dir; } @@ -109,14 +108,4 @@ public class CompressedApplicationInputStream implements AutoCloseable { return Files.createTempFile(applicationDir, "application", null); } - private File findActualApplicationDir(File application) { - // If application is in e.g. application/, use that as root for UnpackedApplication - // TODO: Vespa 8: Remove application/ directory support - File[] files = application.listFiles(); - if (files != null && files.length == 1 && files[0].isDirectory()) { - application = files[0]; - } - return application; - } - } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 15bd898c053..88405b3eef9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -232,7 +232,6 @@ public class FileServer { } private static ConnectionPool createConnectionPool(List<String> configServers, Supervisor supervisor) { - @SuppressWarnings("removal") // TODO Vespa 8: remove ConfigSourceSet configSourceSet = new ConfigSourceSet(configServers); if (configServers.size() == 0) return FileDownloader.emptyConnectionPool(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java index db3c8d0d165..b489eb70927 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationApiHandler.java @@ -1,12 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; -import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.utils.MultiPartFormParser; +import com.yahoo.container.jdisc.utils.MultiPartFormParser.PartItem; import com.yahoo.jdisc.application.BindingMatch; import com.yahoo.jdisc.http.HttpHeaders; import com.yahoo.vespa.config.server.ApplicationRepository; @@ -18,9 +20,7 @@ import com.yahoo.vespa.config.server.http.v2.response.SessionPrepareAndActivateR import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.TenantRepository; import org.apache.hc.core5.http.ContentType; -import org.eclipse.jetty.http.MultiPartFormInputStream; -import javax.servlet.http.Part; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.Duration; @@ -28,7 +28,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.logging.Level; -import java.util.stream.Collectors; import static com.yahoo.vespa.config.server.application.CompressedApplicationInputStream.createFromCompressedStream; import static com.yahoo.vespa.config.server.http.Utils.checkThatTenantExists; @@ -80,15 +79,12 @@ public class ApplicationApiHandler extends SessionHandler { .orElse(false); if (multipartRequest) { try { - MultiPartFormInputStream multiPartFormInputStream = new MultiPartFormInputStream(request.getData(), request.getHeader(CONTENT_TYPE), /* config */null, /* contextTmpDir */null); - Map<String, Part> parts = multiPartFormInputStream.getParts().stream() - .collect(Collectors.toMap(Part::getName, p -> p)); - - byte[] params = parts.get(MULTIPART_PARAMS).getInputStream().readAllBytes(); + Map<String, PartItem> parts = new MultiPartFormParser(request).readParts(); + byte[] params = parts.get(MULTIPART_PARAMS).data().readAllBytes(); log.log(Level.FINE, "Deploy parameters: [{0}]", new String(params, StandardCharsets.UTF_8)); prepareParams = PrepareParams.fromJson(params, tenantName, zookeeperBarrierTimeout); - Part appPackagePart = parts.get(MULTIPART_APPLICATION_PACKAGE); - compressedStream = createFromCompressedStream(appPackagePart.getInputStream(), appPackagePart.getContentType(), maxApplicationPackageSize); + PartItem appPackagePart = parts.get(MULTIPART_APPLICATION_PACKAGE); + compressedStream = createFromCompressedStream(appPackagePart.data(), appPackagePart.contentType(), maxApplicationPackageSize); } catch (IOException e) { log.log(Level.WARNING, "Unable to parse multipart in deploy", e); throw new BadRequestException("Request contains invalid data"); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java index ccb17146df0..b2915bc74fb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java @@ -96,7 +96,6 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { File downloadDirectory, Supervisor supervisor) { List<String> otherConfigServersInCluster = getOtherConfigServersInCluster(configserverConfig); - @SuppressWarnings("removal") // TODO Vespa 8: remove ConfigSourceSet configSourceSet = new ConfigSourceSet(otherConfigServersInCluster); ConnectionPool connectionPool = (otherConfigServersInCluster.isEmpty()) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index f572f241ce0..5603ef2df51 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -782,7 +782,7 @@ public class SessionRepository { } } - // TODO: Remove in Vespa 8 (when we don't allow files in SEARCH_DEFINITIONS_DIR) + // TODO: Remove on Vespa 9 (when we don't allow files in SEARCH_DEFINITIONS_DIR) // Copies schemas from searchdefinitions/ to schemas/ if searchdefinitions/ exists private void moveSearchDefinitionsToSchemasDir(java.nio.file.Path applicationDir) throws IOException { File schemasDir = applicationDir.resolve(ApplicationPackage.SCHEMAS_DIR.getRelative()).toFile(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java index 6a483c38aee..0fa6e9a0704 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -122,12 +122,6 @@ public class ZKApplicationPackage extends AbstractApplicationPackage { } @Override - @SuppressWarnings("deprecation") - public String getApplicationName() { - return metaData.getApplicationId().application().value(); - } - - @Override public ApplicationId getApplicationId() { return metaData.getApplicationId(); } @Override diff --git a/configserver/src/main/sh/start-configserver b/configserver/src/main/sh/start-configserver index 6cf668e346a..cf0ebfbbe40 100755 --- a/configserver/src/main/sh/start-configserver +++ b/configserver/src/main/sh/start-configserver @@ -161,10 +161,15 @@ rm -f $cfpfile vespa-run-as-vespa-user sh -c "printenv > $cfpfile" fixddir $bundlecachedir +# Note that the hardcoded memory Xmx=2048 migh be overridden by jvmargs, +# hence rendering get_jvm_hugepage_settings incorrect, but it is better than not. +# TODO Better way of extracting heap size arguments. +heap_min=128 +heap_max=2048 vespa-run-as-vespa-user vespa-runserver -s ${VESPA_SERVICE_NAME} -r 30 -p $pidfile -- \ java \ - -Xms128m -Xmx2048m \ - -XX:+UseTransparentHugePages \ + -Xms${heap_min}m -Xmx${heap_max}m \ + $(get_jvm_hugepage_settings $heap_max) \ -XX:+PreserveFramePointer \ -XX:+HeapDumpOnOutOfMemoryError \ -XX:HeapDumpPath="${VESPA_HOME}/var/crash" \ diff --git a/configserver/src/test/apps/hosted-no-write-access-control/services.xml b/configserver/src/test/apps/hosted-no-write-access-control/services.xml index 9b36f71179b..578a1b02e8e 100644 --- a/configserver/src/test/apps/hosted-no-write-access-control/services.xml +++ b/configserver/src/test/apps/hosted-no-write-access-control/services.xml @@ -19,7 +19,7 @@ <documents> <document type="music" mode="index" /> </documents> - <nodes count="2" groups="2"/> + <nodes count="4" groups="2"/> </content> </services> diff --git a/configserver/src/test/apps/hosted/services.xml b/configserver/src/test/apps/hosted/services.xml index badd5f93c27..f1435d8cc4f 100644 --- a/configserver/src/test/apps/hosted/services.xml +++ b/configserver/src/test/apps/hosted/services.xml @@ -22,7 +22,7 @@ <documents> <document type="music" mode="index" /> </documents> - <nodes count="2" groups="2"/> + <nodes count="4" groups="2"/> </content> </services> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java index c3baaf8b118..9a90d517ca5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java @@ -66,7 +66,7 @@ public class ConfigServerBootstrapTest { @Test public void testBootstrap() throws Exception { ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); - InMemoryProvisioner provisioner = new InMemoryProvisioner(7, false); + InMemoryProvisioner provisioner = new InMemoryProvisioner(9, false); DeployTester tester = new DeployTester.Builder(temporaryFolder).modelFactory(createHostedModelFactory()) .configserverConfig(configserverConfig) .hostProvisioner(provisioner).build(); @@ -104,7 +104,7 @@ public class ConfigServerBootstrapTest { @Test public void testBootstrapWithVipStatusFile() throws Exception { ConfigserverConfig configserverConfig = createConfigserverConfig(temporaryFolder); - InMemoryProvisioner provisioner = new InMemoryProvisioner(7, false); + InMemoryProvisioner provisioner = new InMemoryProvisioner(9, false); DeployTester tester = new DeployTester.Builder(temporaryFolder).modelFactory(createHostedModelFactory()) .configserverConfig(configserverConfig).hostProvisioner(provisioner).build(); tester.deployApp("src/test/apps/hosted/"); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java index c7662ac9ee4..d3927309f65 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java @@ -77,35 +77,6 @@ public class CompressedApplicationInputStreamTest { } @Test - public void require_that_valid_tar_application_in_subdir_can_be_unpacked() throws IOException { - File outFile = Files.createTempFile(temporaryFolder.getRoot().toPath(), "testapp", ".tar.gz").toFile(); - ArchiveOutputStream archiveOutputStream = new TarArchiveOutputStream(new GZIPOutputStream(new FileOutputStream(outFile))); - - File app = new File("src/test/resources/deploy/validapp"); - - File file = new File(app, "services.xml"); - archiveOutputStream.putArchiveEntry(archiveOutputStream.createArchiveEntry(file, "application/" + file.getName())); - ByteStreams.copy(new FileInputStream(file), archiveOutputStream); - archiveOutputStream.closeArchiveEntry(); - file = new File(app, "hosts.xml"); - archiveOutputStream.putArchiveEntry(archiveOutputStream.createArchiveEntry(file, "application/" + file.getName())); - ByteStreams.copy(new FileInputStream(file), archiveOutputStream); - archiveOutputStream.closeArchiveEntry(); - file = new File(app, "deployment.xml"); - archiveOutputStream.putArchiveEntry(archiveOutputStream.createArchiveEntry(file, "application/" + file.getName())); - ByteStreams.copy(new FileInputStream(file), archiveOutputStream); - archiveOutputStream.closeArchiveEntry(); - - archiveOutputStream.close(); - - try (CompressedApplicationInputStream unpacked = streamFromTarGz(outFile)) { - File outApp = unpacked.decompress(); - assertEquals("application", outApp.getName()); // gets the name of the subdir - assertTestApp(outApp); - } - } - - @Test public void require_that_valid_zip_application_can_be_unpacked() throws IOException { File outFile = createZipFile(temporaryFolder.getRoot().toPath()); try (CompressedApplicationInputStream unpacked = streamFromZip(outFile)) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 9d01d56d72f..0ae4c43695c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -173,7 +173,7 @@ public class DeployTester { } private static HostProvisioner createProvisioner() { - return new InMemoryProvisioner(7, false); + return new InMemoryProvisioner(9, false); } private static class FailingModelFactory implements ModelFactory { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index c58b550d660..10e3ab1ec97 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -165,7 +165,7 @@ public class HostedDeployTest { .hostedConfigserverConfig(Zone.defaultZone()) .build(); tester.deployApp("src/test/apps/hosted/", "6.2.0"); - assertEquals(7, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); + assertEquals(9, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); } /** @@ -174,7 +174,7 @@ public class HostedDeployTest { */ @Test public void testCreateOnlyNeededModelVersions() { - List<Host> hosts = createHosts(7, "6.0.0", "6.1.0", null, "6.1.0"); // Use a host without a version as well. + List<Host> hosts = createHosts(9, "6.0.0", "6.1.0", null, "6.1.0"); // Use a host without a version as well. CountingModelFactory factory600 = createHostedModelFactory(Version.fromString("6.0.0")); CountingModelFactory factory610 = createHostedModelFactory(Version.fromString("6.1.0")); @@ -188,7 +188,7 @@ public class HostedDeployTest { DeployTester tester = createTester(hosts, modelFactories, prodZone); // Deploy with version that does not exist on hosts, the model for this version should also be created tester.deployApp("src/test/apps/hosted/", "7.0.0"); - assertEquals(7, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); + assertEquals(9, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); // Check >0 not ==0 as the session watcher thread is running and will redeploy models in the background assertTrue(factory600.creationCount() > 0); @@ -205,7 +205,7 @@ public class HostedDeployTest { */ @Test public void testCreateOnlyNeededModelVersionsNewNodes() { - List<Host> hosts = createHosts(7, (String) null); + List<Host> hosts = createHosts(9, (String) null); CountingModelFactory factory600 = createHostedModelFactory(Version.fromString("6.0.0")); CountingModelFactory factory610 = createHostedModelFactory(Version.fromString("6.1.0")); @@ -216,7 +216,7 @@ public class HostedDeployTest { DeployTester tester = createTester(hosts, modelFactories, prodZone); // Deploy with version that does not exist on hosts, the model for this version should also be created tester.deployApp("src/test/apps/hosted/", "7.0.0"); - assertEquals(7, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); + assertEquals(9, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); // Check >0 not ==0 as the session watcher thread is running and will redeploy models in the background assertTrue(factory700.creationCount() > 0); @@ -229,7 +229,7 @@ public class HostedDeployTest { */ @Test public void testCreateNeededModelVersionsForManuallyDeployedApps() { - List<Host> hosts = createHosts(5, "7.0.0"); + List<Host> hosts = createHosts(7, "7.0.0"); CountingModelFactory factory700 = createHostedModelFactory(Version.fromString("7.0.0"), devZone); CountingModelFactory factory710 = createHostedModelFactory(Version.fromString("7.1.0"), devZone); @@ -239,7 +239,7 @@ public class HostedDeployTest { DeployTester tester = createTester(hosts, modelFactories, devZone); // Deploy with version that does not exist on hosts, the model for this version should also be created tester.deployApp("src/test/apps/hosted/", "7.2.0"); - assertEquals(5, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); + assertEquals(7, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); // Check >0 not ==0 as the session watcher thread is running and will redeploy models in the background // Nodes are on 7.0.0 (should be created), no nodes on 7.1.0 (should not be created), 7.2.0 should always be created @@ -254,7 +254,7 @@ public class HostedDeployTest { */ @Test public void testCreateModelVersionsForManuallyDeployedAppsWhenCreatingFailsForOneVersion() { - List<Host> hosts = createHosts(5, "7.0.0"); + List<Host> hosts = createHosts(7, "7.0.0"); ModelFactory factory700 = createFailingModelFactory(Version.fromString("7.0.0")); CountingModelFactory factory720 = createHostedModelFactory(Version.fromString("7.2.0"), devZone); @@ -264,7 +264,7 @@ public class HostedDeployTest { // Deploy with version that does not exist on hosts, the model for this version should be created even // if creating 7.0.0 fails tester.deployApp("src/test/apps/hosted/", "7.2.0"); - assertEquals(5, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); + assertEquals(7, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); // Check >0 not ==0 as the session watcher thread is running and will redeploy models in the background assertTrue("Newest model for latest major version is always included", factory720.creationCount() > 0); @@ -299,7 +299,7 @@ public class HostedDeployTest { String oldestVersion = oldestMajorVersion + ".0.0"; String newestOnOldMajorVersion = oldestMajorVersion + ".1.0"; String newestOnNewMajorVersion = newestMajorVersion + ".2.0"; - List<Host> hosts = createHosts(7, oldestVersion, newestOnNewMajorVersion); + List<Host> hosts = createHosts(9, oldestVersion, newestOnNewMajorVersion); CountingModelFactory factory1 = createHostedModelFactory(Version.fromString(oldestVersion)); CountingModelFactory factory2 = createHostedModelFactory(Version.fromString(newestOnOldMajorVersion)); @@ -308,7 +308,7 @@ public class HostedDeployTest { DeployTester tester = createTester(hosts, modelFactories, prodZone); tester.deployApp("src/test/apps/hosted/", oldestVersion); - assertEquals(7, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); + assertEquals(9, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); // Check >0 not ==0 as the session watcher thread is running and will redeploy models in the background assertTrue(factory1.creationCount() > 0); @@ -350,7 +350,7 @@ public class HostedDeployTest { @Test public void testAccessControlIsOnlyCheckedWhenNoProdDeploymentExists() { // Provisioner does not reuse hosts, so need twice as many hosts as app requires - List<Host> hosts = createHosts(14, "6.0.0"); + List<Host> hosts = createHosts(18, "6.0.0"); List<ModelFactory> modelFactories = List.of(createHostedModelFactory(Version.fromString("6.0.0")), createHostedModelFactory(Version.fromString("6.1.0")), @@ -360,12 +360,12 @@ public class HostedDeployTest { ApplicationId applicationId = tester.applicationId(); // Deploy with oldest version tester.deployApp("src/test/apps/hosted/", "6.0.0"); - assertEquals(7, tester.getAllocatedHostsOf(applicationId).getHosts().size()); + assertEquals(9, tester.getAllocatedHostsOf(applicationId).getHosts().size()); // Deploy with version that does not exist on hosts and with app package that has no write access control, // validation of access control should not be done, since the app is already deployed in prod tester.deployApp("src/test/apps/hosted-no-write-access-control", "6.1.0"); - assertEquals(7, tester.getAllocatedHostsOf(applicationId).getHosts().size()); + assertEquals(9, tester.getAllocatedHostsOf(applicationId).getHosts().size()); } @Test @@ -408,7 +408,7 @@ public class HostedDeployTest { @Test public void testThatConfigChangeActionsAreCollectedFromAllModels() { - List<Host> hosts = createHosts(7, "6.1.0", "6.2.0"); + List<Host> hosts = createHosts(9, "6.1.0", "6.2.0"); List<ServiceInfo> services = List.of( new ServiceInfo("serviceName", "serviceType", null, new HashMap<>(), "configId", "hostName")); @@ -421,12 +421,12 @@ public class HostedDeployTest { DeployTester tester = createTester(hosts, modelFactories, prodZone); tester.deployApp("src/test/apps/hosted/", "6.2.0"); - assertEquals(7, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); + assertEquals(9, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); } @Test public void testThatAllowedConfigChangeActionsAreActedUpon() { - List<Host> hosts = createHosts(7, "6.1.0"); + List<Host> hosts = createHosts(9, "6.1.0"); List<ServiceInfo> services = List.of( new ServiceInfo("serviceName", "serviceType", null, Map.of("clustername", "cluster"), "configId", "hostName")); @@ -447,7 +447,7 @@ public class HostedDeployTest { .build(); PrepareResult prepareResult = tester.deployApp("src/test/apps/hosted/", "6.1.0"); - assertEquals(7, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); + assertEquals(9, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); assertTrue(prepareResult.configChangeActions().getRestartActions().isEmpty()); // Handled by deployment. assertEquals(Optional.of(ApplicationReindexing.empty() .withPending("cluster", "music", prepareResult.sessionId())), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java index ff6fc7d74c5..a2dc0216b72 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java @@ -31,7 +31,7 @@ class MaintainerTester { MaintainerTester(Clock clock, TemporaryFolder temporaryFolder) throws IOException { this.curator = new MockCurator(); - InMemoryProvisioner hostProvisioner = new InMemoryProvisioner(7, false); + InMemoryProvisioner hostProvisioner = new InMemoryProvisioner(9, false); Provisioner provisioner = new MockProvisioner().hostProvisioner(hostProvisioner); ConfigserverConfig configserverConfig = new ConfigserverConfig.Builder() .hostedVespa(true) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java index ee35ca572e1..8167cd8a2ee 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java @@ -222,11 +222,6 @@ public class LbServicesProducerTest { String services = "<services>" + "<admin version='4.0'><logservers> <nodes count='1' /> </logservers></admin>" + " <container id='mydisc' version='1.0'>" + - " <aliases>" + - " <endpoint-alias>foo2.bar2.com</endpoint-alias>" + - " <service-alias>service1</service-alias>" + - " <endpoint-alias>foo1.bar1.com</endpoint-alias>" + - " </aliases>" + " <nodes count='1' />" + " <search/>" + " </container>" + diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantTest.java index a0bd0e1f130..334119cd050 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantTest.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.tenant; -import com.google.common.testing.EqualsTester; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.TenantName; import org.junit.Before; @@ -50,11 +49,10 @@ public class TenantTest { @Test public void equals() { - new EqualsTester() - .addEqualityGroup(t1, t2) - .addEqualityGroup(t3) - .addEqualityGroup(t4) - .testEquals(); + assertEquals(t1, t2); + assertNotEquals(t1, t3); + assertNotEquals(t1, t4); + assertNotEquals(t3, t4); } @Test diff --git a/container-apache-http-client-bundle/pom.xml b/container-apache-http-client-bundle/pom.xml index 158ac140378..d2f70b91bb3 100644 --- a/container-apache-http-client-bundle/pom.xml +++ b/container-apache-http-client-bundle/pom.xml @@ -7,12 +7,12 @@ <modelVersion>4.0.0</modelVersion> <artifactId>container-apache-http-client-bundle</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <properties> diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 8ca6507f73e..70b4faa342e 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -171,10 +171,6 @@ "methods": [ "public void <init>()", "public void setClusters(java.util.Set)", - "public void setReceiveTrafficByDefault(boolean)", - "public void setUp(java.lang.Object)", - "public void setDown(java.lang.Object)", - "public boolean containerShouldReceiveTraffic()", "public boolean containerShouldReceiveTraffic(com.yahoo.container.handler.ClustersStatus$Require)" ], "fields": [] @@ -359,7 +355,6 @@ "public static java.lang.String getDefMd5()", "public static java.lang.String getDefName()", "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", "public void <init>(com.yahoo.container.handler.ThreadpoolConfig$Builder)", "public int maxthreads()", "public int corePoolSize()", @@ -372,7 +367,6 @@ "public static final java.lang.String CONFIG_DEF_MD5", "public static final java.lang.String CONFIG_DEF_NAME", "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", "public static final java.lang.String[] CONFIG_DEF_SCHEMA" ] }, @@ -402,14 +396,9 @@ "public void <init>(com.yahoo.container.handler.ClustersStatus)", "public void <init>(com.yahoo.container.QrSearchersConfig, com.yahoo.container.handler.ClustersStatus)", "public void <init>(com.yahoo.container.QrSearchersConfig, com.yahoo.container.core.VipStatusConfig, com.yahoo.container.handler.ClustersStatus, com.yahoo.container.jdisc.state.StateMonitor, com.yahoo.jdisc.Metric)", - "public void <init>(com.yahoo.container.QrSearchersConfig, com.yahoo.container.core.VipStatusConfig, com.yahoo.container.handler.ClustersStatus, com.yahoo.container.jdisc.state.StateMonitor)", - "public void <init>(com.yahoo.container.QrSearchersConfig, com.yahoo.container.handler.ClustersStatus, com.yahoo.container.jdisc.state.StateMonitor)", - "public void <init>(com.yahoo.container.QrSearchersConfig, com.yahoo.container.core.VipStatusConfig, com.yahoo.container.handler.ClustersStatus)", "public void setInRotation(java.lang.Boolean)", "public void addToRotation(java.lang.String)", "public void removeFromRotation(java.lang.String)", - "public void addToRotation(java.lang.Object)", - "public void removeFromRotation(java.lang.Object)", "public boolean isInRotation()" ], "fields": [] @@ -732,11 +721,9 @@ "public" ], "methods": [ - "public void <init>(java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.jdisc.Metric)", "public void <init>(java.util.concurrent.Executor, com.yahoo.jdisc.Metric)", "public void <init>(com.yahoo.container.jdisc.LoggingRequestHandler$Context)", "public java.util.concurrent.Executor getExecutor()", - "public com.yahoo.container.logging.AccessLog getAccessLog()", "public com.yahoo.jdisc.Metric getMetric()" ], "fields": [] @@ -751,12 +738,9 @@ "methods": [ "public static com.yahoo.container.jdisc.LoggingRequestHandler$Context testOnlyContext()", "public void <init>(com.yahoo.container.jdisc.LoggingRequestHandler$Context)", - "public void <init>(java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog)", "public void <init>(java.util.concurrent.Executor)", "public void <init>(com.yahoo.container.jdisc.LoggingRequestHandler$Context, boolean)", "public void <init>(java.util.concurrent.Executor, com.yahoo.jdisc.Metric)", - "public void <init>(java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.jdisc.Metric)", - "public void <init>(java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.jdisc.Metric, boolean)", "public void <init>(java.util.concurrent.Executor, com.yahoo.jdisc.Metric, boolean)", "protected com.yahoo.container.jdisc.LoggingCompletionHandler createLoggingCompletionHandler(long, long, com.yahoo.container.jdisc.HttpResponse, com.yahoo.container.jdisc.HttpRequest, com.yahoo.container.jdisc.ContentChannelOutputStream)" ], @@ -1057,7 +1041,6 @@ "public com.yahoo.jdisc.http.ConnectorConfig$Builder acceptQueueSize(int)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder reuseAddress(boolean)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder idleTimeout(double)", - "public com.yahoo.jdisc.http.ConnectorConfig$Builder stopTimeout(double)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder tcpKeepAliveEnabled(boolean)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder tcpNoDelay(boolean)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder throttling(com.yahoo.jdisc.http.ConnectorConfig$Throttling$Builder)", @@ -1409,7 +1392,6 @@ "public static java.lang.String getDefMd5()", "public static java.lang.String getDefName()", "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$Builder)", "public int listenPort()", "public java.lang.String name()", @@ -1420,7 +1402,6 @@ "public int acceptQueueSize()", "public boolean reuseAddress()", "public double idleTimeout()", - "public double stopTimeout()", "public boolean tcpKeepAliveEnabled()", "public boolean tcpNoDelay()", "public com.yahoo.jdisc.http.ConnectorConfig$Throttling throttling()", @@ -1439,7 +1420,6 @@ "public static final java.lang.String CONFIG_DEF_MD5", "public static final java.lang.String CONFIG_DEF_NAME", "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", "public static final java.lang.String[] CONFIG_DEF_SCHEMA" ] }, @@ -1493,7 +1473,6 @@ "public static java.lang.String toCookieHeader(java.lang.Iterable)", "public static java.util.List fromCookieHeader(java.lang.String)", "public static java.util.List toSetCookieHeaders(java.lang.Iterable)", - "public static java.util.List toSetCookieHeaderAll(java.lang.Iterable)", "public static com.yahoo.jdisc.http.Cookie fromSetCookieHeader(java.lang.String)" ], "fields": [] @@ -1768,20 +1747,6 @@ ], "fields": [] }, - "com.yahoo.jdisc.http.SecretStore": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract java.lang.String getSecret(java.lang.String)", - "public java.lang.String getSecret(java.lang.String, int)" - ], - "fields": [] - }, "com.yahoo.jdisc.http.ServerConfig$AccessLog$Builder": { "superClass": "java.lang.Object", "interfaces": [ @@ -1833,8 +1798,6 @@ "public void <init>(com.yahoo.jdisc.http.ServerConfig)", "public com.yahoo.jdisc.http.ServerConfig$Builder developerMode(boolean)", "public com.yahoo.jdisc.http.ServerConfig$Builder responseCompressionLevel(int)", - "public com.yahoo.jdisc.http.ServerConfig$Builder httpKeepAliveEnabled(boolean)", - "public com.yahoo.jdisc.http.ServerConfig$Builder maxKeepAliveRequests(int)", "public com.yahoo.jdisc.http.ServerConfig$Builder removeRawPostBodyForWwwUrlEncodedPost(boolean)", "public com.yahoo.jdisc.http.ServerConfig$Builder filter(com.yahoo.jdisc.http.ServerConfig$Filter$Builder)", "public com.yahoo.jdisc.http.ServerConfig$Builder filter(java.util.function.Consumer)", @@ -2062,12 +2025,9 @@ "public static java.lang.String getDefMd5()", "public static java.lang.String getDefName()", "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", "public void <init>(com.yahoo.jdisc.http.ServerConfig$Builder)", "public boolean developerMode()", "public int responseCompressionLevel()", - "public boolean httpKeepAliveEnabled()", - "public int maxKeepAliveRequests()", "public boolean removeRawPostBodyForWwwUrlEncodedPost()", "public java.util.List filter()", "public com.yahoo.jdisc.http.ServerConfig$Filter filter(int)", @@ -2086,7 +2046,6 @@ "public static final java.lang.String CONFIG_DEF_MD5", "public static final java.lang.String CONFIG_DEF_NAME", "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", "public static final java.lang.String[] CONFIG_DEF_SCHEMA" ] }, @@ -2168,8 +2127,6 @@ "public java.lang.String getMethod()", "public com.yahoo.jdisc.http.HttpRequest$Version getVersion()", "public java.net.URI getUri()", - "public void setUri(java.net.URI)", - "public com.yahoo.jdisc.http.HttpRequest getParentRequest()", "public java.lang.String getRemoteAddr()", "public void setRemoteAddr(java.lang.String)", "public java.lang.String getLocalAddr()", @@ -2211,7 +2168,6 @@ "public java.lang.String getRequestURI()", "public java.lang.String getRequestedSessionId()", "public java.lang.String getScheme()", - "public void setScheme(java.lang.String, boolean)", "public java.lang.String getServerName()", "public int getServerPort()", "public java.security.Principal getUserPrincipal()", @@ -2263,7 +2219,6 @@ "public void setCookies(java.util.List)", "public int getStatus()", "public void setStatus(int)", - "public com.yahoo.jdisc.http.HttpResponse getParentResponse()", "public void addCookie(com.yahoo.jdisc.http.filter.JDiscCookieWrapper)", "public void sendError(int)", "public void setCookie(java.lang.String, java.lang.String)" @@ -2311,17 +2266,6 @@ ], "fields": [] }, - "com.yahoo.jdisc.http.filter.JdiscFilterRequest": { - "superClass": "com.yahoo.jdisc.http.filter.DiscFilterRequest", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.jdisc.http.HttpRequest)" - ], - "fields": [] - }, "com.yahoo.jdisc.http.filter.RequestFilter": { "superClass": "java.lang.Object", "interfaces": [ @@ -2850,8 +2794,7 @@ "public void <init>(com.yahoo.processing.Request, com.yahoo.processing.request.ErrorMessage)", "public void mergeWith(com.yahoo.processing.Response)", "public com.yahoo.processing.response.DataList data()", - "public static java.util.concurrent.CompletableFuture recursiveFuture(com.yahoo.processing.response.DataList)", - "public static com.google.common.util.concurrent.ListenableFuture recursiveComplete(com.yahoo.processing.response.DataList)" + "public static java.util.concurrent.CompletableFuture recursiveFuture(com.yahoo.processing.response.DataList)" ], "fields": [] }, @@ -3152,7 +3095,6 @@ "public final java.util.concurrent.CompletableFuture renderResponse(java.io.OutputStream, com.yahoo.processing.Response, com.yahoo.processing.execution.Execution, com.yahoo.processing.Request)", "public void deconstruct()", "public final java.util.concurrent.CompletableFuture renderResponseBeforeHandover(java.io.OutputStream, com.yahoo.processing.Response, com.yahoo.processing.execution.Execution, com.yahoo.processing.Request)", - "public final com.google.common.util.concurrent.ListenableFuture renderBeforeHandover(java.io.OutputStream, com.yahoo.processing.Response, com.yahoo.processing.execution.Execution, com.yahoo.processing.Request)", "public com.yahoo.processing.execution.Execution getExecution()", "public com.yahoo.processing.Response getResponse()", "protected boolean clientClosed()", @@ -3197,8 +3139,7 @@ "public void <init>()", "public com.yahoo.processing.rendering.Renderer clone()", "public void init()", - "public com.google.common.util.concurrent.ListenableFuture render(java.io.OutputStream, com.yahoo.processing.Response, com.yahoo.processing.execution.Execution, com.yahoo.processing.Request)", - "public java.util.concurrent.CompletableFuture renderResponse(java.io.OutputStream, com.yahoo.processing.Response, com.yahoo.processing.execution.Execution, com.yahoo.processing.Request)", + "public abstract java.util.concurrent.CompletableFuture renderResponse(java.io.OutputStream, com.yahoo.processing.Response, com.yahoo.processing.execution.Execution, com.yahoo.processing.Request)", "public abstract java.lang.String getEncoding()", "public abstract java.lang.String getMimeType()", "public bridge synthetic com.yahoo.component.AbstractComponent clone()", @@ -3437,7 +3378,6 @@ "protected void <init>(com.yahoo.processing.Request, com.yahoo.processing.response.IncomingData, boolean, boolean)", "public com.yahoo.processing.Request request()", "public com.yahoo.processing.response.IncomingData incoming()", - "public com.google.common.util.concurrent.ListenableFuture complete()", "public java.util.concurrent.CompletableFuture completeFuture()", "public boolean isOrdered()", "public boolean isStreamed()", @@ -3498,7 +3438,6 @@ "public abstract java.util.List asList()", "public abstract com.yahoo.processing.response.IncomingData incoming()", "public abstract java.util.concurrent.CompletableFuture completeFuture()", - "public abstract com.google.common.util.concurrent.ListenableFuture complete()", "public abstract void addDataListener(java.lang.Runnable)", "public void close()" ], @@ -3517,7 +3456,6 @@ "public void <init>(com.yahoo.processing.response.DataList)", "public final void assignOwner(com.yahoo.processing.response.DataList)", "public com.yahoo.processing.response.DataList getOwner()", - "public com.google.common.util.concurrent.ListenableFuture completed()", "public java.util.concurrent.CompletableFuture completedFuture()", "public synchronized boolean isComplete()", "public synchronized void addLast(com.yahoo.processing.response.Data)", @@ -3582,7 +3520,6 @@ ], "methods": [ "public void <init>(com.yahoo.processing.response.DataList)", - "public com.google.common.util.concurrent.ListenableFuture completed()", "public java.util.concurrent.CompletableFuture completedFuture()", "public com.yahoo.processing.response.DataList getOwner()", "public boolean isComplete()", @@ -3608,7 +3545,6 @@ "methods": [ "public abstract com.yahoo.processing.response.DataList getOwner()", "public abstract java.util.concurrent.CompletableFuture completedFuture()", - "public abstract com.google.common.util.concurrent.ListenableFuture completed()", "public abstract boolean isComplete()", "public abstract void addLast(com.yahoo.processing.response.Data)", "public abstract void add(com.yahoo.processing.response.Data)", diff --git a/container-core/pom.xml b/container-core/pom.xml index 4b47cfde122..ed1ec8adca2 100644 --- a/container-core/pom.xml +++ b/container-core/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-core</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> @@ -104,21 +104,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>statistics</artifactId> - <version>${project.version}</version> - <exclusions> - <exclusion> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </exclusion> - <exclusion> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>vdslib</artifactId> <version>${project.version}</version> </dependency> @@ -126,11 +111,45 @@ <groupId>org.hdrhistogram</groupId> <artifactId>HdrHistogram</artifactId> </dependency> + + <!-- START JETTY embedded jars --> + <dependency> + <groupId>org.eclipse.jetty.alpn</groupId> + <artifactId>alpn-api</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty.http2</groupId> + <artifactId>http2-server</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-alpn-java-server</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-client</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-continuation</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-jmx</artifactId> + </dependency> <dependency> - <!-- TODO Vespa 8: stop providing org.json:json --> - <groupId>org.json</groupId> - <artifactId>json</artifactId> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlet</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlets</artifactId> + </dependency> + <!-- END JETTY embedded jars --> <!-- PROVIDED scope --> <dependency> @@ -220,12 +239,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>jdisc_jetty</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java index decb0513e26..8aee2d43187 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java @@ -20,7 +20,6 @@ import com.yahoo.jdisc.service.ClientProvider; import com.yahoo.jdisc.service.ServerProvider; import com.yahoo.osgi.OsgiImpl; import com.yahoo.osgi.OsgiWrapper; -import com.yahoo.statistics.Statistics; import org.osgi.framework.Bundle; import java.util.ArrayList; @@ -118,7 +117,6 @@ public class HandlersConfigurerDi { return result.oldComponentsCleanupTask(); } - @SuppressWarnings("deprecation") private Injector createFallbackInjector(com.yahoo.container.Container vespaContainer, Injector discInjector) { return discInjector.createChildInjector(new AbstractModule() { @Override @@ -126,7 +124,6 @@ public class HandlersConfigurerDi { // Provide a singleton instance for all component fallbacks, // otherwise fallback injection may lead to a cascade of components requiring reconstruction. bind(com.yahoo.container.Container.class).toInstance(vespaContainer); - bind(com.yahoo.statistics.Statistics.class).toInstance(Statistics.nullImplementation); bind(AccessLog.class).toInstance(AccessLog.NONE_INSTANCE); bind(Executor.class).toInstance(fallbackExecutor); if (vespaContainer.getFileAcquirer() != null) diff --git a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java index 96a479ac7ad..62d36ec34ec 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java @@ -39,7 +39,6 @@ import java.util.concurrent.Executors; */ public class HandlersConfigurerTestWrapper { - @SuppressWarnings("removal") // TODO Vespa 8: remove private final ConfigSourceSet configSources = new ConfigSourceSet(this.getClass().getSimpleName() + ": " + new Random().nextLong()); private final HandlersConfigurerDi configurer; diff --git a/container-core/src/main/java/com/yahoo/container/di/CloudSubscriber.java b/container-core/src/main/java/com/yahoo/container/di/CloudSubscriber.java index 2acbec2e388..5cbb009d54b 100644 --- a/container-core/src/main/java/com/yahoo/container/di/CloudSubscriber.java +++ b/container-core/src/main/java/com/yahoo/container/di/CloudSubscriber.java @@ -7,11 +7,9 @@ import com.yahoo.config.subscription.ConfigSource; import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.container.di.config.Subscriber; import com.yahoo.vespa.config.ConfigKey; - import java.util.HashMap; import java.util.Map; import java.util.Set; -import java.util.logging.Level; import java.util.logging.Logger; import static java.util.logging.Level.FINE; @@ -21,7 +19,6 @@ import static java.util.logging.Level.FINE; * @author Tony Vaagenes * @author ollivir */ -@SuppressWarnings("removal") // TODO Vespa 8: remove public class CloudSubscriber implements Subscriber { private static final Logger log = Logger.getLogger(CloudSubscriber.class.getName()); diff --git a/container-core/src/main/java/com/yahoo/container/di/CloudSubscriberFactory.java b/container-core/src/main/java/com/yahoo/container/di/CloudSubscriberFactory.java index a58baa5b2bd..a6327b01e21 100644 --- a/container-core/src/main/java/com/yahoo/container/di/CloudSubscriberFactory.java +++ b/container-core/src/main/java/com/yahoo/container/di/CloudSubscriberFactory.java @@ -20,7 +20,6 @@ import java.util.WeakHashMap; * @author Tony Vaagenes * @author ollivir */ -@SuppressWarnings("removal") // TODO Vespa 8: remove public class CloudSubscriberFactory implements SubscriberFactory { private final ConfigSource configSource; diff --git a/container-core/src/main/java/com/yahoo/container/di/Container.java b/container-core/src/main/java/com/yahoo/container/di/Container.java index 9977813b533..68dda5fbc4c 100644 --- a/container-core/src/main/java/com/yahoo/container/di/Container.java +++ b/container-core/src/main/java/com/yahoo/container/di/Container.java @@ -237,7 +237,6 @@ public class Container { } } - @SuppressWarnings("removal") // TODO Vespa 8: remove private void invalidateGeneration(long generation, Throwable cause) { leastGeneration = Math.max(retriever.getComponentsGeneration(), retriever.getBootstrapGeneration()) + 1; if (!(cause instanceof InterruptedException) && !(cause instanceof ConfigInterruptedException) && !(cause instanceof SubscriberClosedException)) { diff --git a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java index ad797c283c6..13839dd82c1 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java +++ b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java @@ -51,11 +51,6 @@ public class ClustersStatus extends AbstractComponent { } } - /** @deprecated this is ignored */ - @Deprecated // TODO: remove on Vespa 8 - public void setReceiveTrafficByDefault(boolean receiveTrafficByDefault) { - } - void setUp(String clusterIdentifier) { synchronized (mutex) { clusterStatus.put(clusterIdentifier, Boolean.TRUE); @@ -68,23 +63,6 @@ public class ClustersStatus extends AbstractComponent { } } - /** @deprecated use setUp(String) instead */ - @Deprecated // TODO: Remove on Vespa 8 - public void setUp(Object clusterIdentifier) { - setUp((String) clusterIdentifier); - } - - /** @deprecated use setDown(String) instead */ - @Deprecated // TODO: Remove on Vespa 8 - public void setDown(Object clusterIdentifier) { - setDown((String) clusterIdentifier); - } - - @Deprecated // TODO: Remove on Vespa 8 - public boolean containerShouldReceiveTraffic() { - return containerShouldReceiveTraffic(Require.ONE); - } - /** * Returns whether this container should receive traffic based on the state of this * diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java index 559c097618f..8e8b674c888 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java @@ -54,7 +54,7 @@ public class VipStatus { /** For testing */ public VipStatus(QrSearchersConfig dispatchers, ClustersStatus clustersStatus) { - this(dispatchers, new VipStatusConfig.Builder().build(), clustersStatus, StateMonitor.createForTesting()); + this(dispatchers, new VipStatusConfig.Builder().build(), clustersStatus, StateMonitor.createForTesting(), new NullMetric()); } @Inject @@ -71,24 +71,6 @@ public class VipStatus { updateCurrentlyInRotation(); } - @Deprecated // TODO: Remove on Vespa 8 - public VipStatus(QrSearchersConfig dispatchers, - VipStatusConfig vipStatusConfig, - ClustersStatus clustersStatus, - StateMonitor healthState) { - this(dispatchers, vipStatusConfig, clustersStatus, healthState, new NullMetric()); - } - - @Deprecated // TODO: Remove on Vespa 8 - public VipStatus(QrSearchersConfig dispatchers, ClustersStatus clustersStatus, StateMonitor healthState) { - this(dispatchers, new VipStatusConfig.Builder().build(), clustersStatus, healthState); - } - - @Deprecated // TODO: Remove on Vespa 8 - public VipStatus(QrSearchersConfig dispatchers, VipStatusConfig ignored, ClustersStatus clustersStatus) { - this(dispatchers, clustersStatus); - } - /** * Explicitly set this container in or out of rotation * @@ -114,18 +96,6 @@ public class VipStatus { updateCurrentlyInRotation(); } - /** @deprecated use addToRotation(String) instead */ - @Deprecated // TODO: Remove on Vespa 8 - public void addToRotation(Object clusterIdentifier) { - addToRotation((String) clusterIdentifier); - } - - /** @deprecated use removeFromRotation(String) instead */ - @Deprecated // TODO: Remove on Vespa 8 - public void removeFromRotation(Object clusterIdentifier) { - removeFromRotation((String) clusterIdentifier); - } - private void updateCurrentlyInRotation() { synchronized (mutex) { if (rotationOverride != null) { diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java index c6735559429..fa4ed9a4afa 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java @@ -28,7 +28,6 @@ import java.util.logging.Level; * * @deprecated Use {@link ThreadedHttpRequestHandler}, which provides the same level of functionality. */ -// TODO Vespa 8: Remove deprecated constructors @Deprecated public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { @@ -38,12 +37,6 @@ public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { final Executor executor; final Metric metric; - /** @deprecated Use {@link #Context(Executor, Metric)} instead */ - @Deprecated(forRemoval = true, since = "7") - public Context(Executor executor, AccessLog ignored, Metric metric) { - this(executor, metric); - } - @Inject public Context(Executor executor, Metric metric) { this.executor = executor; @@ -56,7 +49,6 @@ public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { } public Executor getExecutor() { return executor; } - @Deprecated(forRemoval = true, since = "7") public AccessLog getAccessLog() { return null; } public Metric getMetric() { return metric; } } @@ -76,12 +68,6 @@ public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { this(ctx.executor, ctx.metric); } - /** @deprecated Use {@link #LoggingRequestHandler(Executor)} instead */ - @Deprecated(forRemoval = true, since = "7") - public LoggingRequestHandler(Executor executor, AccessLog ignored) { - this(executor, (Metric)null); - } - public LoggingRequestHandler(Executor executor) { this(executor, (Metric)null); } @@ -94,18 +80,6 @@ public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { this(executor, metric, false); } - /** @deprecated Use {@link #LoggingRequestHandler(Executor, Metric)} instead */ - @Deprecated(forRemoval = true, since = "7") - public LoggingRequestHandler(Executor executor, AccessLog ignored, Metric metric) { - this(executor, metric, false); - } - - /** @deprecated Use {@link #LoggingRequestHandler(Executor, Metric, boolean)} instead */ - @Deprecated(forRemoval = true, since = "7") - public LoggingRequestHandler(Executor executor, AccessLog ignored, Metric metric, boolean allowAsyncResponse) { - this(executor, metric, allowAsyncResponse); - } - public LoggingRequestHandler(Executor executor, Metric metric, boolean allowAsyncResponse) { super(executor, metric, allowAsyncResponse); } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/utils/MultiPartFormParser.java b/container-core/src/main/java/com/yahoo/container/jdisc/utils/MultiPartFormParser.java new file mode 100644 index 00000000000..104d2f8ae4a --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/utils/MultiPartFormParser.java @@ -0,0 +1,64 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc.utils; + +import com.yahoo.container.jdisc.HttpRequest; +import org.eclipse.jetty.http.MultiPartFormInputStream; + +import javax.servlet.http.Part; +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; +import java.util.TreeMap; + +/** + * Wrapper around Jetty's {@link MultiPartFormInputStream}. + * + * @author bjorncs + */ +public class MultiPartFormParser { + + private final MultiPartFormInputStream multipart; + + public MultiPartFormParser(InputStream in, String contentType) { + this.multipart = new MultiPartFormInputStream(in, contentType, /*config*/null, /*contextTmpDir*/null); + } + + public MultiPartFormParser(HttpRequest request) { this(request.getData(), request.getHeader("Content-Type")); } + + public Map<String, PartItem> readParts() throws MultiPartException { + try { + Map<String, PartItem> result = new TreeMap<>(); + for (Part servletPart : multipart.getParts()) { + result.put(servletPart.getName(), new PartItem(servletPart)); + } + return result; + } catch (Exception e) { + throw new MultiPartException(e.getCause()); + } + } + + public static class PartItem { + private final String name; + private final InputStream data; + private final String contentType; + + private PartItem(Part servletPart) throws IOException { + this(servletPart.getName(), servletPart.getInputStream(), servletPart.getContentType()); + } + + public PartItem(String name, InputStream data, String contentType) { + this.name = name; + this.data = data; + this.contentType = contentType; + } + + public String name() { return name; } + public InputStream data() { return data; } + public String contentType() { return contentType; } + } + + public static class MultiPartException extends IOException { + public MultiPartException(Throwable cause) { super(cause.getMessage(), cause); } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/package-info.java b/container-core/src/main/java/com/yahoo/container/jdisc/utils/package-info.java index fc2cd94d4c4..d21e73a4ec2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/package-info.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/utils/package-info.java @@ -1,5 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Internal utils that are not public api + * + * @author bjorncs + */ @ExportPackage -package com.yahoo.vespa.model.generic.builder; +package com.yahoo.container.jdisc.utils; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/DatatypeFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/DatatypeFactoryProvider.java deleted file mode 100644 index 4777d588d48..00000000000 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/DatatypeFactoryProvider.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.xml.providers; - -import com.yahoo.container.di.componentgraph.Provider; - -import javax.xml.datatype.DatatypeConfigurationException; -import javax.xml.datatype.DatatypeFactory; - -/** - * @author Einar M R Rosenvinge - * @deprecated Do not use! - */ -@Deprecated // TODO: Remove on Vespa 8 -public class DatatypeFactoryProvider implements Provider<DatatypeFactory> { - - public static final String FACTORY_CLASS = DatatypeFactory.DATATYPEFACTORY_IMPLEMENTATION_CLASS; - - @Override - public DatatypeFactory get() { - try { - return DatatypeFactory.newInstance(FACTORY_CLASS, this.getClass().getClassLoader()); - } catch (DatatypeConfigurationException e) { - throw new IllegalStateException(e); - } - } - - @Override - public void deconstruct() { } - -} diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/DocumentBuilderFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/DocumentBuilderFactoryProvider.java deleted file mode 100644 index 769c3d7d7da..00000000000 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/DocumentBuilderFactoryProvider.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.xml.providers; - -import com.yahoo.container.di.componentgraph.Provider; - -import javax.xml.parsers.DocumentBuilderFactory; - -/** - * @author Einar M R Rosenvinge - * @deprecated Do not use! - */ -@Deprecated // TODO: Remove on Vespa 8 -public class DocumentBuilderFactoryProvider implements Provider<DocumentBuilderFactory> { - - public static final String FACTORY_CLASS = "com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"; - - @Override - public DocumentBuilderFactory get() { - return DocumentBuilderFactory.newInstance(FACTORY_CLASS, this.getClass().getClassLoader()); - } - - @Override - public void deconstruct() { } - -} diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/SAXParserFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/SAXParserFactoryProvider.java deleted file mode 100644 index 8dafbe3042a..00000000000 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/SAXParserFactoryProvider.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.xml.providers; - -import com.yahoo.container.di.componentgraph.Provider; - -import javax.xml.parsers.SAXParserFactory; - -/** - * @author Einar M R Rosenvinge - * @deprecated Do not use! - */ -@Deprecated // TODO: Remove on Vespa 8 -public class SAXParserFactoryProvider implements Provider<SAXParserFactory> { - - public static final String FACTORY_CLASS = "com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl"; - - @Override - public SAXParserFactory get() { - return SAXParserFactory.newInstance(FACTORY_CLASS, this.getClass().getClassLoader()); - } - - @Override - public void deconstruct() { } - -} diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/SchemaFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/SchemaFactoryProvider.java deleted file mode 100644 index 344686a252e..00000000000 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/SchemaFactoryProvider.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.xml.providers; - -import com.yahoo.container.di.componentgraph.Provider; - -import javax.xml.XMLConstants; -import javax.xml.validation.SchemaFactory; - -/** - * @author Einar M R Rosenvinge - * @deprecated Do not use! - */ -@Deprecated // TODO: Remove on Vespa 8 -public class SchemaFactoryProvider implements Provider<SchemaFactory> { - - public static final String FACTORY_CLASS = "com.sun.org.apache.xerces.internal.jaxp.validation.XMLSchemaFactory"; - - @Override - public SchemaFactory get() { - return SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI, FACTORY_CLASS, this.getClass().getClassLoader()); - } - - @Override - public void deconstruct() { } - -} diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/TransformerFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/TransformerFactoryProvider.java deleted file mode 100644 index 851c4c1c51c..00000000000 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/TransformerFactoryProvider.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.xml.providers; - -import com.yahoo.container.di.componentgraph.Provider; - -import javax.xml.transform.TransformerFactory; - -/** - * @author Einar M R Rosenvinge - * @deprecated Do not use! - */ -@Deprecated // TODO: Remove on Vespa 8 -public class TransformerFactoryProvider implements Provider<TransformerFactory> { - - public static final String FACTORY_CLASS = "com.sun.org.apache.xalan.internal.xsltc.trax.TransformerFactoryImpl"; - - @Override - public TransformerFactory get() { - return TransformerFactory.newInstance(FACTORY_CLASS, this.getClass().getClassLoader()); - } - - @Override - public void deconstruct() { } - -} diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/XMLEventFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/XMLEventFactoryProvider.java deleted file mode 100644 index 5039d47855d..00000000000 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/XMLEventFactoryProvider.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.xml.providers; - -import com.yahoo.container.di.componentgraph.Provider; - -import javax.xml.stream.XMLEventFactory; - -/** - * @author Einar M R Rosenvinge - * @deprecated Do not use! - */ -@Deprecated // TODO: Remove on Vespa 8 -public class XMLEventFactoryProvider implements Provider<XMLEventFactory> { - - public static final String FACTORY_CLASS = "com.sun.xml.internal.stream.events.XMLEventFactoryImpl"; - - @Override - public XMLEventFactory get() { - System.setProperty("javax.xml.stream.XMLEventFactory", FACTORY_CLASS); - // NOTE: In case the newFactory(String, ClassLoader) is used, XMLEventFactory treats the string as classname. - return XMLEventFactory.newFactory(); - } - - @Override - public void deconstruct() { } - -} diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/XMLInputFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/XMLInputFactoryProvider.java deleted file mode 100644 index 07a0f740d55..00000000000 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/XMLInputFactoryProvider.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.xml.providers; - -import com.yahoo.container.di.componentgraph.Provider; - -import javax.xml.stream.XMLInputFactory; - -/** - * @author Einar M R Rosenvinge - * @deprecated Do not use! - */ -@Deprecated // TODO: Remove on Vespa 8 -public class XMLInputFactoryProvider implements Provider<XMLInputFactory> { - - private static final String INPUT_FACTORY_INTERFACE = XMLInputFactory.class.getName(); - public static final String FACTORY_CLASS = "com.sun.xml.internal.stream.XMLInputFactoryImpl"; - - @Override - @SuppressWarnings("deprecation") - public XMLInputFactory get() { - //ugly, but must be done - System.setProperty(INPUT_FACTORY_INTERFACE, FACTORY_CLASS); - - // NOTE: In case the newFactory(String, ClassLoader) is used, - // the given class loader is ignored if the system property is set! - return XMLInputFactory.newFactory(); - } - - @Override - public void deconstruct() { } - -} diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/XMLOutputFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/XMLOutputFactoryProvider.java deleted file mode 100644 index 50d10836938..00000000000 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/XMLOutputFactoryProvider.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.xml.providers; - -import com.yahoo.container.di.componentgraph.Provider; - -import javax.xml.stream.XMLOutputFactory; - -/** - * @author Einar M R Rosenvinge - * @deprecated Do not use! - */ -@Deprecated // TODO: Remove on Vespa 8 -public class XMLOutputFactoryProvider implements Provider<XMLOutputFactory> { - - public static final String FACTORY_CLASS = "com.sun.xml.internal.stream.XMLOutputFactoryImpl"; - @Override - public XMLOutputFactory get() { - System.setProperty("javax.xml.stream.XMLOutputFactory", FACTORY_CLASS); - - // NOTE: In case the newFactory(String, ClassLoader) is used, XMLOutputFactory treats the string as system - // property name. Also, the given class loader is ignored if the property is set! - return XMLOutputFactory.newFactory(); - } - - @Override - public void deconstruct() { } - -} diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/XPathFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/XPathFactoryProvider.java deleted file mode 100644 index 429cf09b5ab..00000000000 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/XPathFactoryProvider.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.xml.providers; - -import com.yahoo.container.di.componentgraph.Provider; - -import javax.xml.xpath.XPathFactory; -import javax.xml.xpath.XPathFactoryConfigurationException; - -/** - * @author Einar M R Rosenvinge - * @deprecated Do not use! - */ -@Deprecated // TODO: Remove on Vespa 8 -public class XPathFactoryProvider implements Provider<XPathFactory> { - - public static final String FACTORY_CLASS = "com.sun.org.apache.xpath.internal.jaxp.XPathFactoryImpl"; - - @Override - public XPathFactory get() { - try { - return XPathFactory.newInstance(XPathFactory.DEFAULT_OBJECT_MODEL_URI, - FACTORY_CLASS, - this.getClass().getClassLoader()); - } catch (XPathFactoryConfigurationException e) { - throw new IllegalStateException(e); - } - } - - @Override - public void deconstruct() { } - -} diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/package-info.java b/container-core/src/main/java/com/yahoo/container/xml/providers/package-info.java deleted file mode 100644 index d29f104910f..00000000000 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.container.xml.providers; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/Cookie.java b/container-core/src/main/java/com/yahoo/jdisc/http/Cookie.java index c3f787882ec..b194124294c 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/Cookie.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/Cookie.java @@ -215,11 +215,6 @@ public class Cookie { .collect(toList()); } - @Deprecated // TODO Vespa 8 Remove - public static List<String> toSetCookieHeaderAll(Iterable<? extends Cookie> cookies) { - return toSetCookieHeaders(cookies); - } - public static Cookie fromSetCookieHeader(String headerVal) { return java.net.HttpCookie.parse(headerVal).stream() .map(httpCookie -> { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/SecretStore.java b/container-core/src/main/java/com/yahoo/jdisc/http/SecretStore.java deleted file mode 100644 index 30e3e770a00..00000000000 --- a/container-core/src/main/java/com/yahoo/jdisc/http/SecretStore.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http; - -/** - * An abstraction of a secret store for e.g passwords. - * Implementations can be plugged in to provide passwords for various keys. - * - * @author bratseth - * @author bjorncs - * @deprecated Use com.yahoo.container.jdisc.secretstore.SecretStore - */ -@Deprecated // Vespa 8 -public interface SecretStore { - - /** Returns the secret for this key */ - String getSecret(String key); - - /** Returns the secret for this key and version */ - default String getSecret(String key, int version) { - throw new UnsupportedOperationException("SecretStore implementation does not support versioned secrets"); - } - -} diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterRequest.java b/container-core/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterRequest.java index a5d133b9eb4..cace3854973 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterRequest.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterRequest.java @@ -65,17 +65,6 @@ public class DiscFilterRequest { return parent.getUri(); } - @Deprecated - public void setUri(URI uri) { parent.setUri(uri); } - - /** - * @deprecated Use methods on {@link DiscFilterRequest} instead to inspect request - */ - @Deprecated(forRemoval = true, since = "7.511") - public HttpRequest getParentRequest() { - return parent; - } - /** * Returns the Internet Protocol (IP) address of the client * or last proxy that sent the request. @@ -341,14 +330,6 @@ public class DiscFilterRequest { return getUri().getScheme(); } - @Deprecated - public void setScheme(String scheme, boolean isSecure) { - String uri = getUri().toString(); - String arr [] = uri.split("://"); - URI newUri = URI.create(scheme + "://" + arr[1]); - setUri(newUri); - } - public String getServerName() { return getUri().getHost(); } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterResponse.java b/container-core/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterResponse.java index af768a98d2d..e67677c4e7d 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterResponse.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/filter/DiscFilterResponse.java @@ -133,14 +133,6 @@ public class DiscFilterResponse { parent.setStatus(status); } - /** - * Return the parent HttpResponse - * - * @deprecated Use methods on {@link DiscFilterResponse} instead - */ - @Deprecated(forRemoval = true, since = "7.511") - public HttpResponse getParentResponse() { return (HttpResponse) parent; } - public void addCookie(JDiscCookieWrapper cookie) { if(cookie != null) { List<Cookie> cookies = new ArrayList<>(); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/filter/JdiscFilterRequest.java b/container-core/src/main/java/com/yahoo/jdisc/http/filter/JdiscFilterRequest.java deleted file mode 100644 index 74c3b8adc7d..00000000000 --- a/container-core/src/main/java/com/yahoo/jdisc/http/filter/JdiscFilterRequest.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.filter; - -import com.yahoo.jdisc.http.HttpRequest; - -/** - * JDisc implementation of a filter request. - * - */ -@Deprecated(forRemoval = true, since = "7.511") -public class JdiscFilterRequest extends DiscFilterRequest { - - public JdiscFilterRequest(HttpRequest parent) { - super(parent); - } - -} diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java index b56743954f4..7060a23111b 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java @@ -189,19 +189,9 @@ public class ConnectorFactory { @SuppressWarnings("removal") private SslContextFactory createSslContextFactory() { - try { - DefaultConnectorSsl ssl = new DefaultConnectorSsl(); - sslProvider.configureSsl(ssl, connectorConfig.name(), connectorConfig.listenPort()); - return ssl.createSslContextFactory(); - } catch (UnsupportedOperationException e) { - // TODO(bjorncs) Vespa 8 Remove this compatibility workaround - if (sslProvider instanceof SslContextFactoryProvider) { - return ((SslContextFactoryProvider) sslProvider) - .getInstance(connectorConfig.name(), connectorConfig.listenPort()); - } else { - throw e; - } - } + DefaultConnectorSsl ssl = new DefaultConnectorSsl(); + sslProvider.configureSsl(ssl, connectorConfig.name(), connectorConfig.listenPort()); + return ssl.createSslContextFactory(); } private ALPNServerConnectionFactory newAlpnConnectionFactory() { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java index 631f4080c7e..22c5b2ebfdb 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollector.java @@ -317,9 +317,7 @@ class HttpResponseStatisticsCollector extends HandlerWrapper implements Graceful @SuppressWarnings("removal") private static Collection<String> metricNames(Request req) { int code = req.getResponse().getStatus(); - if (code == 401) return Set.of(MetricDefinitions.RESPONSES_401, MetricDefinitions.RESPONSES_4XX); - else if (code == 403) return Set.of(MetricDefinitions.RESPONSES_403, MetricDefinitions.RESPONSES_4XX); - else if (code < 200) return Set.of(MetricDefinitions.RESPONSES_1XX); + if (code < 200) return Set.of(MetricDefinitions.RESPONSES_1XX); else if (code < 300) return Set.of(MetricDefinitions.RESPONSES_2XX); else if (code < 400) return Set.of(MetricDefinitions.RESPONSES_3XX); else if (code < 500) return Set.of(MetricDefinitions.RESPONSES_4XX); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java index 686d23c4c99..29858644d56 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java @@ -50,8 +50,6 @@ class MetricDefinitions { static final String RESPONSES_3XX = "http.status.3xx"; static final String RESPONSES_4XX = "http.status.4xx"; static final String RESPONSES_5XX = "http.status.5xx"; - @Deprecated(forRemoval = true, since = "7") static final String RESPONSES_401 = "http.status.401"; - @Deprecated(forRemoval = true, since = "7") static final String RESPONSES_403 = "http.status.403"; static final String STARTED_MILLIS = "serverStartedMillis"; diff --git a/container-core/src/main/java/com/yahoo/processing/Response.java b/container-core/src/main/java/com/yahoo/processing/Response.java index cf54d043c5f..59533900a0c 100644 --- a/container-core/src/main/java/com/yahoo/processing/Response.java +++ b/container-core/src/main/java/com/yahoo/processing/Response.java @@ -1,9 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.processing; -import com.google.common.util.concurrent.ListenableFuture; import com.yahoo.component.provider.ListenableFreezableClass; -import com.yahoo.concurrent.CompletableFutures; import com.yahoo.concurrent.SystemTimer; import com.yahoo.processing.execution.ResponseReceiver; import com.yahoo.processing.impl.ProcessingFuture; @@ -102,13 +100,6 @@ public class Response extends ListenableFreezableClass { return new CompleteAllOnGetFuture<D>(futures); } - /** @deprecated Use {@link #recursiveFuture(DataList)} instead */ - @Deprecated(forRemoval = true, since = "7") - @SuppressWarnings("removal") - public static <D extends Data> ListenableFuture<DataList<D>> recursiveComplete(DataList<D> rootDataList) { - return CompletableFutures.toGuavaListenableFuture(recursiveFuture(rootDataList)); - } - @SuppressWarnings("unchecked") private static <D extends Data> void collectCompletionFutures(DataList<D> dataList, List<CompletableFuture<DataList<D>>> futures) { futures.add(dataList.completeFuture()); diff --git a/container-core/src/main/java/com/yahoo/processing/impl/ProcessingFuture.java b/container-core/src/main/java/com/yahoo/processing/impl/ProcessingFuture.java index ab597fffaff..07923de6e09 100644 --- a/container-core/src/main/java/com/yahoo/processing/impl/ProcessingFuture.java +++ b/container-core/src/main/java/com/yahoo/processing/impl/ProcessingFuture.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.processing.impl; -import com.google.common.util.concurrent.ListenableFuture; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -14,8 +13,7 @@ import java.util.concurrent.TimeoutException; * * @author bjorncs */ -// TODO Vespa 8 remove ListenableFuture implementation -public abstract class ProcessingFuture<V> extends CompletableFuture<V> implements ListenableFuture<V> { +public abstract class ProcessingFuture<V> extends CompletableFuture<V> { @Override public boolean cancel(boolean mayInterruptIfRunning) { return false; } @Override public boolean isCancelled() { return false; } @@ -23,7 +21,6 @@ public abstract class ProcessingFuture<V> extends CompletableFuture<V> implement @Override public abstract V get() throws InterruptedException, ExecutionException; @Override public abstract V get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException; - @Override public void addListener(Runnable listener, Executor executor) { whenCompleteAsync((__, ___) -> listener.run(), executor); } diff --git a/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java b/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java index bb5fe7a1f76..aa753beb6a7 100644 --- a/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java +++ b/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.processing.rendering; -import com.google.common.util.concurrent.ListenableFuture; -import com.yahoo.concurrent.CompletableFutures; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; @@ -223,15 +221,6 @@ public abstract class AsynchronousSectionedRenderer<RESPONSE extends Response> e return startRender(stream, response, execution, request); } - - /** @deprecated Use {@link #renderResponseBeforeHandover(OutputStream, Response, Execution, Request)} */ - @Deprecated(forRemoval = true, since = "7") - @SuppressWarnings("removal") - public final ListenableFuture<Boolean> renderBeforeHandover(OutputStream stream, RESPONSE response, - Execution execution, Request request) { - return CompletableFutures.toGuavaListenableFuture(renderResponseBeforeHandover(stream, response, execution, request)); - } - private CompletableFuture<Boolean> startRender(OutputStream stream, RESPONSE response, Execution execution, Request request) { this.response = response; diff --git a/container-core/src/main/java/com/yahoo/processing/rendering/Renderer.java b/container-core/src/main/java/com/yahoo/processing/rendering/Renderer.java index 8db4ed4f624..df53ac846f2 100644 --- a/container-core/src/main/java/com/yahoo/processing/rendering/Renderer.java +++ b/container-core/src/main/java/com/yahoo/processing/rendering/Renderer.java @@ -1,9 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.processing.rendering; -import com.google.common.util.concurrent.ListenableFuture; import com.yahoo.component.AbstractComponent; -import com.yahoo.concurrent.CompletableFutures; import com.yahoo.processing.Request; import com.yahoo.processing.Response; import com.yahoo.processing.execution.Execution; @@ -43,17 +41,6 @@ public abstract class Renderer<RESPONSE extends Response> extends AbstractCompon } /** - * @deprecated Use/implement {@link #renderResponse(OutputStream, Response, Execution, Request)} instead. - * Return type changed from {@link ListenableFuture} to {@link CompletableFuture}. - */ - @Deprecated(forRemoval = true, since = "7") - @SuppressWarnings("removal") - public ListenableFuture<Boolean> render(OutputStream stream, RESPONSE response, Execution execution, - Request request) { - return CompletableFutures.toGuavaListenableFuture(renderResponse(stream, response, execution, request)); - } - - /** * Render a response to a stream. The stream also exposes a ByteBuffer API * for efficient transactions to JDisc. The returned future will throw the * exception causing failure wrapped in an ExecutionException if rendering @@ -65,11 +52,8 @@ public abstract class Renderer<RESPONSE extends Response> extends AbstractCompon * @param request the request matching the response * @return a {@link CompletableFuture} containing a boolean where true indicates a successful rendering */ - @SuppressWarnings("removal") - public CompletableFuture<Boolean> renderResponse(OutputStream stream, RESPONSE response, - Execution execution, Request request) { - return CompletableFutures.toCompletableFuture(render(stream, response, execution, request)); - } + public abstract CompletableFuture<Boolean> renderResponse(OutputStream stream, RESPONSE response, + Execution execution, Request request); /** * Name of the output encoding, if applicable. diff --git a/container-core/src/main/java/com/yahoo/processing/response/AbstractDataList.java b/container-core/src/main/java/com/yahoo/processing/response/AbstractDataList.java index b1ce0643487..15b1abc10cd 100644 --- a/container-core/src/main/java/com/yahoo/processing/response/AbstractDataList.java +++ b/container-core/src/main/java/com/yahoo/processing/response/AbstractDataList.java @@ -1,9 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.processing.response; -import com.google.common.util.concurrent.ListenableFuture; import com.yahoo.component.provider.ListenableFreezableClass; -import com.yahoo.concurrent.CompletableFutures; import com.yahoo.processing.Request; import com.yahoo.processing.impl.ProcessingFuture; @@ -92,13 +90,6 @@ public abstract class AbstractDataList<DATATYPE extends Data> extends Listenable return incomingData; } - @Override - @SuppressWarnings("removal") - @Deprecated(forRemoval = true, since = "7") - public ListenableFuture<DataList<DATATYPE>> complete() { - return CompletableFutures.toGuavaListenableFuture(completedFuture); - } - @Override public CompletableFuture<DataList<DATATYPE>> completeFuture() { return completedFuture; } @Override @@ -108,7 +99,7 @@ public abstract class AbstractDataList<DATATYPE extends Data> extends Listenable public boolean isStreamed() { return streamed; } public String toString() { - return super.toString() + (complete().isDone() ? " [completed]" : " [incomplete, " + incoming() + "]"); + return super.toString() + (completeFuture().isDone() ? " [completed]" : " [incomplete, " + incoming() + "]"); } public static final class DrainOnGetFuture<DATATYPE extends Data> extends ProcessingFuture<DataList<DATATYPE>> { diff --git a/container-core/src/main/java/com/yahoo/processing/response/DataList.java b/container-core/src/main/java/com/yahoo/processing/response/DataList.java index dbda8983f12..8c63dd2f7fe 100644 --- a/container-core/src/main/java/com/yahoo/processing/response/DataList.java +++ b/container-core/src/main/java/com/yahoo/processing/response/DataList.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.processing.response; -import com.google.common.util.concurrent.ListenableFuture; - import java.util.List; import java.util.concurrent.CompletableFuture; @@ -74,10 +72,6 @@ public interface DataList<DATATYPE extends Data> extends Data { */ CompletableFuture<DataList<DATATYPE>> completeFuture(); - /** @deprecated Use {@link #completeFuture()} instead */ - @Deprecated(forRemoval = true, since = "7") - ListenableFuture<DataList<DATATYPE>> complete(); - /** * Adds a listener which is invoked every time data is added to this list. * The listener is always invoked on the same thread which is adding the data, diff --git a/container-core/src/main/java/com/yahoo/processing/response/DefaultIncomingData.java b/container-core/src/main/java/com/yahoo/processing/response/DefaultIncomingData.java index 813d6ac54d8..eab033f8bc9 100644 --- a/container-core/src/main/java/com/yahoo/processing/response/DefaultIncomingData.java +++ b/container-core/src/main/java/com/yahoo/processing/response/DefaultIncomingData.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.processing.response; -import com.google.common.util.concurrent.ListenableFuture; import com.yahoo.collections.Tuple2; import com.yahoo.concurrent.CompletableFutures; @@ -50,13 +49,6 @@ public class DefaultIncomingData<DATATYPE extends Data> implements IncomingData< return owner; } - @Override - @Deprecated(forRemoval = true, since = "7") - @SuppressWarnings("removal") - public ListenableFuture<DataList<DATATYPE>> completed() { - return CompletableFutures.toGuavaListenableFuture(completionFuture); - } - @Override public CompletableFuture<DataList<DATATYPE>> completedFuture() { return completionFuture; } /** Returns whether the data in this is complete */ diff --git a/container-core/src/main/java/com/yahoo/processing/response/IncomingData.java b/container-core/src/main/java/com/yahoo/processing/response/IncomingData.java index 54ba0fa8031..fc0e6d21e13 100644 --- a/container-core/src/main/java/com/yahoo/processing/response/IncomingData.java +++ b/container-core/src/main/java/com/yahoo/processing/response/IncomingData.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.processing.response; -import com.google.common.util.concurrent.ListenableFuture; -import com.yahoo.concurrent.CompletableFutures; import com.yahoo.processing.impl.ProcessingFuture; import java.util.Collections; @@ -39,10 +37,6 @@ public interface IncomingData<DATATYPE extends Data> { */ CompletableFuture<DataList<DATATYPE>> completedFuture(); - /** @deprecated Use {@link #completedFuture()} instead */ - @Deprecated(forRemoval = true, since = "7") - ListenableFuture<DataList<DATATYPE>> completed(); - /** * Returns whether this is complete */ @@ -114,13 +108,6 @@ public interface IncomingData<DATATYPE extends Data> { completionFuture = new ImmediateFuture<>(owner); } - @Override - @SuppressWarnings("removal") - @Deprecated(forRemoval = true, since = "7") - public ListenableFuture<DataList<DATATYPE>> completed() { - return CompletableFutures.toGuavaListenableFuture(completionFuture); - } - @Override public CompletableFuture<DataList<DATATYPE>> completedFuture() { return completionFuture; } @Override diff --git a/container-core/src/main/java/org/json/package-info.java b/container-core/src/main/java/org/json/package-info.java deleted file mode 100644 index bacf02a3adc..00000000000 --- a/container-core/src/main/java/org/json/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package org.json; -// TODO Vespa 8: stop providing org.json -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/resources/configdefinitions/container.core.vip-status.def b/container-core/src/main/resources/configdefinitions/container.core.vip-status.def index 61abda33302..44165bf14c3 100644 --- a/container-core/src/main/resources/configdefinitions/container.core.vip-status.def +++ b/container-core/src/main/resources/configdefinitions/container.core.vip-status.def @@ -8,5 +8,5 @@ accessdisk bool default=false ## If the path is relative vespa home is prepended statusfile string default="share/qrsdocs/status.html" -## Not used TODO: Remove on Vespa 8 +## Whether this container should be in rotation by default on startup initiallyInRotation bool default=true diff --git a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def index b4bd624c22f..ca02c245015 100644 --- a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def +++ b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def @@ -28,10 +28,6 @@ reuseAddress bool default=true # The maximum idle time for a connection, which roughly translates to the Socket.setSoTimeout(int). idleTimeout double default=180.0 -# DEPRECATED - Ignored, no longer in use -stopTimeout double default = 30.0 -# TODO Vespa 8 Remove stop timeout - # Whether or not to have socket keep alive turned on. tcpKeepAliveEnabled bool default=false diff --git a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def index aed92b1459f..f34fd523207 100644 --- a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def +++ b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.server.def @@ -7,17 +7,6 @@ developerMode bool default=false # The gzip compression level to use, if compression is enabled in a request. responseCompressionLevel int default=6 -# DEPRECATED - Ignored, no longer in use. -httpKeepAliveEnabled bool default=true -# TODO Vespa 8 Remove httpKeepAliveEnabled - -# Maximum number of request per http connection before server will hangup. -# Naming taken from apache http server. -# 0 means never hangup. -# DEPRECATED - Ignored, no longer in use. Use similar parameter in connector config instead. -maxKeepAliveRequests int default=0 -# TODO Vespa 8 Remove maxKeepAliveRequests - # Whether the request body of POSTed forms should be removed (form parameters are available as request parameters). removeRawPostBodyForWwwUrlEncodedPost bool default=false diff --git a/container-core/src/main/resources/configdefinitions/metrics.metrics-presentation.def b/container-core/src/main/resources/configdefinitions/metrics.metrics-presentation.def deleted file mode 100644 index 2e537a00342..00000000000 --- a/container-core/src/main/resources/configdefinitions/metrics.metrics-presentation.def +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -namespace=metrics - -## TODO Vespa 8: remove -slidingwindow bool default=true diff --git a/container-core/src/test/java/com/yahoo/container/di/DirConfigSource.java b/container-core/src/test/java/com/yahoo/container/di/DirConfigSource.java index 34119163d2f..d459070e2bd 100644 --- a/container-core/src/test/java/com/yahoo/container/di/DirConfigSource.java +++ b/container-core/src/test/java/com/yahoo/container/di/DirConfigSource.java @@ -16,7 +16,6 @@ import java.util.Random; * @author gjoranv * @author ollivir */ -@SuppressWarnings("removal") // TODO Vespa 8: remove public class DirConfigSource { private final TemporaryFolder tempFolder = createTemporaryFolder(); public final ConfigSource configSource; diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java index 58263548085..b87a1179832 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java @@ -9,7 +9,6 @@ import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.container.jdisc.RequestHandlerTestDriver; import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.jdisc.Timer; -import com.yahoo.metrics.MetricsPresentationConfig; import org.junit.Before; import org.junit.BeforeClass; diff --git a/container-core/src/test/java/com/yahoo/container/xml/providers/XMLProviderTest.java b/container-core/src/test/java/com/yahoo/container/xml/providers/XMLProviderTest.java deleted file mode 100644 index e9983341507..00000000000 --- a/container-core/src/test/java/com/yahoo/container/xml/providers/XMLProviderTest.java +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.xml.providers; - -import org.junit.Test; - -import javax.xml.datatype.DatatypeFactory; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.SAXParserFactory; -import javax.xml.stream.XMLEventFactory; -import javax.xml.stream.XMLInputFactory; -import javax.xml.stream.XMLOutputFactory; -import javax.xml.transform.TransformerFactory; -import javax.xml.validation.SchemaFactory; -import javax.xml.xpath.XPathFactory; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.junit.Assert.assertThat; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.29 - */ -@SuppressWarnings("deprecation") -public class XMLProviderTest { - - @Test - public void testInstantiationAndDestruction() { - { - DatatypeFactoryProvider provider = new DatatypeFactoryProvider(); - DatatypeFactory factory = provider.get(); - assertThat(factory.getClass().getName(), equalTo(DatatypeFactoryProvider.FACTORY_CLASS)); - provider.deconstruct(); - } - { - DocumentBuilderFactoryProvider provider = new DocumentBuilderFactoryProvider(); - DocumentBuilderFactory factory = provider.get(); - assertThat(factory.getClass().getName(), equalTo(DocumentBuilderFactoryProvider.FACTORY_CLASS)); - provider.deconstruct(); - } - { - SAXParserFactoryProvider provider = new SAXParserFactoryProvider(); - SAXParserFactory factory = provider.get(); - assertThat(factory.getClass().getName(), equalTo(SAXParserFactoryProvider.FACTORY_CLASS)); - provider.deconstruct(); - } - { - SchemaFactoryProvider provider = new SchemaFactoryProvider(); - SchemaFactory factory = provider.get(); - assertThat(factory.getClass().getName(), equalTo(SchemaFactoryProvider.FACTORY_CLASS)); - provider.deconstruct(); - } - { - TransformerFactoryProvider provider = new TransformerFactoryProvider(); - TransformerFactory factory = provider.get(); - assertThat(factory.getClass().getName(), equalTo(TransformerFactoryProvider.FACTORY_CLASS)); - provider.deconstruct(); - } - { - XMLEventFactoryProvider provider = new XMLEventFactoryProvider(); - XMLEventFactory factory = provider.get(); - assertThat(factory.getClass().getName(), equalTo(XMLEventFactoryProvider.FACTORY_CLASS)); - provider.deconstruct(); - } - { - XMLInputFactoryProvider provider = new XMLInputFactoryProvider(); - XMLInputFactory factory = provider.get(); - assertThat(factory.getClass().getName(), equalTo(XMLInputFactoryProvider.FACTORY_CLASS)); - provider.deconstruct(); - } - { - XMLOutputFactoryProvider provider = new XMLOutputFactoryProvider(); - XMLOutputFactory factory = provider.get(); - assertThat(factory.getClass().getName(), equalTo(XMLOutputFactoryProvider.FACTORY_CLASS)); - provider.deconstruct(); - } - { - XPathFactoryProvider provider = new XPathFactoryProvider(); - XPathFactory factory = provider.get(); - assertThat(factory.getClass().getName(), equalTo(XPathFactoryProvider.FACTORY_CLASS)); - provider.deconstruct(); - } - } - -} diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/HttpRequestTestCase.java b/container-core/src/test/java/com/yahoo/jdisc/http/HttpRequestTestCase.java index fd0cfe5718c..7f9e1e8bd64 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/HttpRequestTestCase.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/HttpRequestTestCase.java @@ -73,15 +73,12 @@ public class HttpRequestTestCase { } @Test - @SuppressWarnings("deprecation") public void requireThatAccessorsWork() { URI uri = URI.create("http://localhost/path?foo=bar&foo=baz&cox=69"); InetSocketAddress address = new InetSocketAddress("remotehost", 69); final HttpRequest request = HttpRequest.newServerRequest(mockContainer(), uri, HttpRequest.Method.GET, HttpRequest.Version.HTTP_1_1, address, 1L); assertEquals(uri, request.getUri()); - request.setUri(uri = URI.create("http://remotehost/")); - assertEquals(uri, request.getUri()); assertEquals(HttpRequest.Method.GET, request.getMethod()); request.setMethod(HttpRequest.Method.CONNECT); diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/filter/DiscFilterRequestTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/filter/DiscFilterRequestTest.java index 25efe4ac5f6..294e19aa2a1 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/filter/DiscFilterRequestTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/filter/DiscFilterRequestTest.java @@ -262,41 +262,36 @@ public class DiscFilterRequestTest { } @Test - @SuppressWarnings("deprecation") - public void testSetScheme() { - URI uri = URI.create("https://example.yahoo.com:8080/test"); - HttpRequest httpReq = newRequest(uri, HttpRequest.Method.GET, HttpRequest.Version.HTTP_1_1); - DiscFilterRequest request = new DiscFilterRequest(httpReq); - - request.setScheme("http", true); - System.out.println(request.getUri().toString()); - Assert.assertEquals(request.getUri().toString(), "http://example.yahoo.com:8080/test"); - } - - @Test - @SuppressWarnings("deprecation") public void testGetServerPort() { - URI uri = URI.create("http://example.yahoo.com/test"); - HttpRequest httpReq = newRequest(uri, HttpRequest.Method.GET, HttpRequest.Version.HTTP_1_1); - DiscFilterRequest request = new DiscFilterRequest(httpReq); - Assert.assertEquals(request.getServerPort(), 80); - - request.setUri(URI.create("https://example.yahoo.com/test")); - Assert.assertEquals(request.getServerPort(), 443); - + { + URI uri = URI.create("http://example.yahoo.com/test"); + HttpRequest httpReq = newRequest(uri, HttpRequest.Method.GET, HttpRequest.Version.HTTP_1_1); + DiscFilterRequest request = new DiscFilterRequest(httpReq); + Assert.assertEquals(request.getServerPort(), 80); + + } + { + URI uri = URI.create("https://example.yahoo.com/test"); + HttpRequest httpReq = newRequest(uri, HttpRequest.Method.GET, HttpRequest.Version.HTTP_1_1); + DiscFilterRequest request = new DiscFilterRequest(httpReq); + Assert.assertEquals(request.getServerPort(), 443); + } } @Test - @SuppressWarnings("deprecation") public void testIsSecure() { - URI uri = URI.create("http://example.yahoo.com/test"); - HttpRequest httpReq = newRequest(uri, HttpRequest.Method.GET, HttpRequest.Version.HTTP_1_1); - DiscFilterRequest request = new DiscFilterRequest(httpReq); - Assert.assertFalse(request.isSecure()); - - request.setUri(URI.create("https://example.yahoo.com/test")); - Assert.assertTrue(request.isSecure()); - + { + URI uri = URI.create("http://example.yahoo.com/test"); + HttpRequest httpReq = newRequest(uri, HttpRequest.Method.GET, HttpRequest.Version.HTTP_1_1); + DiscFilterRequest request = new DiscFilterRequest(httpReq); + Assert.assertFalse(request.isSecure()); + } + { + URI uri = URI.create("https://example.yahoo.com/test"); + HttpRequest httpReq = newRequest(uri, HttpRequest.Method.GET, HttpRequest.Version.HTTP_1_1); + DiscFilterRequest request = new DiscFilterRequest(httpReq); + Assert.assertTrue(request.isSecure()); + } } @Test diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/filter/DiscFilterResponseTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/filter/DiscFilterResponseTest.java index bbb81ae3308..ace41473eb4 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/filter/DiscFilterResponseTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/filter/DiscFilterResponseTest.java @@ -101,14 +101,4 @@ public class DiscFilterResponseTest { Assert.assertEquals(response.getHeader("name"), "value"); } - @Test - @SuppressWarnings("removal") - public void testGetParentResponse() { - URI uri = URI.create("http://example.com/test"); - HttpRequest httpReq = newRequest(uri, HttpRequest.Method.GET, HttpRequest.Version.HTTP_1_1); - HttpResponse httpResp = newResponse(httpReq, 200); - DiscFilterResponse response = new DiscFilterResponse(httpResp); - Assert.assertSame(response.getParentResponse(), httpResp); - } - } diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollectorTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollectorTest.java index 231a9f1384a..89908e6d56b 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollectorTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpResponseStatisticsCollectorTest.java @@ -86,8 +86,6 @@ public class HttpResponseStatisticsCollectorTest { assertStatisticsEntry(stats, "http", "GET", MetricDefinitions.RESPONSES_4XX, "read", 401, 1L); assertStatisticsEntry(stats, "http", "GET", MetricDefinitions.RESPONSES_4XX, "read", 403, 1L); assertStatisticsEntry(stats, "http", "GET", MetricDefinitions.RESPONSES_4XX, "read", 404, 1L); - assertStatisticsEntry(stats, "http", "GET", MetricDefinitions.RESPONSES_401, "read", 401, 1L); - assertStatisticsEntry(stats, "http", "GET", MetricDefinitions.RESPONSES_403, "read", 403, 1L); } diff --git a/container-core/src/test/java/com/yahoo/processing/ResponseTestCase.java b/container-core/src/test/java/com/yahoo/processing/ResponseTestCase.java index efcf608b6f0..f8cd646909d 100644 --- a/container-core/src/test/java/com/yahoo/processing/ResponseTestCase.java +++ b/container-core/src/test/java/com/yahoo/processing/ResponseTestCase.java @@ -22,7 +22,7 @@ public class ResponseTestCase { * Check the recursive toString printing along the way. * List variable names ends by numbers specifying the index of the list at each level. */ - @SuppressWarnings({"unchecked", "removal"}) + @SuppressWarnings({"unchecked"}) @Test public void testRecursiveCompletionAndToString() throws InterruptedException, ExecutionException { // create lists @@ -68,7 +68,7 @@ public class ResponseTestCase { assertEqualsIgnoreObjectNumbers("Uncompleted tree, incoming complete", uncompletedTreeCompletedIncoming, Responses.recursiveToString(list1)); // complete all - Response.recursiveComplete(list1).get(); + Response.recursiveFuture(list1).get(); assertEqualsIgnoreObjectNumbers("Completed tree", completedTree, Responses.recursiveToString(list1)); } diff --git a/container-core/src/test/java/com/yahoo/processing/execution/test/FutureDataTestCase.java b/container-core/src/test/java/com/yahoo/processing/execution/test/FutureDataTestCase.java index 2fb32271419..7dd043383a1 100644 --- a/container-core/src/test/java/com/yahoo/processing/execution/test/FutureDataTestCase.java +++ b/container-core/src/test/java/com/yahoo/processing/execution/test/FutureDataTestCase.java @@ -25,7 +25,7 @@ import static org.junit.Assert.assertEquals; public class FutureDataTestCase { /** Run a chain which ends in a processor which returns a response containing future data. */ - @SuppressWarnings({"unchecked", "removal"}) + @SuppressWarnings({"unchecked"}) @Test public void testFutureDataPassThrough() throws InterruptedException, ExecutionException, TimeoutException { // Set up @@ -45,14 +45,14 @@ public class FutureDataTestCase { futureDataSource.incomingData.get(0).add(new StringData(request, "d1")); futureDataSource.incomingData.get(0).addLast(new StringData(request, "d2")); assertEquals("New data is not visible because we haven't asked for it", 1, response.data().asList().size()); - response.data().complete().get(1000, TimeUnit.MILLISECONDS); + response.data().completeFuture().get(1000, TimeUnit.MILLISECONDS); assertEquals("Now the data is available", 3, response.data().asList().size()); assertEquals("d1",response.data().get(1).toString().toString()); assertEquals("d2",response.data().get(2).toString().toString()); } /** Federate to one source which returns data immediately and one who return future data */ - @SuppressWarnings({"unchecked", "removal"}) + @SuppressWarnings({"unchecked"}) @Test public void testFederateSyncAndAsyncData() throws InterruptedException, ExecutionException, TimeoutException { // Set up @@ -81,7 +81,7 @@ public class FutureDataTestCase { futureDataSource.incomingData.get(0).add(new StringData(request, "d1")); futureDataSource.incomingData.get(0).addLast(new StringData(request, "d2")); assertEquals("New data is not visible because we haven't asked for it", 0, asyncData.asList().size()); - asyncData.complete().get(1000, TimeUnit.MILLISECONDS); + asyncData.completeFuture().get(1000, TimeUnit.MILLISECONDS); assertEquals("Now the data is available", 2, asyncData.asList().size()); assertEquals("d1",asyncData.get(0).toString().toString()); assertEquals("d2", asyncData.get(1).toString().toString()); @@ -108,7 +108,7 @@ public class FutureDataTestCase { assertEquals("New data is not visible because it is not complete", 0, response.data().asList().size()); futureDataSource.incomingData.get(0).addLast(new StringData(request, "d2")); assertEquals("Not visible because it has not been synced yet", 0, response.data().asList().size()); - response.data().complete().get(1000, TimeUnit.MILLISECONDS); + response.data().completeFuture().get(1000, TimeUnit.MILLISECONDS); assertEquals("Now the data as well as the count is available", 3, response.data().asList().size()); assertEquals("d1",response.data().get(0).toString().toString()); assertEquals("d2",response.data().get(1).toString().toString()); @@ -120,7 +120,7 @@ public class FutureDataTestCase { * When the first of the futures are done one additional chain is to be run. * When both are done another chain is to be run. */ - @SuppressWarnings({"unchecked", "removal"}) + @SuppressWarnings({"unchecked"}) @Test public void testAsyncDataProcessingOfFederatedResult() throws InterruptedException, ExecutionException, TimeoutException { // Set up @@ -154,7 +154,7 @@ public class FutureDataTestCase { // complete async data in source1 futureSource1.incomingData.get(0).addLast(new StringData(request,"source1Data")); assertEquals("Not visible yet", 0, source1Data.asList().size()); - source1Data.complete().get(1000, TimeUnit.MILLISECONDS); + source1Data.completeFuture().get(1000, TimeUnit.MILLISECONDS); assertEquals(2, source1Data.asList().size()); assertEquals("source1Data",source1Data.get(0).toString()); assertEquals("Completion listener chain on this has run", "[source1] Data count: 1", source1Data.get(1).toString()); @@ -164,7 +164,7 @@ public class FutureDataTestCase { futureSource2.incomingData.get(0).addLast(new StringData(request, "source2Data")); assertEquals("Main completion listener has not run", 3, response.data().asList().size()); - Response.recursiveComplete(response.data()).get(); + Response.recursiveFuture(response.data()).get(); assertEquals("Main completion listener has run", 4, response.data().asList().size()); assertEquals("The main data counter saw all sync data, but not source2 data as it executes after this", "[main] Data count: " + (2 + 0 + 3), response.data().get(3).toString()); diff --git a/container-core/src/test/java/com/yahoo/processing/execution/test/StreamingTestCase.java b/container-core/src/test/java/com/yahoo/processing/execution/test/StreamingTestCase.java index bd1307ff77c..d494e774f8e 100644 --- a/container-core/src/test/java/com/yahoo/processing/execution/test/StreamingTestCase.java +++ b/container-core/src/test/java/com/yahoo/processing/execution/test/StreamingTestCase.java @@ -26,7 +26,7 @@ import static org.junit.Assert.assertEquals; public class StreamingTestCase { /** Tests adding a chain which is called every time new data is added to a data list */ - @SuppressWarnings({"unchecked", "removal"}) + @SuppressWarnings({"unchecked"}) @Test public void testStreamingData() throws InterruptedException, ExecutionException, TimeoutException { // Set up @@ -66,7 +66,7 @@ public class StreamingTestCase { assertEquals("We are getting data add events also the last time", 4, streamProcessor.invocationCount); assertEquals("New data is consumed", 4, response.data().asList().size()); - response.data().complete().get(1000, TimeUnit.MILLISECONDS); // no-op here + response.data().completeFuture().get(1000, TimeUnit.MILLISECONDS); // no-op here assertEquals("d1",response.data().get(1).toString().toString()); assertEquals("d2",response.data().get(2).toString().toString()); assertEquals("d3",response.data().get(3).toString().toString()); diff --git a/container-core/src/test/java/com/yahoo/processing/rendering/AsynchronousSectionedRendererTest.java b/container-core/src/test/java/com/yahoo/processing/rendering/AsynchronousSectionedRendererTest.java index a6db21d5094..5dfe29dd4a1 100644 --- a/container-core/src/test/java/com/yahoo/processing/rendering/AsynchronousSectionedRendererTest.java +++ b/container-core/src/test/java/com/yahoo/processing/rendering/AsynchronousSectionedRendererTest.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.processing.rendering; -import com.google.common.util.concurrent.ListenableFuture; import com.yahoo.component.provider.ListenableFreezableClass; import com.yahoo.container.jdisc.ContentChannelOutputStream; import com.yahoo.processing.Processor; @@ -23,8 +22,6 @@ import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotSame; @@ -403,41 +400,6 @@ public class AsynchronousSectionedRendererTest { } @Override - @SuppressWarnings("removal") - public ListenableFuture<DataList<StringData>> complete() { - return new ListenableFuture<>() { - @Override - public void addListener(Runnable runnable, Executor executor) { - } - - @Override - public boolean cancel(boolean b) { - return false; - } - - @Override - public boolean isCancelled() { - return false; - } - - @Override - public boolean isDone() { - return true; - } - - @Override - public DataList<StringData> get() { - return StringDataList.this; - } - - @Override - public DataList<StringData> get(long l, TimeUnit timeUnit) { - return StringDataList.this; - } - }; - } - - @Override public CompletableFuture<DataList<StringData>> completeFuture() { return CompletableFuture.completedFuture(this); } diff --git a/container-core/src/test/java/com/yahoo/processing/test/documentation/AsyncDataProcessingInitiator.java b/container-core/src/test/java/com/yahoo/processing/test/documentation/AsyncDataProcessingInitiator.java index 21731f7d714..a2a028772e5 100644 --- a/container-core/src/test/java/com/yahoo/processing/test/documentation/AsyncDataProcessingInitiator.java +++ b/container-core/src/test/java/com/yahoo/processing/test/documentation/AsyncDataProcessingInitiator.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.processing.test.documentation; -import com.google.common.util.concurrent.MoreExecutors; import com.yahoo.component.chain.Chain; import com.yahoo.processing.Processor; import com.yahoo.processing.Request; @@ -22,13 +21,11 @@ public class AsyncDataProcessingInitiator extends Processor { this.asyncChain=asyncChain; } - @SuppressWarnings({"removal"}) @Override public Response process(Request request, Execution execution) { Response response=execution.process(request); - response.data().complete().addListener(new RunnableExecution(request, - new ExecutionWithResponse(asyncChain, response, execution)), - MoreExecutors.directExecutor()); + response.data().completeFuture().whenComplete((__, ___) -> new RunnableExecution(request, + new ExecutionWithResponse(asyncChain, response, execution)).run()); return response; } diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml index 211d0019774..bbd816d349d 100644 --- a/container-dependencies-enforcer/pom.xml +++ b/container-dependencies-enforcer/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-dependencies-enforcer</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>pom</packaging> <dependencies> @@ -65,49 +65,21 @@ <include>com.fasterxml.jackson.datatype:jackson-datatype-jdk8:[${jackson2.version}]:jar:provided</include> <include>com.fasterxml.jackson.datatype:jackson-datatype-jsr310:[${jackson2.version}]:jar:provided</include> - - <!-- Use version range for jax deps, because jersey and junit affect the versions. --> - <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:[2.5.4, ${jackson2.version}]:jar:provided</include> - <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:[2.5.4, ${jackson2.version}]:jar:provided</include> - <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations:[2.5.4, ${jackson2.version}]:jar:provided</include> - - <include>com.google.code.findbugs:jsr305:[${findbugs.version}]:jar:provided</include> + <!-- Guava with its internal dependencies --> <include>com.google.guava:guava:[${guava.version}]:jar:provided</include> - <include>com.google.inject.extensions:guice-assistedinject:[${guice.version}]:jar:provided</include> - <include>com.google.inject.extensions:guice-multibindings:[${guice.version}]:jar:provided</include> + <include>com.google.errorprone:error_prone_annotations:[2.2.0]:jar:provided</include> + <include>com.google.guava:failureaccess:[1.0.1]:jar:provided</include> + <include>com.google.j2objc:j2objc-annotations:[1.1]:jar:provided</include> + <include>com.google.inject:guice:[${guice.version}]:jar:provided:no_aop</include> <include>com.sun.activation:javax.activation:[1.2.0]:jar:provided</include> <include>com.sun.xml.bind:jaxb-core:[${jaxb.version}]:jar:provided</include> <include>com.sun.xml.bind:jaxb-impl:[${jaxb.version}]:jar:provided</include> <include>commons-logging:commons-logging:[1.2]:jar:provided</include> - <include>javax.annotation:javax.annotation-api:[${javax.annotation-api.version}]:jar:provided</include> <include>javax.inject:javax.inject:[${javax.inject.version}]:jar:provided</include> <include>javax.servlet:javax.servlet-api:[${javax.servlet-api.version}]:jar:provided</include> - <include>javax.validation:validation-api:[${javax.validation-api.version}]:jar:provided</include> <include>javax.ws.rs:javax.ws.rs-api:[${javax.ws.rs-api.version}]:jar:provided</include> <include>javax.xml.bind:jaxb-api:[${jaxb.version}]:jar:provided</include> - <include>net.jcip:jcip-annotations:[1.0]:jar:provided</include> - <include>org.lz4:lz4-java:[${org.lz4.version}]:jar:provided</include> - <include>org.apache.felix:org.apache.felix.framework:[${felix.version}]:jar:provided</include> - <include>org.apache.felix:org.apache.felix.log:[${felix.log.version}]:jar:provided</include> - <include>org.apache.felix:org.apache.felix.main:[${felix.version}]:jar:provided</include> - <include>org.bouncycastle:bcpkix-jdk15on:[${bouncycastle.version}]:jar:provided</include> - <include>org.bouncycastle:bcprov-jdk15on:[${bouncycastle.version}]:jar:provided</include> - <include>org.eclipse.jetty:jetty-http:[${jetty.version}]:jar:provided</include> - <include>org.eclipse.jetty:jetty-io:[${jetty.version}]:jar:provided</include> - <include>org.eclipse.jetty:jetty-util:[${jetty.version}]:jar:provided</include> - <include>org.glassfish.hk2.external:aopalliance-repackaged:[${hk2.version}]:jar:provided</include> - <include>org.glassfish.hk2.external:javax.inject:[${hk2.version}]:jar:provided</include> - <include>org.glassfish.hk2:hk2-api:[${hk2.version}]:jar:provided</include> - <include>org.glassfish.hk2:hk2-locator:[${hk2.version}]:jar:provided</include> - <include>org.glassfish.hk2:hk2-utils:[${hk2.version}]:jar:provided</include> - <include>org.glassfish.hk2:osgi-resource-locator:[${hk2.osgi-resource-locator.version}]:jar:provided</include> - <include>org.glassfish.jersey.bundles.repackaged:jersey-guava:[${jersey2.version}]:jar:provided</include> - <include>org.glassfish.jersey.core:jersey-client:[${jersey2.version}]:jar:provided</include> - <include>org.glassfish.jersey.core:jersey-common:[${jersey2.version}]:jar:provided</include> - <include>org.glassfish.jersey.core:jersey-server:[${jersey2.version}]:jar:provided</include> - <include>org.javassist:javassist:[${javassist.version}]:jar:provided</include> - <include>org.json:json:[${org.json.version}]:jar:provided</include> <include>org.slf4j:jcl-over-slf4j:[${slf4j.version}]:jar:provided</include> <include>org.slf4j:log4j-over-slf4j:[${slf4j.version}]:jar:provided</include> <include>org.slf4j:slf4j-api:[${slf4j.version}]:jar:provided</include> diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index 6b8fa8ffeb9..709f9dc333a 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -5,7 +5,7 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>container-dependency-versions</artifactId> <packaging>pom</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>container-dependency-versions</name> <description>Versions for all 3rd party dependencies provided from the Vespa container.</description> <url>https://github.com/vespa-engine</url> @@ -29,8 +29,6 @@ </scm> <dependencyManagement> - <!-- TODO Vespa 8: remove all artifacts that are no longer installed in jdisc. - Those used in configserver/controller must be moved to parent/pom.xml --> <dependencies> <dependency> <groupId>aopalliance</groupId> @@ -63,43 +61,11 @@ <version>${jackson2.version}</version> </dependency> <dependency> - <groupId>com.fasterxml.jackson.jaxrs</groupId> - <artifactId>jackson-jaxrs-base</artifactId> - <version>${jackson2.version}</version> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.jaxrs</groupId> - <artifactId>jackson-jaxrs-json-provider</artifactId> - <version>${jackson2.version}</version> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.module</groupId> - <artifactId>jackson-module-jaxb-annotations</artifactId> - <version>${jackson2.version}</version> - </dependency> - <dependency> - <!-- Not provided runtime, but necessary to build with guava in many cases. - Guava has an optional dep on this and uses the annotations in some classes. --> - <groupId>com.google.code.findbugs</groupId> - <artifactId>jsr305</artifactId> - <version>${findbugs.version}</version> - </dependency> - <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <version>${guava.version}</version> </dependency> <dependency> - <groupId>com.google.inject.extensions</groupId> - <artifactId>guice-assistedinject</artifactId> - <version>${guice.version}</version> - </dependency> - <dependency> - <groupId>com.google.inject.extensions</groupId> - <artifactId>guice-multibindings</artifactId> - <version>${guice.version}</version> - </dependency> - <dependency> <groupId>com.google.inject</groupId> <artifactId>guice</artifactId> <version>${guice.version}</version> @@ -117,26 +83,17 @@ <version>1.2</version> </dependency> <dependency> - <groupId>javax.annotation</groupId> - <artifactId>javax.annotation-api</artifactId> - <version>${javax.annotation-api.version}</version> - </dependency> - <dependency> <groupId>javax.inject</groupId> <artifactId>javax.inject</artifactId> <version>${javax.inject.version}</version> </dependency> <dependency> + <!-- TODO Vespa 9 Stop providing servlet-api 3.x --> <groupId>javax.servlet</groupId> <artifactId>javax.servlet-api</artifactId> <version>${javax.servlet-api.version}</version> </dependency> <dependency> - <groupId>javax.validation</groupId> - <artifactId>validation-api</artifactId> - <version>${javax.validation-api.version}</version> - </dependency> - <dependency> <groupId>javax.ws.rs</groupId> <artifactId>javax.ws.rs-api</artifactId> <version>${javax.ws.rs-api.version}</version> @@ -144,25 +101,27 @@ <!-- jaxb start --> <dependency> - <!-- Needed by felix --> + <!-- Exported from jdisc_core. Imported and directly used by e.g. vespa.factory --> + <!-- TODO Vespa 9: stop exporting/providing and move to parent? --> <groupId>javax.xml.bind</groupId> <artifactId>jaxb-api</artifactId> <version>${jaxb.version}</version> </dependency> <dependency> - <!-- Needed by jersey --> + <!-- TODO Vespa 9: stop exporting/providing and move to parent? --> <groupId>com.sun.xml.bind</groupId> <artifactId>jaxb-core</artifactId> <version>${jaxb.version}</version> </dependency> <dependency> - <!-- Needed by jersey --> + <!-- TODO Vespa 9: stop exporting/providing and move to parent? --> <groupId>com.sun.xml.bind</groupId> <artifactId>jaxb-impl</artifactId> <version>${jaxb.version}</version> </dependency> <dependency> - <!-- Needed by jersey --> + <!-- Needed by jaxb-api, and possibly guice --> + <!-- TODO Vespa 9: stop exporting/providing and move to parent? --> <groupId>com.sun.activation</groupId> <artifactId>javax.activation</artifactId> <version>1.2.0</version> @@ -170,116 +129,6 @@ <!-- jaxb end --> <dependency> - <groupId>net.jcip</groupId> - <artifactId>jcip-annotations</artifactId> - <version>1.0</version> - </dependency> - <dependency> - <groupId>org.lz4</groupId> - <artifactId>lz4-java</artifactId> - <version>${org.lz4.version}</version> - </dependency> - <dependency> - <groupId>org.apache.felix</groupId> - <artifactId>org.apache.felix.framework</artifactId> - <version>${felix.version}</version> - </dependency> - <dependency> - <groupId>org.apache.felix</groupId> - <artifactId>org.apache.felix.log</artifactId> - <version>${felix.log.version}</version> - </dependency> - <dependency> - <groupId>org.apache.felix</groupId> - <artifactId>org.apache.felix.main</artifactId> - <version>${felix.version}</version> - </dependency> - <dependency> - <groupId>org.bouncycastle</groupId> - <artifactId>bcpkix-jdk15on</artifactId> - <version>${bouncycastle.version}</version> - </dependency> - <dependency> - <groupId>org.bouncycastle</groupId> - <artifactId>bcprov-jdk15on</artifactId> - <version>${bouncycastle.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-http</artifactId> - <version>${jetty.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-io</artifactId> - <version>${jetty.version}</version> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-util</artifactId> - <version>${jetty.version}</version> - </dependency> - <dependency> - <groupId>org.glassfish.hk2.external</groupId> - <artifactId>aopalliance-repackaged</artifactId> - <version>${hk2.version}</version> - </dependency> - <dependency> - <groupId>org.glassfish.hk2.external</groupId> - <artifactId>javax.inject</artifactId> - <version>${hk2.version}</version> - </dependency> - <dependency> - <groupId>org.glassfish.hk2</groupId> - <artifactId>hk2-api</artifactId> - <version>${hk2.version}</version> - </dependency> - <dependency> - <groupId>org.glassfish.hk2</groupId> - <artifactId>hk2-locator</artifactId> - <version>${hk2.version}</version> - </dependency> - <dependency> - <groupId>org.glassfish.hk2</groupId> - <artifactId>hk2-utils</artifactId> - <version>${hk2.version}</version> - </dependency> - <dependency> - <groupId>org.glassfish.hk2</groupId> - <artifactId>osgi-resource-locator</artifactId> - <version>${hk2.osgi-resource-locator.version}</version> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.bundles.repackaged</groupId> - <artifactId>jersey-guava</artifactId> - <version>${jersey2.version}</version> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.core</groupId> - <artifactId>jersey-client</artifactId> - <version>${jersey2.version}</version> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.core</groupId> - <artifactId>jersey-common</artifactId> - <version>${jersey2.version}</version> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.core</groupId> - <artifactId>jersey-server</artifactId> - <version>${jersey2.version}</version> - </dependency> - <dependency> - <groupId>org.javassist</groupId> - <artifactId>javassist</artifactId> - <version>${javassist.version}</version> - </dependency> - <dependency> <!-- TODO Vespa 8: remove as provided dependency and move to parent --> - <groupId>org.json</groupId> - <artifactId>json</artifactId> - <version>${org.json.version}</version> - </dependency> - <dependency> <groupId>org.slf4j</groupId> <artifactId>jcl-over-slf4j</artifactId> <version>${slf4j.version}</version> @@ -385,36 +234,18 @@ <properties> <aopalliance.version>1.0</aopalliance.version> - <bouncycastle.version>1.68</bouncycastle.version> - <felix.version>7.0.1</felix.version> - <felix.log.version>1.0.1</felix.log.version> - <findbugs.version>1.3.9</findbugs.version> - <guava.version>20.0</guava.version> - <guice.version>3.0</guice.version> + <guava.version>27.1-jre</guava.version> + <guice.version>4.2.3</guice.version> + <jackson2.version>2.13.3</jackson2.version> + <jackson-databind.version>${jackson2.version}</jackson-databind.version> <javax.inject.version>1</javax.inject.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> + <javax.ws.rs-api.version>2.0.1</javax.ws.rs-api.version> <jaxb.version>2.3.0</jaxb.version> - <jetty.version>9.4.46.v20220331</jetty.version> - <jetty-alpn.version>1.1.3.v20160715</jetty-alpn.version> - <org.lz4.version>1.8.0</org.lz4.version> <org.json.version>20090211</org.json.version> <slf4j.version>1.7.32</slf4j.version> <!-- WARNING: when updated, also update c.y.v.tenant:base pom --> <xml-apis.version>1.4.01</xml-apis.version> - <!-- These must be kept in sync with version used by current jersey2.version. --> - <!-- MUST be updated each time jersey2 is upgraded! --> - <!-- Check versions in the properties section of the pom for org/glassfish/jersey/project/ --> - <!-- and then verify by doing: ' ls -l vespa/vespa_jersey2/target/dependency' --> - <hk2.version>2.5.0-b32</hk2.version> - <hk2.osgi-resource-locator.version>1.0.1</hk2.osgi-resource-locator.version> - <jackson2.version>2.13.3</jackson2.version> - <jackson-databind.version>${jackson2.version}</jackson-databind.version> - <javassist.version>3.20.0-GA</javassist.version> - <javax.annotation-api.version>1.2</javax.annotation-api.version> - <javax.validation-api.version>1.1.0.Final</javax.validation-api.version> - <javax.ws.rs-api.version>2.0.1</javax.ws.rs-api.version> - <jersey2.version>2.25</jersey2.version> - <!-- Not a dependency. Only included to allow the versions-maven-plugin to check for updates of itself --> <versions-maven-plugin.version>2.8.1</versions-maven-plugin.version> </properties> diff --git a/container-dev/pom.xml b/container-dev/pom.xml index 6268e1e6fb4..715340c4913 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -9,11 +9,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-dev</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>jar</packaging> <dependencies> <dependency> @@ -29,6 +29,14 @@ <version>${project.version}</version> <exclusions> <exclusion> + <groupId>org.apache.felix</groupId> + <artifactId>org.apache.felix.framework</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.felix</groupId> + <artifactId>org.apache.felix.log</artifactId> + </exclusion> + <exclusion> <groupId>org.jvnet.hudson</groupId> <artifactId>annotation-indexer</artifactId> </exclusion> @@ -53,10 +61,6 @@ <classifier>no_aop</classifier> </dependency> <dependency> - <groupId>org.apache.felix</groupId> - <artifactId>org.apache.felix.main</artifactId> - </dependency> - <dependency> <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId> </dependency> @@ -74,6 +78,12 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>model-evaluation</artifactId> <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -92,11 +102,50 @@ <groupId>org.hdrhistogram</groupId> <artifactId>HdrHistogram</artifactId> </exclusion> + + <!-- START JETTY embedded jars --> + <exclusion> + <groupId>org.eclipse.jetty.alpn</groupId> + <artifactId>alpn-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jetty.http2</groupId> + <artifactId>http2-server</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-alpn-java-server</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-client</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-continuation</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-jmx</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlet</artifactId> + </exclusion> + <exclusion> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlets</artifactId> + </exclusion> + <!-- END JETTY embedded jars --> </exclusions> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>vespa_jersey2</artifactId> + <artifactId>vespa-3party-bundles</artifactId> <version>${project.version}</version> <type>pom</type> </dependency> @@ -125,6 +174,10 @@ <groupId>org.apache.commons</groupId> <artifactId>commons-compress</artifactId> </exclusion> + <exclusion> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -165,18 +218,43 @@ <groupId>org.apache.commons</groupId> <artifactId>commons-math3</artifactId> </exclusion> + <exclusion> + <groupId>org.json</groupId> + <artifactId>json</artifactId> + </exclusion> + <exclusion> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> + </exclusion> </exclusions> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>config-bundle</artifactId> <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>hosted-zone-api</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>jrt</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <groupId>org.bouncycastle</groupId> + <artifactId>bcpkix-jdk15on</artifactId> + </exclusion> + </exclusions> + </dependency> <!-- NOTE: Dependencies below are added explicitly to exclude transitive deps that are not provided runtime by the container, and hence make them invisible to user projects' build classpath. diff --git a/container-disc/abi-spec.json b/container-disc/abi-spec.json index d924d5196b9..aae37552283 100644 --- a/container-disc/abi-spec.json +++ b/container-disc/abi-spec.json @@ -190,7 +190,6 @@ "public static java.lang.String getDefMd5()", "public static java.lang.String getDefName()", "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", "public void <init>(com.yahoo.container.jdisc.secretstore.SecretStoreConfig$Builder)", "public java.util.List groups()", "public com.yahoo.container.jdisc.secretstore.SecretStoreConfig$Groups groups(int)", @@ -201,7 +200,6 @@ "public static final java.lang.String CONFIG_DEF_MD5", "public static final java.lang.String CONFIG_DEF_NAME", "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", "public static final java.lang.String[] CONFIG_DEF_SCHEMA" ] } diff --git a/container-disc/pom.xml b/container-disc/pom.xml index a94225f6b5e..95f54e5d932 100644 --- a/container-disc/pom.xml +++ b/container-disc/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-disc</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> @@ -71,10 +71,6 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>org.json</groupId> - <artifactId>json</artifactId> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>component</artifactId> <version>${project.version}</version> @@ -148,17 +144,6 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>jdisc_jetty</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>javax.servlet</groupId> - <artifactId>javax.servlet-api</artifactId> - <scope>provided</scope> - </dependency> <!-- end WARNING --> <!-- ensure that transitive Jackson dependencies are not included in compile scope --> @@ -211,68 +196,34 @@ linguistics-components-jar-with-dependencies.jar, vespaclient-container-plugin-jar-with-dependencies.jar, vespa-athenz-jar-with-dependencies.jar, - container-apache-http-client-bundle-jar-with-dependencies.jar, <!-- Apache http client repackaged as bundle --> + + <!-- Apache http client repackaged as bundle --> + container-apache-http-client-bundle-jar-with-dependencies.jar, <!-- Vespa security utils with necessary 3rd party bundles --> security-utils.jar, bcpkix-jdk15on-${bouncycastle.version}.jar, bcprov-jdk15on-${bouncycastle.version}.jar, - <!-- Jetty --> - <!-- TODO Vespa 8: embed these in container-core/disc, saving a lot of dep management. --> - <!-- Check if those listed in container-dep-versions are not used by hosted apps! --> - alpn-api-${jetty-alpn.version}.jar, - http2-server-${jetty.version}.jar, - http2-common-${jetty.version}.jar, - http2-hpack-${jetty.version}.jar, - jetty-alpn-java-server-${jetty.version}.jar, - jetty-alpn-server-${jetty.version}.jar, - jetty-client-${jetty.version}.jar, - jetty-continuation-${jetty.version}.jar, - jetty-http-${jetty.version}.jar, - jetty-io-${jetty.version}.jar, - jetty-jmx-${jetty.version}.jar, - jetty-security-${jetty.version}.jar, - jetty-server-${jetty.version}.jar, - jetty-servlet-${jetty.version}.jar, - jetty-servlets-${jetty.version}.jar, - jetty-util-${jetty.version}.jar, - jetty-util-ajax-${jetty.version}.jar, + <!-- TODO Vespa 9 Stop providing servlet-api 3.x --> javax.servlet-api-3.1.0.jar, - <!-- Spifly (required for OSGi service loader used by Jetty) --> - <!-- TODO: embed these in container-core/disc, not used across bundles --> - org.apache.aries.spifly.dynamic.bundle-${spifly.version}.jar, - asm-${asm.version}.jar, - asm-analysis-${asm.version}.jar, - asm-commons-${asm.version}.jar, - asm-tree-${asm.version}.jar, - asm-util-${asm.version}.jar, - <!-- Spifly end --> + <!-- Aries SPIFly repackaged --> + container-spifly.jar, - <!-- Jersey 2 + Jackson 2 --> - aopalliance-repackaged-${hk2.version}.jar, - hk2-api-${hk2.version}.jar, - hk2-locator-${hk2.version}.jar, - hk2-utils-${hk2.version}.jar, jackson-annotations-${jackson2.version}.jar, jackson-core-${jackson2.version}.jar, jackson-databind-${jackson-databind.version}.jar, jackson-datatype-jdk8-${jackson2.version}.jar, jackson-datatype-jsr310-${jackson2.version}.jar, - jackson-jaxrs-base-${jackson2.version}.jar, - jackson-jaxrs-json-provider-${jackson2.version}.jar, - jackson-module-jaxb-annotations-${jackson2.version}.jar, - javassist-${javassist.version}.jar, - javax.ws.rs-api-${javax.ws.rs-api.version}.jar, - jersey-client-${jersey2.version}.jar, - jersey-common-${jersey2.version}.jar, - jersey-guava-${jersey2.version}.jar, - jersey-server-${jersey2.version}.jar, - osgi-resource-locator-1.0.1.jar, - validation-api-1.1.0.Final.jar - <!-- jersey2 end --> + + <!-- Only one instance of javax.ws.rs.client.ClientBuilder can be loaded by the JVM --> + javax.ws.rs-api-${javax.ws.rs-api.version}.jar </discPreInstallBundle> + + <!-- Enable Spifly ServiceLoader weaving. Required for Jetty ALPN/HTTP2 --> + <SPI-Provider>*</SPI-Provider> + <SPI-Consumer>*</SPI-Consumer> </configuration> </plugin> <plugin> diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index 71fe4f8d81c..7782e54dc90 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -308,7 +308,6 @@ public final class ConfiguredApplication implements Application { return builder; } - @SuppressWarnings("removal") // TODO Vespa 8: remove private void doReconfigurationLoop() { while (!shutdownReconfiguration) { try { diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/DeprecatedSecretStoreProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/DeprecatedSecretStoreProvider.java deleted file mode 100644 index 3292030784b..00000000000 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/DeprecatedSecretStoreProvider.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.jdisc; - -import com.yahoo.container.di.componentgraph.Provider; - -/** - * An secret store provider which provides a factory which throws exception on - * invocation - as no secret store is currently provided by default. - * The purpose of this is to provide a secret store for injection in the case where - * no secret store component is provided. - * - * @author bratseth - */ -@SuppressWarnings({"deprecation", "unused"}) -public class DeprecatedSecretStoreProvider implements Provider<com.yahoo.jdisc.http.SecretStore> { - - private static final ThrowingSecretStore instance = new ThrowingSecretStore(); - - @Override - public com.yahoo.jdisc.http.SecretStore get() { return instance; } - - @Override - public void deconstruct() { } - - private static final class ThrowingSecretStore implements com.yahoo.jdisc.http.SecretStore { - - @Override - public String getSecret(String key) { - throw new UnsupportedOperationException("A secret store is not available"); - } - - } - -} diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderProvider.java index 6d12a9251ee..f765397446a 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderProvider.java @@ -5,8 +5,6 @@ import com.yahoo.component.annotation.Inject; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.container.jdisc.MetricConsumerFactory; -import com.yahoo.container.jdisc.state.StateMonitor; -import com.yahoo.metrics.MetricsPresentationConfig; /** * A dependency injection provider which provides the default metrics provider diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh index 7220fded911..b27e02b6c23 100755 --- a/container-disc/src/main/sh/vespa-start-container-daemon.sh +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -174,7 +174,7 @@ configure_memory() { memory_options="-Xms${jvm_minHeapsize}m -Xmx${jvm_heapsize}m" memory_options="${memory_options} -XX:ThreadStackSize=${jvm_stacksize}" memory_options="${memory_options} -XX:MaxDirectMemorySize=${maxDirectMemorySize}m" - memory_options="${memory_options} -XX:+UseTransparentHugePages" + memory_options="${memory_options} $(get_jvm_hugepage_settings $jvm_heapsize)" if ((jvm_compressedClassSpaceSize != 0)); then memory_options="${memory_options} -XX:CompressedClassSpaceSize=${jvm_compressedClassSpaceSize}m" @@ -252,13 +252,6 @@ import_cfg_var () { fi } -# TODO Vespa 8: Remove when all containers use JDK 17 -configure_illegal_access() { - if [[ "$VESPA_JDK_VERSION" = "11" ]]; then - illegal_access_option="--illegal-access=debug" - fi -} - getconfig configure_memory configure_gcopts @@ -267,7 +260,6 @@ configure_classpath configure_numactl configure_cpu configure_preload -configure_illegal_access exec $numactlcmd $envcmd java \ -Dconfig.id="${VESPA_CONFIG_ID}" \ @@ -281,7 +273,6 @@ exec $numactlcmd $envcmd java \ -XX:HeapDumpPath="${VESPA_HOME}/var/crash" \ -XX:ErrorFile="${VESPA_HOME}/var/crash/hs_err_pid%p.log" \ -XX:+ExitOnOutOfMemoryError \ - ${illegal_access_option} \ --add-opens=java.base/java.io=ALL-UNNAMED \ --add-opens=java.base/java.lang=ALL-UNNAMED \ --add-opens=java.base/java.net=ALL-UNNAMED \ diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricConsumerProviders.java b/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricConsumerProviders.java index 8363705af2f..68203725d9d 100644 --- a/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricConsumerProviders.java +++ b/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricConsumerProviders.java @@ -4,9 +4,7 @@ package com.yahoo.container.jdisc.metric; import com.yahoo.component.ComponentId; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.jdisc.MetricConsumerFactory; -import com.yahoo.container.jdisc.state.StateMonitor; import com.yahoo.jdisc.application.MetricConsumer; -import com.yahoo.metrics.MetricsPresentationConfig; /** * @author Simon Thoresen Hult diff --git a/container-documentapi/pom.xml b/container-documentapi/pom.xml index 8fd786fa512..9a2572a9918 100644 --- a/container-documentapi/pom.xml +++ b/container-documentapi/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-documentapi</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> diff --git a/container-integration-test/pom.xml b/container-integration-test/pom.xml index 5a0bee3b03b..d38ca130fdb 100644 --- a/container-integration-test/pom.xml +++ b/container-integration-test/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-integration-test</artifactId> <packaging>jar</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/container-messagebus/pom.xml b/container-messagebus/pom.xml index c8e7d57a730..2ad5633b7dc 100644 --- a/container-messagebus/pom.xml +++ b/container-messagebus/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-messagebus</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>jar</packaging> <dependencies> <!-- PROVIDED scope --> diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java index b092f4fce10..baff0437b5d 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java @@ -6,7 +6,6 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.container.jdisc.ContainerMbusConfig; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentUtil; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.jdisc.ReferencedResource; @@ -28,7 +27,6 @@ import com.yahoo.messagebus.shared.SharedIntermediateSession; import com.yahoo.messagebus.shared.SharedMessageBus; import com.yahoo.messagebus.shared.SharedSourceSession; import com.yahoo.vespa.config.content.DistributionConfig; -import com.yahoo.vespa.config.content.LoadTypeConfig; import com.yahoo.yolean.concurrent.Memoized; import java.util.HashMap; @@ -72,17 +70,13 @@ public final class SessionCache extends AbstractComponent { DocumentProtocolPoliciesConfig policiesConfig, DistributionConfig distributionConfig) { this(nets::net, containerMbusConfig, documentTypeManager, - null/*TODO: Remove on Vespa 8*/, messagebusConfig, policiesConfig, distributionConfig); + messagebusConfig, policiesConfig, distributionConfig); } - /** - * @deprecated load types are deprecated. Use constructor without LoadTypeSet instead. - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 public SessionCache(Supplier<NetworkMultiplexer> net, ContainerMbusConfig containerMbusConfig, DocumentTypeManager documentTypeManager, - LoadTypeConfig loadTypeConfig, MessagebusConfig messagebusConfig, + MessagebusConfig messagebusConfig, DocumentProtocolPoliciesConfig policiesConfig, DistributionConfig distributionConfig) { this(net, diff --git a/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java b/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java index b9f33506894..e0cd9ca6dde 100644 --- a/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java +++ b/container-messagebus/src/test/java/com/yahoo/container/jdisc/messagebus/MbusClientProviderTest.java @@ -10,7 +10,6 @@ import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.messagebus.network.NetworkMultiplexer; import com.yahoo.messagebus.shared.NullNetwork; import com.yahoo.vespa.config.content.DistributionConfig; -import com.yahoo.vespa.config.content.LoadTypeConfig; import org.junit.Test; import static org.junit.Assert.assertNotNull; @@ -36,12 +35,10 @@ public class MbusClientProviderTest { testClient(new SessionConfig(builder)); } - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 private void testClient(SessionConfig config) { SessionCache cache = new SessionCache(() -> NetworkMultiplexer.dedicated(new NullNetwork()), new ContainerMbusConfig.Builder().build(), new DocumentTypeManager(new DocumentmanagerConfig.Builder().build()), - new LoadTypeConfig.Builder().build(), new MessagebusConfig.Builder().build(), new DocumentProtocolPoliciesConfig.Builder().build(), new DistributionConfig.Builder().build()); diff --git a/container-search-and-docproc/pom.xml b/container-search-and-docproc/pom.xml index e98b2a08662..036a378da3a 100644 --- a/container-search-and-docproc/pom.xml +++ b/container-search-and-docproc/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-search-and-docproc</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> @@ -204,11 +204,6 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> - <dependency> - <groupId>org.json</groupId> - <artifactId>json</artifactId> - <scope>provided</scope> - </dependency> <!-- TEST scope --> <dependency> diff --git a/container-search-and-docproc/src/main/java/com/yahoo/container/handler/observability/ApplicationStatusHandler.java b/container-search-and-docproc/src/main/java/com/yahoo/container/handler/observability/ApplicationStatusHandler.java index 7361723a3a5..46fd7358288 100644 --- a/container-search-and-docproc/src/main/java/com/yahoo/container/handler/observability/ApplicationStatusHandler.java +++ b/container-search-and-docproc/src/main/java/com/yahoo/container/handler/observability/ApplicationStatusHandler.java @@ -16,7 +16,7 @@ import com.yahoo.container.Container; import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; import com.yahoo.docproc.Call; -import com.yahoo.docproc.DocprocService; +import com.yahoo.docproc.impl.DocprocService; import com.yahoo.docproc.jdisc.DocumentProcessingHandler; import com.yahoo.jdisc.handler.AbstractRequestHandler; import com.yahoo.jdisc.handler.CompletionHandler; @@ -274,7 +274,6 @@ public class ApplicationStatusHandler extends AbstractRequestHandler { return jsonMapper.createObjectNode(); } - @SuppressWarnings("removal") // TODO Vespa 8: remove private static JsonNode renderDocprocChains(Container container) { ObjectNode ret = jsonMapper.createObjectNode(); for (RequestHandler h : container.getRequestHandlerRegistry().allComponents()) { diff --git a/container-search-gui/pom.xml b/container-search-gui/pom.xml index 3f6f7f48921..d0c6706189d 100644 --- a/container-search-gui/pom.xml +++ b/container-search-gui/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-search-gui</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <dependency> <groupId>com.google.inject</groupId> diff --git a/container-search-gui/src/main/resources/gui/_includes/search-api-reference.html b/container-search-gui/src/main/resources/gui/_includes/search-api-reference.html index e7d9004e109..086ff293b14 100644 --- a/container-search-gui/src/main/resources/gui/_includes/search-api-reference.html +++ b/container-search-gui/src/main/resources/gui/_includes/search-api-reference.html @@ -184,7 +184,7 @@ - <li class="collapseable"><a href="/documentation/vespa-http-client.html">Vespa Feeding Client API</a></li> + <li class="collapseable"><a href="/documentation/vespa-feed-client.html">Vespa Feeding Client API</a></li> diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index d4aadb11ba0..1ec90e23074 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -177,8 +177,6 @@ "public java.lang.String toString()", "public boolean fillWeightedSetItem(com.yahoo.prelude.query.WeightedSetItem)", "public java.lang.String getContent()", - "public java.lang.Object getParsedJSON()", - "public void setParsedJSON(java.lang.Object)", "public java.lang.String renderFromInspector()" ], "fields": [] @@ -1289,18 +1287,6 @@ "public static final java.lang.String queryCanonicalization" ] }, - "com.yahoo.prelude.query.QueryException": { - "superClass": "java.lang.RuntimeException", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String)", - "public void <init>(java.lang.String, java.lang.Throwable)" - ], - "fields": [] - }, "com.yahoo.prelude.query.RangeItem": { "superClass": "com.yahoo.prelude.query.IntItem", "interfaces": [], @@ -1752,8 +1738,6 @@ "public java.lang.String getIndexName()", "public int getN()", "public void setN(int)", - "public int getScoreThreshold()", - "public void setScoreThreshold(int)", "public void disclose(com.yahoo.prelude.query.textualrepresentation.Discloser)", "public int hashCode()", "public boolean equals(java.lang.Object)" @@ -1990,7 +1974,6 @@ "public varargs void trace(boolean, int, java.lang.Object[])", "public void attachContext(com.yahoo.search.Query)", "public java.lang.String yqlRepresentation()", - "public java.lang.String yqlRepresentation(com.yahoo.collections.Tuple2, boolean)", "public java.lang.String yqlRepresentation(boolean)", "public com.yahoo.search.query.context.QueryContext getContext(boolean)", "public int hashCode()", @@ -2003,8 +1986,6 @@ "public com.yahoo.container.jdisc.HttpRequest getHttpRequest()", "public com.yahoo.search.query.SessionId getSessionId()", "public com.yahoo.search.query.SessionId getSessionId(java.lang.String)", - "public boolean hasEncodableProperties()", - "public int encodeAsProperties(java.nio.ByteBuffer, boolean)", "public void prepare()", "public bridge synthetic com.yahoo.processing.Request clone()", "public bridge synthetic com.yahoo.processing.request.Properties properties()", @@ -2102,10 +2083,8 @@ "public void <init>(boolean)", "public java.lang.Object getNode()", "public boolean isWorking()", - "public boolean isQuarantined()", "public abstract void failed(com.yahoo.search.result.ErrorMessage)", "public abstract void responded()", - "public boolean isIdle()", "protected long now()", "protected abstract void setWorking(boolean, java.lang.String)", "public boolean isInternal()" @@ -2134,7 +2113,6 @@ "public com.yahoo.search.cluster.MonitorConfiguration getConfiguration()", "public boolean isClosed()", "public void add(java.lang.Object, boolean)", - "public com.yahoo.search.cluster.BaseNodeMonitor getNodeMonitor(java.lang.Object)", "public synchronized void failed(java.lang.Object, com.yahoo.search.result.ErrorMessage)", "public synchronized void responded(java.lang.Object)", "public void ping(java.util.concurrent.Executor)", @@ -2228,16 +2206,9 @@ ], "methods": [ "public void <init>()", - "public void setCheckInterval(long)", "public long getCheckInterval()", - "public void setResponseAfterFailLimit(int)", - "public void setIdleLimit(int)", - "public long getIdleLimit()", "public long getRequestTimeout()", - "public void setFailLimit(long)", "public long getFailLimit()", - "public void setFailQuarantineLimit(int)", - "public void setQuarantineTime(long)", "public java.lang.String toString()" ], "fields": [] @@ -2254,8 +2225,7 @@ "public java.lang.String name()", "public abstract void working(java.lang.Object)", "public abstract void failed(java.lang.Object)", - "public void ping(java.lang.Object, java.util.concurrent.Executor)", - "public void ping(com.yahoo.search.cluster.ClusterMonitor, java.lang.Object, java.util.concurrent.Executor)", + "public abstract void ping(com.yahoo.search.cluster.ClusterMonitor, java.lang.Object, java.util.concurrent.Executor)", "public void pingIterationCompleted()" ], "fields": [] @@ -4369,8 +4339,6 @@ ], "methods": [ "public void <init>(int, com.yahoo.search.Result, com.yahoo.search.Query, com.yahoo.processing.rendering.Renderer)", - "public com.google.common.util.concurrent.ListenableFuture waitableRender(java.io.OutputStream)", - "public static com.google.common.util.concurrent.ListenableFuture waitableRender(com.yahoo.search.Result, com.yahoo.search.Query, com.yahoo.processing.rendering.Renderer, java.io.OutputStream)", "public java.util.concurrent.CompletableFuture asyncRender(java.io.OutputStream)", "public static java.util.concurrent.CompletableFuture asyncRender(com.yahoo.search.Result, com.yahoo.search.Query, com.yahoo.processing.rendering.Renderer, java.io.OutputStream)", "public void render(java.io.OutputStream, com.yahoo.jdisc.handler.ContentChannel, com.yahoo.jdisc.handler.CompletionHandler)", @@ -4395,13 +4363,6 @@ ], "methods": [ "public void <init>(com.yahoo.jdisc.Metric, com.yahoo.container.handler.threadpool.ContainerThreadPool, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.container.core.ContainerHttpConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.search.searchchain.ExecutionFactory, ai.vespa.cloud.ZoneInfo)", - "public void <init>(com.yahoo.jdisc.Metric, com.yahoo.container.handler.threadpool.ContainerThreadPool, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.container.core.ContainerHttpConfig, com.yahoo.language.process.Embedder, com.yahoo.search.searchchain.ExecutionFactory)", - "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, com.yahoo.container.handler.threadpool.ContainerThreadPool, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.container.core.ContainerHttpConfig, com.yahoo.language.process.Embedder, com.yahoo.search.searchchain.ExecutionFactory)", - "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, com.yahoo.container.handler.threadpool.ContainerThreadPool, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.container.core.ContainerHttpConfig, com.yahoo.search.searchchain.ExecutionFactory)", - "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.container.core.ContainerHttpConfig, com.yahoo.search.searchchain.ExecutionFactory)", - "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.config.QueryProfilesConfig, com.yahoo.container.core.ContainerHttpConfig, com.yahoo.search.searchchain.ExecutionFactory)", - "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.search.searchchain.ExecutionFactory, java.util.Optional)", - "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.statistics.Statistics, com.yahoo.language.Linguistics, com.yahoo.jdisc.Metric, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.config.QueryProfilesConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.container.core.ContainerHttpConfig)", "public final com.yahoo.container.jdisc.HttpResponse handle(com.yahoo.container.jdisc.HttpRequest)", "public java.util.Optional getRequestType()", "public com.yahoo.search.Result searchAndFill(com.yahoo.search.Query, com.yahoo.component.chain.Chain)", @@ -4475,7 +4436,6 @@ "public static java.lang.String getDefMd5()", "public static java.lang.String getDefName()", "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", "public void <init>(com.yahoo.search.handler.SearchWithRendererHandlerConfig$Builder)", "public java.lang.String rendererId()" ], @@ -4483,7 +4443,6 @@ "public static final java.lang.String CONFIG_DEF_MD5", "public static final java.lang.String CONFIG_DEF_NAME", "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", "public static final java.lang.String[] CONFIG_DEF_SCHEMA" ] }, @@ -4720,7 +4679,6 @@ "public static java.lang.String getDefMd5()", "public static java.lang.String getDefName()", "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", "public void <init>(com.yahoo.search.pagetemplates.PageTemplatesConfig$Builder)", "public java.util.List page()", "public java.lang.String page(int)" @@ -4729,7 +4687,6 @@ "public static final java.lang.String CONFIG_DEF_MD5", "public static final java.lang.String CONFIG_DEF_NAME", "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", "public static final java.lang.String[] CONFIG_DEF_SCHEMA" ] }, @@ -4820,7 +4777,6 @@ "public static java.lang.String getDefMd5()", "public static java.lang.String getDefName()", "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", "public void <init>(com.yahoo.search.pagetemplates.ResolversConfig$Builder)", "public java.util.List component()", "public com.yahoo.search.pagetemplates.ResolversConfig$Component component(int)" @@ -4829,7 +4785,6 @@ "public static final java.lang.String CONFIG_DEF_MD5", "public static final java.lang.String CONFIG_DEF_NAME", "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", "public static final java.lang.String[] CONFIG_DEF_SCHEMA" ] }, @@ -6324,7 +6279,8 @@ "public void <init>(com.yahoo.search.query.profile.types.QueryProfileTypeRegistry)", "public final void register(com.yahoo.search.query.profile.compiled.CompiledQueryProfile)", "public com.yahoo.search.query.profile.types.QueryProfileTypeRegistry getTypeRegistry()", - "public com.yahoo.search.query.profile.compiled.CompiledQueryProfile findQueryProfile(java.lang.String)" + "public com.yahoo.search.query.profile.compiled.CompiledQueryProfile findQueryProfile(java.lang.String)", + "public static com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry fromConfig(com.yahoo.search.query.profile.config.QueryProfilesConfig)" ], "fields": [ "public static final com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry empty" @@ -6691,7 +6647,6 @@ "public" ], "methods": [ - "public void <init>(com.yahoo.search.Query, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.language.process.Embedder)", "public void <init>(com.yahoo.search.Query, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, java.util.Map)", "public void setParentQuery(com.yahoo.search.Query)", "public java.lang.Object get(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)", @@ -6889,12 +6844,10 @@ "public" ], "methods": [ - "public void <init>()", "public void <init>(com.yahoo.search.query.Ranking)", "public void put(java.lang.String, double)", "public void put(java.lang.String, com.yahoo.tensor.Tensor)", "public void put(java.lang.String, java.lang.String)", - "public java.lang.String get(java.lang.String)", "public java.lang.Object getObject(java.lang.String)", "public java.util.OptionalDouble getDouble(java.lang.String)", "public java.util.Optional getTensor(java.lang.String)", @@ -7146,7 +7099,6 @@ "public static java.lang.String getDefMd5()", "public static java.lang.String getDefName()", "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", "public void <init>(com.yahoo.search.query.rewrite.RewritesConfig$Builder)", "public java.util.List fsaDict()", "public com.yahoo.search.query.rewrite.RewritesConfig$FsaDict fsaDict(int)" @@ -7155,7 +7107,6 @@ "public static final java.lang.String CONFIG_DEF_MD5", "public static final java.lang.String CONFIG_DEF_NAME", "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", "public static final java.lang.String[] CONFIG_DEF_SCHEMA" ] }, @@ -7258,10 +7209,7 @@ "public" ], "methods": [ - "public void <init>(com.fasterxml.jackson.core.JsonGenerator, boolean)", - "public void <init>(com.fasterxml.jackson.core.JsonGenerator, boolean, boolean)", "protected void <init>(boolean, boolean, boolean)", - "public void <init>(com.fasterxml.jackson.core.JsonGenerator, boolean, boolean, boolean)", "public void accept(java.lang.String, java.lang.Object)", "public void accept(java.lang.String, byte[], int, int)", "protected boolean shouldRender(java.lang.String, java.lang.Object)", @@ -7281,7 +7229,6 @@ "methods": [ "public void <init>()", "public void <init>(java.util.concurrent.Executor)", - "protected static com.fasterxml.jackson.databind.ObjectMapper createJsonCodec()", "public void init()", "public void beginResponse(java.io.OutputStream)", "protected void renderTrace(com.yahoo.processing.execution.Execution$Trace)", @@ -7302,8 +7249,7 @@ "public void endResponse()", "public java.lang.String getEncoding()", "public java.lang.String getMimeType()", - "protected com.yahoo.search.rendering.JsonRenderer$FieldConsumer createFieldConsumer(boolean)", - "protected com.yahoo.search.rendering.JsonRenderer$FieldConsumer createFieldConsumer(com.fasterxml.jackson.core.JsonGenerator, boolean)" + "protected com.yahoo.search.rendering.JsonRenderer$FieldConsumer createFieldConsumer(boolean)" ], "fields": [] }, @@ -7817,7 +7763,6 @@ "public boolean isFillable()", "public java.util.Set getFilled()", "public com.yahoo.processing.response.IncomingData incoming()", - "public com.google.common.util.concurrent.ListenableFuture complete()", "public java.util.concurrent.CompletableFuture completeFuture()", "public void addDataListener(java.lang.Runnable)", "public void close()", @@ -8180,7 +8125,6 @@ "methods": [ "public void <init>(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts, com.yahoo.search.schema.SchemaInfo, com.yahoo.language.process.SpecialTokenRegistry, com.yahoo.search.rendering.RendererRegistry, com.yahoo.language.Linguistics, java.util.concurrent.Executor)", "public void <init>(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts, com.yahoo.language.process.SpecialTokenRegistry, com.yahoo.search.rendering.RendererRegistry, com.yahoo.language.Linguistics, java.util.concurrent.Executor)", - "public void <init>(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts, com.yahoo.language.process.SpecialTokenRegistry, com.yahoo.search.rendering.RendererRegistry, com.yahoo.language.Linguistics)", "public static com.yahoo.search.searchchain.Execution$Context createContextStub()", "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.search.searchchain.SearchChainRegistry)", "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.prelude.IndexFacts)", @@ -8237,9 +8181,7 @@ "public com.yahoo.search.searchchain.Execution$Context context()", "public com.yahoo.search.statistics.TimeTracker timer()" ], - "fields": [ - "public static final java.lang.String ATTRIBUTEPREFETCH" - ] + "fields": [] }, "com.yahoo.search.searchchain.ExecutionFactory": { "superClass": "com.yahoo.component.AbstractComponent", @@ -8251,7 +8193,6 @@ "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.search.config.SchemaInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.language.Linguistics, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor)", "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.search.schema.SchemaInfo, com.yahoo.container.QrSearchersConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.language.Linguistics, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor)", "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.language.Linguistics, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor)", - "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.language.Linguistics, com.yahoo.component.provider.ComponentRegistry)", "public com.yahoo.search.searchchain.Execution newExecution(com.yahoo.component.chain.Chain)", "public com.yahoo.search.searchchain.Execution newExecution(java.lang.String)", "public com.yahoo.search.searchchain.SearchChainRegistry searchChainRegistry()", @@ -8521,132 +8462,6 @@ ], "fields": [] }, - "com.yahoo.search.statistics.MeasureQpsConfig$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigInstance$Builder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.search.statistics.MeasureQpsConfig)", - "public com.yahoo.search.statistics.MeasureQpsConfig$Builder outputmethod(com.yahoo.search.statistics.MeasureQpsConfig$Outputmethod$Enum)", - "public com.yahoo.search.statistics.MeasureQpsConfig$Builder queryproperty(java.lang.String)", - "public com.yahoo.search.statistics.MeasureQpsConfig$Builder eventname(java.lang.String)", - "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", - "public final java.lang.String getDefMd5()", - "public final java.lang.String getDefName()", - "public final java.lang.String getDefNamespace()", - "public final boolean getApplyOnRestart()", - "public final void setApplyOnRestart(boolean)", - "public com.yahoo.search.statistics.MeasureQpsConfig build()" - ], - "fields": [] - }, - "com.yahoo.search.statistics.MeasureQpsConfig$Outputmethod$Enum": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.search.statistics.MeasureQpsConfig$Outputmethod$Enum[] values()", - "public static com.yahoo.search.statistics.MeasureQpsConfig$Outputmethod$Enum valueOf(java.lang.String)" - ], - "fields": [ - "public static final enum com.yahoo.search.statistics.MeasureQpsConfig$Outputmethod$Enum STATISTICS", - "public static final enum com.yahoo.search.statistics.MeasureQpsConfig$Outputmethod$Enum METAHIT" - ] - }, - "com.yahoo.search.statistics.MeasureQpsConfig$Outputmethod": { - "superClass": "com.yahoo.config.EnumNode", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.search.statistics.MeasureQpsConfig$Outputmethod$Enum)" - ], - "fields": [ - "public static final com.yahoo.search.statistics.MeasureQpsConfig$Outputmethod$Enum STATISTICS", - "public static final com.yahoo.search.statistics.MeasureQpsConfig$Outputmethod$Enum METAHIT" - ] - }, - "com.yahoo.search.statistics.MeasureQpsConfig$Producer": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigInstance$Producer" - ], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void getConfig(com.yahoo.search.statistics.MeasureQpsConfig$Builder)" - ], - "fields": [] - }, - "com.yahoo.search.statistics.MeasureQpsConfig": { - "superClass": "com.yahoo.config.ConfigInstance", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public static java.lang.String getDefMd5()", - "public static java.lang.String getDefName()", - "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", - "public void <init>(com.yahoo.search.statistics.MeasureQpsConfig$Builder)", - "public com.yahoo.search.statistics.MeasureQpsConfig$Outputmethod$Enum outputmethod()", - "public java.lang.String queryproperty()", - "public java.lang.String eventname()" - ], - "fields": [ - "public static final java.lang.String CONFIG_DEF_MD5", - "public static final java.lang.String CONFIG_DEF_NAME", - "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", - "public static final java.lang.String[] CONFIG_DEF_SCHEMA" - ] - }, - "com.yahoo.search.statistics.PeakQpsSearcher$QpsHit": { - "superClass": "com.yahoo.search.result.Hit", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.Integer, java.lang.Double)", - "public boolean isMeta()", - "public java.lang.String toString()" - ], - "fields": [ - "public static final java.lang.String MEAN_QPS", - "public static final java.lang.String PEAK_QPS", - "public static final java.lang.String SCHEME" - ] - }, - "com.yahoo.search.statistics.PeakQpsSearcher": { - "superClass": "com.yahoo.search.Searcher", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.search.statistics.MeasureQpsConfig, com.yahoo.statistics.Statistics)", - "public com.yahoo.search.Result search(com.yahoo.search.Query, com.yahoo.search.searchchain.Execution)" - ], - "fields": [] - }, "com.yahoo.search.statistics.TimeTracker$Activity": { "superClass": "java.lang.Enum", "interfaces": [], @@ -8691,166 +8506,6 @@ ], "fields": [] }, - "com.yahoo.search.statistics.TimingSearcher$Parameters": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String, com.yahoo.search.statistics.TimeTracker$Activity)" - ], - "fields": [] - }, - "com.yahoo.search.statistics.TimingSearcher": { - "superClass": "com.yahoo.search.cluster.PingableSearcher", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.component.ComponentId, com.yahoo.search.statistics.TimingSearcherConfig, com.yahoo.statistics.Statistics)", - "public void fill(com.yahoo.search.Result, java.lang.String, com.yahoo.search.searchchain.Execution)", - "public com.yahoo.prelude.Pong ping(com.yahoo.prelude.Ping, com.yahoo.search.searchchain.Execution)", - "public com.yahoo.search.Result search(com.yahoo.search.Query, com.yahoo.search.searchchain.Execution)", - "public void setMeasurements(com.yahoo.statistics.Value)", - "public void deconstruct()" - ], - "fields": [] - }, - "com.yahoo.search.statistics.TimingSearcherConfig$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigInstance$Builder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.search.statistics.TimingSearcherConfig)", - "public com.yahoo.search.statistics.TimingSearcherConfig$Builder timer(com.yahoo.search.statistics.TimingSearcherConfig$Timer$Builder)", - "public com.yahoo.search.statistics.TimingSearcherConfig$Builder timer(java.util.function.Consumer)", - "public com.yahoo.search.statistics.TimingSearcherConfig$Builder timer(java.util.List)", - "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", - "public final java.lang.String getDefMd5()", - "public final java.lang.String getDefName()", - "public final java.lang.String getDefNamespace()", - "public final boolean getApplyOnRestart()", - "public final void setApplyOnRestart(boolean)", - "public com.yahoo.search.statistics.TimingSearcherConfig build()" - ], - "fields": [ - "public java.util.List timer" - ] - }, - "com.yahoo.search.statistics.TimingSearcherConfig$Producer": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigInstance$Producer" - ], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void getConfig(com.yahoo.search.statistics.TimingSearcherConfig$Builder)" - ], - "fields": [] - }, - "com.yahoo.search.statistics.TimingSearcherConfig$Timer$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigBuilder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.search.statistics.TimingSearcherConfig$Timer)", - "public com.yahoo.search.statistics.TimingSearcherConfig$Timer$Builder name(java.lang.String)", - "public com.yahoo.search.statistics.TimingSearcherConfig$Timer$Builder eventname(java.lang.String)", - "public com.yahoo.search.statistics.TimingSearcherConfig$Timer$Builder measure(com.yahoo.search.statistics.TimingSearcherConfig$Timer$Measure$Enum)", - "public com.yahoo.search.statistics.TimingSearcherConfig$Timer build()" - ], - "fields": [] - }, - "com.yahoo.search.statistics.TimingSearcherConfig$Timer$Measure$Enum": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.search.statistics.TimingSearcherConfig$Timer$Measure$Enum[] values()", - "public static com.yahoo.search.statistics.TimingSearcherConfig$Timer$Measure$Enum valueOf(java.lang.String)" - ], - "fields": [ - "public static final enum com.yahoo.search.statistics.TimingSearcherConfig$Timer$Measure$Enum FILL", - "public static final enum com.yahoo.search.statistics.TimingSearcherConfig$Timer$Measure$Enum PING", - "public static final enum com.yahoo.search.statistics.TimingSearcherConfig$Timer$Measure$Enum SEARCH" - ] - }, - "com.yahoo.search.statistics.TimingSearcherConfig$Timer$Measure": { - "superClass": "com.yahoo.config.EnumNode", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.search.statistics.TimingSearcherConfig$Timer$Measure$Enum)" - ], - "fields": [ - "public static final com.yahoo.search.statistics.TimingSearcherConfig$Timer$Measure$Enum FILL", - "public static final com.yahoo.search.statistics.TimingSearcherConfig$Timer$Measure$Enum PING", - "public static final com.yahoo.search.statistics.TimingSearcherConfig$Timer$Measure$Enum SEARCH" - ] - }, - "com.yahoo.search.statistics.TimingSearcherConfig$Timer": { - "superClass": "com.yahoo.config.InnerNode", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(com.yahoo.search.statistics.TimingSearcherConfig$Timer$Builder)", - "public java.lang.String name()", - "public java.lang.String eventname()", - "public com.yahoo.search.statistics.TimingSearcherConfig$Timer$Measure$Enum measure()" - ], - "fields": [] - }, - "com.yahoo.search.statistics.TimingSearcherConfig": { - "superClass": "com.yahoo.config.ConfigInstance", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public static java.lang.String getDefMd5()", - "public static java.lang.String getDefName()", - "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", - "public void <init>(com.yahoo.search.statistics.TimingSearcherConfig$Builder)", - "public java.util.List timer()", - "public com.yahoo.search.statistics.TimingSearcherConfig$Timer timer(int)" - ], - "fields": [ - "public static final java.lang.String CONFIG_DEF_MD5", - "public static final java.lang.String CONFIG_DEF_NAME", - "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", - "public static final java.lang.String[] CONFIG_DEF_SCHEMA" - ] - }, "com.yahoo.text.interpretation.AnnotationClass": { "superClass": "java.lang.Object", "interfaces": [], diff --git a/container-search/pom.xml b/container-search/pom.xml index 8005b4179e5..ccec1330d68 100644 --- a/container-search/pom.xml +++ b/container-search/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-search</artifactId> <packaging>jar</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <dependency> <groupId>junit</groupId> @@ -64,7 +64,7 @@ <dependency> <groupId>org.json</groupId> <artifactId>json</artifactId> - <scope>provided</scope> + <scope>compile</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -103,10 +103,6 @@ <artifactId>icu4j</artifactId> </dependency> <dependency> - <groupId>net.jcip</groupId> - <artifactId>jcip-annotations</artifactId> - </dependency> - <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-core</artifactId> <scope>provided</scope> diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java index 57a8d518ed2..c79099df1de 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java @@ -6,6 +6,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.logging.Logger; @@ -35,22 +36,11 @@ public final class IndexModel { public IndexModel(Map<String, List<String>> masterClusters, Collection<SearchDefinition> searchDefinitions) { this.masterClusters = masterClusters; - this.searchDefinitions = searchDefinitions.stream().collect(Collectors.toMap(sd -> sd.getName(), sd -> sd)); + this.searchDefinitions = new LinkedHashMap<>(); this.unionSearchDefinition = unionOf(searchDefinitions); - } - - /** - * Use IndexModel as a pure wrapper for the parameters given. - * - * @deprecated use the constructor without the third parameter - */ - @Deprecated // TODO: Remove Vespa 8 - public IndexModel(Map<String, List<String>> masterClusters, - Map<String, SearchDefinition> searchDefinitions, - SearchDefinition unionSearchDefinition) { - this.masterClusters = masterClusters; - this.searchDefinitions = searchDefinitions; - this.unionSearchDefinition = unionSearchDefinition; + for (var sd : searchDefinitions) { + this.searchDefinitions.put(sd.getName(), sd); + } } public IndexModel(IndexInfoConfig indexInfo, QrSearchersConfig clusters) { @@ -135,8 +125,6 @@ public final class IndexModel { public Map<String, SearchDefinition> getSearchDefinitions() { return searchDefinitions; } - /** @deprecated do not use */ - @Deprecated // TODO: Remove on Vespa 8 - public SearchDefinition getUnionSearchDefinition() { return unionSearchDefinition; } + SearchDefinition getUnionSearchDefinition() { return unionSearchDefinition; } } diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index a1cad6de419..496e057a243 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -38,7 +38,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.FutureTask; import java.util.concurrent.RejectedExecutionException; -import java.util.stream.Collectors; import static com.yahoo.container.QrSearchersConfig.Searchcluster.Indexingmode.STREAMING; @@ -166,63 +165,6 @@ public class ClusterSearcher extends Searcher { this(schemas, null, null); } - /** - * Returns an error if the document types do not have the requested rank - * profile. For the case of multiple document types, only returns an - * error if we have restricted the set of documents somehow. This is - * because when searching over all doc types, common ancestors might - * not have the requested rank profile and failing on that basis is - * probably not reasonable. - * - * @param query query - * @param schemas set of requested schemas for this query - * @return null if requested rank profile is ok for the requested - * schemas, a result with error message if not. - */ - // TODO: This should be in a separate searcher - // TODO Vespa 8: This should simply fail if the specified profile isn't present in all schemas - private Result checkValidRankProfiles(Query query, Set<String> schemas, Execution.Context context) { - String rankProfile = query.getRanking().getProfile(); - Set<String> invalidInSchemas = null; - Set<String> schemasHavingProfile = schemasHavingProfile(rankProfile, context); - - if (schemasHavingProfile.isEmpty()) { - invalidInSchemas = schemas; - } - else if (schemas.size() == 1) { - if ( ! schemasHavingProfile.containsAll(schemas)) - invalidInSchemas = schemas; - } - else { - // multiple schemas, only fail when restricting doc types - Set<String> restrict = query.getModel().getRestrict(); - Set<String> sources = query.getModel().getSources(); - boolean validate = restrict != null && !restrict.isEmpty(); - validate = validate || sources != null && !sources.isEmpty(); - if (validate && !schemasHavingProfile.containsAll(schemas)) { - invalidInSchemas = new HashSet<>(schemas); - invalidInSchemas.removeAll(schemasHavingProfile); - } - } - - if (invalidInSchemas != null && !invalidInSchemas.isEmpty()) { - String plural = invalidInSchemas.size() > 1 ? "s" : ""; - return new Result(query, - ErrorMessage.createInvalidQueryParameter("Requested rank profile '" + rankProfile + - "' is undefined for document type" + plural + " '" + - String.join(", ", invalidInSchemas) + "'")); - } - - return null; - } - - private Set<String> schemasHavingProfile(String profile, Execution.Context context) { - return context.schemaInfo().schemas().values().stream() - .filter(schema -> schema.rankProfiles().containsKey(profile)) - .map(schema -> schema.name()) - .collect(Collectors.toSet()); - } - @Override public void fill(com.yahoo.search.Result result, String summaryClass, Execution execution) { Query query = result.getQuery(); @@ -285,12 +227,6 @@ public class ClusterSearcher extends Searcher { return searchMultipleDocumentTypes(searcher, query, execution); } else { String docType = schemas.iterator().next(); - - Result invalidRankProfile = checkValidRankProfiles(query, schemas, execution.context()); - if (invalidRankProfile != null) { - return invalidRankProfile; - } - query.getModel().setRestrict(docType); return searcher.search(query, execution); } @@ -310,10 +246,6 @@ public class ClusterSearcher extends Searcher { private Result searchMultipleDocumentTypes(Searcher searcher, Query query, Execution execution) { Set<String> schemas = resolveSchemas(query, execution.context().getIndexFacts()); - - Result invalidRankProfile = checkValidRankProfiles(query, schemas, execution.context()); - if (invalidRankProfile != null) return invalidRankProfile; - List<Query> queries = createQueries(query, schemas); if (queries.size() == 1) { return searcher.search(queries.get(0), execution); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index 21037be1a8b..584249fa1f3 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -14,7 +14,6 @@ import com.yahoo.protect.Validator; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.cluster.PingableSearcher; -import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.search.schema.RankProfile; import com.yahoo.search.grouping.vespa.GroupingExecutor; import com.yahoo.search.result.ErrorMessage; @@ -168,6 +167,11 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { return new Result(query, ErrorMessage.createNullQuery(query.getHttpRequest().getUri().toString())); } + if ( ! getDocumentDatabase(query).schema().rankProfiles().containsKey(query.getRanking().getProfile())) + return new Result(query, ErrorMessage.createInvalidQueryParameter(getDocumentDatabase(query).schema() + + " does not contain requested rank profile '" + + query.getRanking().getProfile() + "'")); + QueryRewrite.optimizeByRestrict(query); QueryRewrite.optimizeAndNot(query); QueryRewrite.collapseSingleComposites(query); @@ -185,8 +189,6 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { return new Result(query); Result result = doSearch2(query, execution); - if (isLoggingFine()) - getLogger().fine("Result NOT retrieved from cache"); if (query.getTraceLevel() >= 1) query.trace(getName() + " dispatch response: " + result, false, 1); diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java index 3b1b5d5e2d4..3ec25dfe3a8 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java @@ -21,7 +21,6 @@ import java.util.Iterator; * * @author Steinar Knutsen */ -// TODO Vespa 8: remove methods leaking org.json types (replace with Slime equivalent?) public class JSONString implements Inspectable { private Inspector value; @@ -100,8 +99,6 @@ public class JSONString implements Inspectable { return content; } else if (parsedJSON.getClass() == JSONArray.class) { return render((JSONArray) parsedJSON); - } else if (parsedJSON.getClass() == JSONObject.class) { - return render((JSONObject) parsedJSON); } else { return content; } @@ -153,10 +150,6 @@ public class JSONString implements Inspectable { return FieldRenderer.renderMapOrArray(new StringBuilder(), sequence, 2).toString(); } - private static String render(JSONObject structure) { - return FieldRenderer.renderStruct(new StringBuilder(), structure, 2).toString(); - } - private static abstract class FieldRenderer { protected static void indent(StringBuilder renderTarget, int nestingLevel) { @@ -180,20 +173,12 @@ public class JSONString implements Inspectable { return renderTarget; } - @Deprecated // TODO: Remove on Vespa 8 - public static StringBuilder renderStruct(StringBuilder renderTarget, JSONObject object, int nestingLevel) { - StructureFieldRenderer.renderStructure(renderTarget, object, nestingLevel + 1); - indent(renderTarget, nestingLevel); - return renderTarget; - } - public abstract void render(StringBuilder renderTarget, Object value, int nestingLevel); public abstract void closeTag(StringBuilder renderTarget, int nestingLevel, String closing); /** Returns a value from an object, or null if not found */ - @Deprecated // TODO: Remove on Vespa 8 - protected static Object get(String field, JSONObject source) { + static Object get(String field, JSONObject source) { try { return source.get(field); } @@ -209,8 +194,6 @@ public class JSONString implements Inspectable { NumberFieldRenderer.renderNumber(renderTarget, (Number) value); } else if (value.getClass() == String.class) { StringFieldRenderer.renderString(renderTarget, (String) value); - } else if (value.getClass() == JSONObject.class) { - renderStruct(renderTarget, (JSONObject) value, nestingLevel); } else { renderTarget.append(value); } @@ -439,21 +422,6 @@ public class JSONString implements Inspectable { return content; } - /** @deprecated Use {@link #getContent()} instead and parse content yourself */ - @Deprecated(forRemoval = true, since = "7") - public Object getParsedJSON() { - initContent(); - if (parsedJSON == null) { - initJSON(); - } - return parsedJSON; - } - - @Deprecated(forRemoval = true, since = "7") - public void setParsedJSON(Object parsedJSON) { - this.parsedJSON = parsedJSON; - } - public String renderFromInspector() { return XmlRenderer.render(new StringBuilder(), value).toString(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/QueryException.java b/container-search/src/main/java/com/yahoo/prelude/query/QueryException.java deleted file mode 100644 index 60aab75f219..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/query/QueryException.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.query; - -/** - * Runtime exception to mark errors in query parsing. - * - * @author Steinar Knutsen - * @deprecated no methods throw this - */ -@Deprecated // TODO: Remove on Vespa 8 -public class QueryException extends RuntimeException { - - public QueryException(String message) { - super(message); - } - - public QueryException(String message, Throwable cause) { - super(message, cause); - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java index 5ffd4d1584d..859973f3b78 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WeakAndItem.java @@ -24,7 +24,6 @@ public final class WeakAndItem extends NonReducibleCompositeItem { private int n; private String index; - private int scoreThreshold = 0; /** Creates a WAND item with default N */ public WeakAndItem() { @@ -81,17 +80,6 @@ public final class WeakAndItem extends NonReducibleCompositeItem { public void setN(int N) { this.n = N; } - @Deprecated // TODO: Remove on Vespa 8 - public int getScoreThreshold() { return scoreThreshold; } - - /** - * Noop. - * - * @deprecated has no effect - */ - @Deprecated // TODO: Remove on Vespa 8 - public void setScoreThreshold(int scoreThreshold) { this.scoreThreshold = scoreThreshold; } - @Override protected void encodeThis(ByteBuffer buffer) { super.encodeThis(buffer); diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 5e46815a152..dcbda80ecab 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -4,18 +4,12 @@ package com.yahoo.search; import ai.vespa.cloud.ZoneInfo; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.yahoo.collections.Tuple2; -import com.yahoo.component.Version; import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.fs4.MapEncoder; import com.yahoo.language.process.Embedder; -import com.yahoo.prelude.fastsearch.DocumentDatabase; -import com.yahoo.prelude.query.Highlight; import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.dispatch.Dispatcher; -import com.yahoo.search.dispatch.rpc.ProtobufSerialization; import com.yahoo.search.federation.FederationSearcher; import com.yahoo.search.query.Model; import com.yahoo.search.query.ParameterParser; @@ -858,12 +852,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { } } - /** @deprecated remove the ignored segmenterVersion argument from invocations */ - @Deprecated // TODO: Remove on Vespa 8 - public String yqlRepresentation(Tuple2<String, Version> segmenterVersion, boolean includeHitsAndOffset) { - return yqlRepresentation(includeHitsAndOffset); - } - /** * Serialize this query as YQL+. This will create a string representation * which should always be legal YQL+. If a problem occurs, a @@ -894,7 +882,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { yql.append(" where "); String insert = serializeSortingAndLimits(includeHitsAndOffset); yql.append(VespaSerializer.serialize(this, insert)); - yql.append(';'); return yql.toString(); } @@ -1061,73 +1048,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { return new SessionId(requestId, getRanking().getProfile()); } - @Deprecated // TODO: Remove on Vespa 8 - public boolean hasEncodableProperties() { - if ( ! ranking.getProperties().isEmpty()) return true; - if ( ! ranking.getFeatures().isEmpty()) return true; - if ( ranking.getFreshness() != null) return true; - if ( model.getSearchPath() != null) return true; - if ( model.getDocumentDb() != null) return true; - if ( presentation.getHighlight() != null && ! presentation.getHighlight().getHighlightItems().isEmpty()) return true; - return false; - } - - /** - * Encodes properties of this query. - * - * @param buffer the buffer to encode to - * @param encodeQueryData true to encode all properties, false to only include session information, not actual query data - * @return the encoded length - * @deprecated do not use - */ - @Deprecated // TODO: Remove on Vespa 8 - public int encodeAsProperties(ByteBuffer buffer, boolean encodeQueryData) { - // Make sure we don't encode anything here if we have turned the property feature off - // Due to sendQuery we sometimes end up turning this feature on and then encoding a 0 int as the number of - // property maps - that's ok (probably we should simplify by just always turning the feature on) - if (! hasEncodableProperties()) return 0; - int start = buffer.position(); - int mapCountPosition = buffer.position(); - buffer.putInt(0); // map count will go here - int mapCount = 0; - mapCount += ranking.getProperties().encode(buffer, encodeQueryData); - if (encodeQueryData) { - mapCount += ranking.getFeatures().encode(buffer); - if (presentation.getHighlight() != null) { - mapCount += MapEncoder.encodeMultiMap(Highlight.HIGHLIGHTTERMS, presentation.getHighlight().getHighlightTerms(), buffer); - } - mapCount += MapEncoder.encodeMap("model", createModelMap(), buffer); - } - mapCount += MapEncoder.encodeSingleValue(DocumentDatabase.MATCH_PROPERTY, DocumentDatabase.SEARCH_DOC_TYPE_KEY, model.getDocumentDb(), buffer); - mapCount += MapEncoder.encodeMap("caches", createCacheSettingMap(), buffer); - buffer.putInt(mapCountPosition, mapCount); - return buffer.position() - start; - } - - private Map<String, Boolean> createCacheSettingMap() { - if (getGroupingSessionCache() && ranking.getQueryCache()) { - Map<String, Boolean> cacheSettingMap = new HashMap<>(); - cacheSettingMap.put("grouping", true); - cacheSettingMap.put("query", true); - return cacheSettingMap; - } - if (getGroupingSessionCache()) - return Collections.singletonMap("grouping", true); - if (ranking.getQueryCache()) - return Collections.singletonMap("query", true); - return Collections.emptyMap(); - } - - private Map<String, String> createModelMap() { - Map<String, String> m = new HashMap<>(); - if (model.getSearchPath() != null) m.put("searchpath", model.getSearchPath()); - - int traceLevel = ProtobufSerialization.getTraceLevelForBackend(this); - if (traceLevel > 0) m.put("tracelevel", String.valueOf(traceLevel)); - - return m; - } - /** * Prepares this for binary serialization. * diff --git a/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java index 48bfd4c9b66..fd8110e1173 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java @@ -58,10 +58,6 @@ public abstract class BaseNodeMonitor<T> { */ public boolean isWorking() { return isWorking; } - /** @deprecated Not used */ - @Deprecated // TODO: Remove on Vespa 8 - public boolean isQuarantined() { return isQuarantined; } - /** * Called when this node fails. * @@ -79,12 +75,6 @@ public abstract class BaseNodeMonitor<T> { */ public abstract void responded(); - /** @deprecated Not used */ - @Deprecated // TODO: Remove on Vespa 8 - public boolean isIdle() { - return (now()-respondedAt) >= configuration.getIdleLimit(); - } - protected long now() { return System.currentTimeMillis(); } diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java index 360dcd38d3b..4bb1818e5d1 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java @@ -79,15 +79,6 @@ public class ClusterMonitor<T> { nodeMonitors.put(node, new TrafficNodeMonitor<>(node, configuration, internal)); } - /** - * Returns the monitor of the given node, or null if this node has not been added - * @deprecated Will be removed in Vespa 8. - */ - @Deprecated(forRemoval = true, since = "7.537") - public BaseNodeMonitor<T> getNodeMonitor(T node) { - return nodeMonitors.get(node); - } - /** Called from ClusterSearcher/NodeManager when a node failed */ public synchronized void failed(T node, ErrorMessage error) { if (closed.get()) return; // Do not touch state if close has started. diff --git a/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java b/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java index c421e9dbe69..1f6602053d9 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java @@ -9,85 +9,29 @@ package com.yahoo.search.cluster; public class MonitorConfiguration { /** The interval in ms between consecutive checks of the monitored nodes */ - private long checkInterval = 1000; + private final long checkInterval = 1000; /** The number of milliseconds to attempt to complete a request before giving up */ private final long requestTimeout = 980; /** The number of milliseconds a node is allowed to fail before we mark it as not working */ - private long failLimit = 5000; - - /** Sets the interval between each ping of idle or failing nodes. Default is 1000 ms. */ - @Deprecated // TODO: Remove on Vespa 8 - public void setCheckInterval(long intervalMs) { this.checkInterval = intervalMs; } + private final long failLimit = 5000; /** Returns the interval between each ping of idle or failing nodes. Default is 1000 ms. */ public long getCheckInterval() { return checkInterval; } /** - * Sets the number of times a failed node must respond before it is put back in service. Default is 3. - * - * @deprecated will go away in Vespa 8 - */ - @Deprecated // TODO: Remove on Vespa 8 - public void setResponseAfterFailLimit(int responseAfterFailLimit) { } - - /** - * Sets the number of ms a node (failing or working) is allowed to stay idle before it is pinged. Default is 3000. - * - * @deprecated Will go away in Vespa 8 - */ - @Deprecated // TODO: Remove on Vespa 8 - public void setIdleLimit(int idleLimit) { } - - /** - * Gets the number of ms a node (failing or working) is allowed to stay idle before it is pinged. Default is 3000. - * - * @deprecated Will go away in Vespa 8 - */ - @Deprecated // TODO: Remove on Vespa 8 - public long getIdleLimit() { - return 3000; - } - - /** * Returns the number of milliseconds to attempt to service a request * (at different nodes) before giving up. Default is 5000 ms. */ public long getRequestTimeout() { return requestTimeout; } /** - * Sets the number of milliseconds a node is allowed to fail before we - * mark it as not working - */ - @Deprecated // TODO: Remove on Vespa 8 - public void setFailLimit(long failLimit) { this.failLimit=failLimit; } - - /** * Returns the number of milliseconds a node is allowed to fail before we * mark it as not working */ public long getFailLimit() { return failLimit; } - /** - * The number of times a node must fail in one hour to be placed - * in quarantine. Once in quarantine it won't be put back in - * productuion before quarantineTime has expired even if it is - * working. Default is 3 - * - * @deprecated Will go away in Vespa 8 - */ - @Deprecated // TODO: Remove on Vespa 8 - public void setFailQuarantineLimit(int failQuarantineLimit) { } - - /** - * The number of ms an unstable node is quarantined. Default is 100*60*60 - * - * @deprecated Will go away in Vespa 8 - */ - @Deprecated // TODO: Remove on Vespa 8 - public void setQuarantineTime(long quarantineTime) { } - @Override public String toString() { return "monitor configuration [" + diff --git a/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java b/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java index db583a65606..c4a532bdb03 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java @@ -20,25 +20,12 @@ public interface NodeManager<T> { /** Called when a working node fails */ void failed(T node); - /** - * Called when a node should be pinged. - * This *must* lead to either a call to NodeMonitor.failed or NodeMonitor.responded - * - * @deprecated Use ping(ClusterMonitor clusterMonitor, T node, Executor executor) instead. - */ - @Deprecated // TODO: Remove on Vespa 8 - default void ping(T node, Executor executor) { - throw new IllegalStateException("If you have not overrriden ping(ClusterMonitor<T> clusterMonitor, T node, Executor executor), you should at least have overriden this method."); - } - /** * Called when a node should be pinged. * This *must* lead to either a call to ClusterMonitor.failed or ClusterMonitor.responded */ - default void ping(ClusterMonitor<T> clusterMonitor, T node, Executor executor) { - ping(node, executor); - } - + void ping(ClusterMonitor<T> clusterMonitor, T node, Executor executor); + /** Called right after a ping has been issued to each node. This default implementation does nothing. */ default void pingIterationCompleted() {} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java index 57f7ea34a97..92726ef1415 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java @@ -13,6 +13,7 @@ import com.yahoo.data.access.slime.SlimeAdapter; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.fastsearch.TimeoutException; +import com.yahoo.processing.IllegalInputException; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.dispatch.FillInvoker; @@ -65,10 +66,12 @@ public class RpcProtobufFillInvoker extends FillInvoker { @Override protected void sendFillRequest(Result result, String summaryClass) { - if (! documentDb.getDocsumDefinitionSet().hasDocsum(summaryClass)) { - // TODO Vespa 8: - // throw new IllegalArgumentException("invalid summary="+summaryClass); - log.fine("invalid presentation.summary="+summaryClass); + if (summaryClass != null) { + if (summaryClass.equals("")) { + summaryClass = null; + } else if (! documentDb.getDocsumDefinitionSet().hasDocsum(summaryClass)) { + throw new IllegalInputException("invalid presentation.summary=" + summaryClass); + } } ListMap<Integer, FastHit> hitsByNode = hitsByNode(result); diff --git a/container-search/src/main/java/com/yahoo/search/federation/CommonFields.java b/container-search/src/main/java/com/yahoo/search/federation/CommonFields.java deleted file mode 100644 index 5c9b1da8c3d..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/CommonFields.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation; -/** - * A set of string constants for common hit field names. - * - * @author laboisse - * @deprecated do not use - */ -@Deprecated // TODO: Remove on Vespa 8 -public class CommonFields { - - public static final String TITLE = "title"; - public static final String URL = "url"; - public static final String DESCRIPTION = "description"; - public static final String DATE = "date"; - public static final String SIZE = "size"; - public static final String DISP_URL = "dispurl"; - public static final String BASE_URL = "baseurl"; - public static final String MIME_TYPE = "mimetype"; - public static final String RELEVANCY = "relevancy"; - public static final String THUMBNAIL_URL = "thumbnailUrl"; - public static final String THUMBNAIL_WIDTH = "thumbnailWidth"; - public static final String THUMBNAIL_HEIGHT = "thumbnailHeight"; - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java index c0f4060be45..adf03340c6c 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java @@ -58,7 +58,6 @@ import java.util.logging.Logger; import static com.google.common.base.Preconditions.checkNotNull; import static com.yahoo.collections.CollectionUtil.first; -import static com.yahoo.search.federation.StrictContractsConfig.PropagateSourceProperties; /** * This searcher takes a set of sources, looks them up in config and fire off the correct searchchains. @@ -81,34 +80,26 @@ public class FederationSearcher extends ForkingSearcher { private static final List<CompoundName> queryAndHits = ImmutableList.of(Query.OFFSET, Query.HITS); private final SearchChainResolver searchChainResolver; - private final PropagateSourceProperties.Enum propagateSourceProperties; private final SourceRefResolver sourceRefResolver; private final CopyOnWriteHashMap<CompoundKey, CompoundName> map = new CopyOnWriteHashMap<>(); - private final boolean strictSearchchain; private final TargetSelector<?> targetSelector; private final Clock clock = Clock.systemUTC(); @Inject - public FederationSearcher(FederationConfig config, StrictContractsConfig strict, - ComponentRegistry<TargetSelector> targetSelectors) { - this(createResolver(config), strict.searchchains(), strict.propagateSourceProperties(), - resolveSelector(config.targetSelector(), targetSelectors)); + public FederationSearcher(FederationConfig config, ComponentRegistry<TargetSelector> targetSelectors) { + this(createResolver(config), resolveSelector(config.targetSelector(), targetSelectors)); } // for testing public FederationSearcher(ComponentId id, SearchChainResolver searchChainResolver) { - this(searchChainResolver, false, PropagateSourceProperties.EVERY, null); + this(searchChainResolver, null); } private FederationSearcher(SearchChainResolver searchChainResolver, - boolean strictSearchchain, - PropagateSourceProperties.Enum propagateSourceProperties, TargetSelector targetSelector) { this.searchChainResolver = searchChainResolver; sourceRefResolver = new SourceRefResolver(searchChainResolver); - this.strictSearchchain = strictSearchchain; - this.propagateSourceProperties = propagateSourceProperties; this.targetSelector = targetSelector; } @@ -221,13 +212,7 @@ public class FederationSearcher extends ForkingSearcher { if (timeout <= 0) return Optional.empty(); Execution newExecution = new Execution(target.getChain(), execution.context()); - Result result; - if (strictSearchchain) { - query.resetTimeout(); - result = newExecution.search(createFederationQuery(query, query, Window.from(query), timeout, target)); - } else { - result = newExecution.search(cloneFederationQuery(query, Window.from(query), timeout, target)); - } + Result result = newExecution.search(cloneFederationQuery(query, Window.from(query), timeout, target)); target.modifyTargetResult(result); return Optional.of(result); } @@ -265,17 +250,7 @@ public class FederationSearcher extends ForkingSearcher { outgoing.setTimeout(timeout); - switch (propagateSourceProperties) { - case EVERY: - propagatePerSourceQueryProperties(query, outgoing, window, sourceName, providerName, null); - break; - case NATIVE: case ALL: - propagatePerSourceQueryProperties(query, outgoing, window, sourceName, providerName, Query.nativeProperties); - break; - case OFFSET_HITS: - propagatePerSourceQueryProperties(query, outgoing, window, sourceName, providerName, queryAndHits); - break; - } + propagatePerSourceQueryProperties(query, outgoing, window, sourceName, providerName); //TODO: FederationTarget //TODO: only for target produced by this, not others @@ -284,45 +259,13 @@ public class FederationSearcher extends ForkingSearcher { } private void propagatePerSourceQueryProperties(Query original, Query outgoing, Window window, - String sourceName, String providerName, - List<CompoundName> queryProperties) { - if (queryProperties == null) { - outgoing.setHits(window.hits); - outgoing.setOffset(window.offset); - original.properties().listProperties(CompoundName.fromComponents("provider", providerName)).forEach((k, v) -> - outgoing.properties().set(k, v)); - original.properties().listProperties(CompoundName.fromComponents("source", sourceName)).forEach((k, v) -> - outgoing.properties().set(k, v)); - } - else { - for (CompoundName key : queryProperties) { - Object value = getSourceOrProviderProperty(original, key, sourceName, providerName, window.get(key)); - if (value != null) - outgoing.properties().set(key, value); - } - } - } - - private Object getSourceOrProviderProperty(Query query, - CompoundName propertyName, - String sourceName, - String providerName, - Object defaultValue) { - Object result = getProperty(query, new SourceKey(sourceName, propertyName.toString())); - if (result == null) - result = getProperty(query, new ProviderKey(providerName, propertyName.toString())); - if (result == null) - result = defaultValue; - return result; - } - - private Object getProperty(Query query, CompoundKey key) { - CompoundName name = map.get(key); - if (name == null) { - name = new CompoundName(key.toString()); - map.put(key, name); - } - return query.properties().get(name); + String sourceName, String providerName) { + outgoing.setHits(window.hits); + outgoing.setOffset(window.offset); + original.properties().listProperties(CompoundName.fromComponents("provider", providerName)) + .forEach((k, v) -> outgoing.properties().set(k, v)); + original.properties().listProperties(CompoundName.fromComponents("source", sourceName)) + .forEach((k, v) -> outgoing.properties().set(k, v)); } private ErrorMessage missingSearchChainsErrorMessage(List<UnresolvedSearchChainException> unresolvedSearchChainExceptions) { @@ -491,7 +434,7 @@ public class FederationSearcher extends ForkingSearcher { private HitOrderer dirtyCopyIfModifiedOrderer(HitGroup group, HitOrderer orderer) { if (orderer != null) { HitOrderer old = group.getOrderer(); - if ((old == null) || ! orderer.equals(old)) { + if (! orderer.equals(old)) { group.setOrderer(orderer); } } @@ -792,55 +735,6 @@ public class FederationSearcher extends ForkingSearcher { } } - private static class SourceKey extends CompoundKey { - - public static final String SOURCE = "source."; - - SourceKey(String sourceName, String propertyName) { - super(sourceName, propertyName); - } - - @Override - public int hashCode() { - return super.hashCode() ^ 7; - } - - @Override - public boolean equals(Object o) { - return (o instanceof SourceKey) && super.equals(o); - } - - @Override - public String toString() { - return SOURCE + super.toString(); - } - } - - private static class ProviderKey extends CompoundKey { - - public static final String PROVIDER = "provider."; - - ProviderKey(String sourceName, String propertyName) { - super(sourceName, propertyName); - } - - @Override - public int hashCode() { - return super.hashCode() ^ 17; - } - - @Override - public boolean equals(Object o) { - return (o instanceof ProviderKey) && super.equals(o); - } - - @Override - public String toString() { - return PROVIDER + super.toString(); - } - - } - private static class Window { private final int hits; diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java index d9299c07614..694aa76e5f8 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java @@ -48,11 +48,10 @@ public class GroupingExecutor extends Searcher { private final static CompoundName PROP_GROUPINGLIST = newCompoundName(GROUPING_LIST); private final static Logger log = Logger.getLogger(GroupingExecutor.class.getName()); - // TODO Vespa 8: Modify defaults - private static final double DEFAULT_PRECISION_FACTOR = 1; - private static final int DEFAULT_MAX_GROUPS = -1; - private static final int DEFAULT_MAX_HITS = -1; - private static final long DEFAULT_GLOBAL_MAX_GROUPS = -1; + private static final double DEFAULT_PRECISION_FACTOR = 2.0; + private static final int DEFAULT_MAX_GROUPS = 10; + private static final int DEFAULT_MAX_HITS = 10; + private static final long DEFAULT_GLOBAL_MAX_GROUPS = 10000; /** * Constructs a new instance of this searcher without configuration. diff --git a/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java b/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java index 64e7403fa1a..c6906ea7566 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java +++ b/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java @@ -1,9 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.handler; -import com.google.common.util.concurrent.ListenableFuture; import com.yahoo.collections.ListMap; -import com.yahoo.concurrent.CompletableFutures; import com.yahoo.container.handler.Coverage; import com.yahoo.container.handler.Timing; import com.yahoo.container.jdisc.ExtendedResponse; @@ -77,22 +75,6 @@ public class HttpSearchResponse extends ExtendedResponse { } } - /** @deprecated Use {@link #asyncRender(OutputStream)} instead */ - @Deprecated(forRemoval = true, since = "7") - public ListenableFuture<Boolean> waitableRender(OutputStream stream) throws IOException { - return waitableRender(result, query, rendererCopy, stream); - } - - /** @deprecated Use {@link #asyncRender(Result, Query, Renderer, OutputStream)} instead */ - @Deprecated(forRemoval = true, since = "7") - @SuppressWarnings("removal") - public static ListenableFuture<Boolean> waitableRender(Result result, - Query query, - Renderer<Result> renderer, - OutputStream stream) throws IOException { - return CompletableFutures.toGuavaListenableFuture(asyncRender(result, query, renderer, stream)); - } - public CompletableFuture<Boolean> asyncRender(OutputStream stream) { return asyncRender(result, query, rendererCopy, stream); } diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 3eadf29489e..2c26a302ece 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -3,14 +3,11 @@ package com.yahoo.search.handler; import ai.vespa.cloud.ZoneInfo; import com.yahoo.collections.Tuple2; -import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.Vtag; import com.yahoo.component.annotation.Inject; import com.yahoo.component.chain.Chain; import com.yahoo.component.provider.ComponentRegistry; -import com.yahoo.container.QrSearchersConfig; -import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.core.ContainerHttpConfig; import com.yahoo.container.handler.threadpool.ContainerThreadPool; import com.yahoo.container.jdisc.AclMapping; @@ -20,11 +17,9 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.container.jdisc.RequestHandlerSpec; import com.yahoo.container.jdisc.VespaHeaders; -import com.yahoo.container.logging.AccessLog; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Request; -import com.yahoo.language.Linguistics; import com.yahoo.language.process.Embedder; import com.yahoo.language.provider.DefaultEmbedderProvider; import com.yahoo.net.HostName; @@ -36,12 +31,9 @@ import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; -import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.query.context.QueryContext; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; -import com.yahoo.search.query.profile.config.QueryProfileConfigurer; -import com.yahoo.search.query.profile.config.QueryProfilesConfig; import com.yahoo.search.query.properties.DefaultProperties; import com.yahoo.search.query.ranking.SoftTimeout; import com.yahoo.search.result.ErrorMessage; @@ -52,8 +44,6 @@ import com.yahoo.search.statistics.ElapsedTime; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.SlimeUtils; -import com.yahoo.statistics.Statistics; -import com.yahoo.vespa.configdefinition.SpecialtokensConfig; import com.yahoo.yolean.Exceptions; import com.yahoo.yolean.trace.TraceNode; @@ -130,119 +120,6 @@ public class SearchHandler extends LoggingRequestHandler { zoneInfo); } - /** - * @deprecated Use the @Inject annotated constructor instead. - */ - @Deprecated // Vespa 8 - public SearchHandler(Metric metric, - ContainerThreadPool threadpool, - CompiledQueryProfileRegistry queryProfileRegistry, - ContainerHttpConfig config, - Embedder embedder, - ExecutionFactory executionFactory) { - this(metric, threadpool.executor(), queryProfileRegistry, embedder, executionFactory, - config.numQueriesToTraceOnDebugAfterConstruction(), - config.hostResponseHeaderKey().equals("") ? Optional.empty() : Optional.of(config.hostResponseHeaderKey()), - ZoneInfo.defaultInfo()); - } - - /** - * @deprecated Use the @Inject annotated constructor instead. - */ - @Deprecated // Vespa 8 - public SearchHandler(Statistics statistics, - Metric metric, - ContainerThreadPool threadpool, - CompiledQueryProfileRegistry queryProfileRegistry, - ContainerHttpConfig config, - Embedder embedder, - ExecutionFactory executionFactory) { - this(metric, threadpool.executor(), queryProfileRegistry, embedder, executionFactory, - config.numQueriesToTraceOnDebugAfterConstruction(), - config.hostResponseHeaderKey().equals("") ? Optional.empty() : Optional.of(config.hostResponseHeaderKey()), - ZoneInfo.defaultInfo()); - } - - /** - * @deprecated Use the @Inject annotated constructor instead. - */ - @Deprecated // Vespa 8 - public SearchHandler(Statistics statistics, - Metric metric, - ContainerThreadPool threadpool, - AccessLog ignored, - CompiledQueryProfileRegistry queryProfileRegistry, - ContainerHttpConfig config, - ExecutionFactory executionFactory) { - this(statistics, metric, threadpool.executor(), ignored, queryProfileRegistry, config, executionFactory); - } - - /** - * @deprecated Use the @Inject annotated constructor instead. - */ - @Deprecated // Vespa 8 - public SearchHandler(Statistics statistics, - Metric metric, - Executor executor, - AccessLog ignored, - CompiledQueryProfileRegistry queryProfileRegistry, - ContainerHttpConfig containerHttpConfig, - ExecutionFactory executionFactory) { - this(metric, executor, queryProfileRegistry, Embedder.throwsOnUse, executionFactory, - containerHttpConfig.numQueriesToTraceOnDebugAfterConstruction(), - containerHttpConfig.hostResponseHeaderKey().equals("") ? Optional.empty() - : Optional.of(containerHttpConfig.hostResponseHeaderKey()), - ZoneInfo.defaultInfo()); - } - - /** - * @deprecated Use the @Inject annotated constructor instead. - */ - @Deprecated // Vespa 8 - public SearchHandler(Statistics statistics, - Metric metric, - Executor executor, - AccessLog ignored, - QueryProfilesConfig queryProfileConfig, - ContainerHttpConfig containerHttpConfig, - ExecutionFactory executionFactory) { - this(metric, executor, QueryProfileConfigurer.createFromConfig(queryProfileConfig).compile(), - Embedder.throwsOnUse, executionFactory, - containerHttpConfig.numQueriesToTraceOnDebugAfterConstruction(), - containerHttpConfig.hostResponseHeaderKey().equals("") ? Optional.empty() - : Optional.of( containerHttpConfig.hostResponseHeaderKey()), - ZoneInfo.defaultInfo()); - } - - /** - * @deprecated Use the @Inject annotated constructor instead. - */ - @Deprecated // Vespa 8 - public SearchHandler(Statistics statistics, - Metric metric, - Executor executor, - AccessLog ignored, - CompiledQueryProfileRegistry queryProfileRegistry, - ExecutionFactory executionFactory, - Optional<String> hostResponseHeaderKey) { - this(metric, executor, queryProfileRegistry, toRegistry(Embedder.throwsOnUse), - executionFactory, 0, hostResponseHeaderKey, - ZoneInfo.defaultInfo()); - } - - private SearchHandler(Metric metric, - Executor executor, - CompiledQueryProfileRegistry queryProfileRegistry, - Embedder embedder, - ExecutionFactory executionFactory, - long numQueriesToTraceOnDebugAfterStartup, - Optional<String> hostResponseHeaderKey, - ZoneInfo zoneInfo) { - this(metric, executor, queryProfileRegistry, toRegistry(embedder), - executionFactory, numQueriesToTraceOnDebugAfterStartup, hostResponseHeaderKey, - ZoneInfo.defaultInfo()); - } - private SearchHandler(Metric metric, Executor executor, CompiledQueryProfileRegistry queryProfileRegistry, @@ -266,30 +143,6 @@ public class SearchHandler extends LoggingRequestHandler { this.zoneInfo = zoneInfo; } - /** @deprecated use the other constructor */ - @Deprecated // TODO: Remove on Vespa 8 - public SearchHandler(ChainsConfig chainsConfig, - IndexInfoConfig indexInfo, - QrSearchersConfig clusters, - SpecialtokensConfig specialtokens, - Statistics statistics, - Linguistics linguistics, - Metric metric, - ComponentRegistry<Renderer> renderers, - Executor executor, - AccessLog accessLog, - QueryProfilesConfig queryProfileConfig, - ComponentRegistry<Searcher> searchers, - ContainerHttpConfig containerHttpConfig) { - this(statistics, - metric, - executor, - accessLog, - queryProfileConfig, - containerHttpConfig, - new ExecutionFactory(chainsConfig, indexInfo, clusters, searchers, specialtokens, linguistics, renderers, executor)); - } - Metric metric() { return metric; } private static int examineExecutor(Executor executor) { @@ -716,12 +569,6 @@ public class SearchHandler extends LoggingRequestHandler { return Collections.unmodifiableMap(map); } - private static ComponentRegistry<Embedder> toRegistry(Embedder embedder) { - ComponentRegistry<Embedder> emb = new ComponentRegistry<>(); - emb.register(new ComponentId(Embedder.defaultEmbedderId), embedder); - return emb; - } - } diff --git a/container-search/src/main/java/com/yahoo/search/query/Presentation.java b/container-search/src/main/java/com/yahoo/search/query/Presentation.java index 31641b5c2f0..afa87eb4a06 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Presentation.java +++ b/container-search/src/main/java/com/yahoo/search/query/Presentation.java @@ -75,7 +75,7 @@ public class Presentation implements Cloneable { private boolean timing = false; /** Whether to renders tensors in short form */ - private boolean tensorShortForm = false; + private boolean tensorShortForm = true; /** Set of explicitly requested summary fields, instead of summary classes */ private Set<String> summaryFields = LazySet.newHashSet(); @@ -180,7 +180,7 @@ public class Presentation implements Cloneable { * Returns whether tensors should use short form in JSON and textual representations, see * <a href="https://docs.vespa.ai/en/reference/document-json-format.html#tensor">https://docs.vespa.ai/en/reference/document-json-format.html#tensor</a> * and <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form">https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form</a>. - * Default is false. + * Default is true. */ public boolean getTensorShortForm() { return tensorShortForm; } diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index ccf24a13f34..e8ea6f88fce 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -519,9 +519,6 @@ public class SelectParser implements Parser { if (TARGET_NUM_HITS.equals(annotation_name)) { weakAnd.setN((int)(annotation_value.asDouble())); } - if (SCORE_THRESHOLD.equals(annotation_name)) { - weakAnd.setScoreThreshold((int)(annotation_value.asDouble())); - } }); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java index 84246b1eb25..b870b32da77 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java @@ -135,4 +135,8 @@ public class CompiledQueryProfileRegistry extends ComponentRegistry<CompiledQuer return registry; } + public static CompiledQueryProfileRegistry fromConfig(QueryProfilesConfig config) { + return QueryProfileConfigurer.createFromConfig(config).compile(); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java index c3bd4f7b962..31278af9579 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java @@ -3,7 +3,7 @@ package com.yahoo.search.query.profile.config; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; -import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.search.query.profile.DimensionValues; import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; @@ -19,27 +19,10 @@ import java.util.Set; /** * @author bratseth */ -@SuppressWarnings("removal") // TODO Vespa 8: remove -public class QueryProfileConfigurer implements ConfigSubscriber.SingleSubscriber<QueryProfilesConfig> { +public class QueryProfileConfigurer { - private final ConfigSubscriber subscriber = new ConfigSubscriber(); - - private volatile QueryProfileRegistry currentRegistry; - - public QueryProfileConfigurer(String configId) { - subscriber.subscribe(this, QueryProfilesConfig.class, configId); - } - - /** Returns the registry created by the last occurring call to configure */ - public QueryProfileRegistry getCurrentRegistry() { return currentRegistry; } - - private void setCurrentRegistry(QueryProfileRegistry registry) { - this.currentRegistry=registry; - } - - public void configure(QueryProfilesConfig config) { - QueryProfileRegistry registry = createFromConfig(config); - setCurrentRegistry(registry); + public static QueryProfileRegistry createFromConfigId(String configId) { + return createFromConfig(ConfigGetter.getConfig(QueryProfilesConfig.class, configId)); } public static QueryProfileRegistry createFromConfig(QueryProfilesConfig config) { @@ -69,11 +52,6 @@ public class QueryProfileConfigurer implements ConfigSubscriber.SingleSubscriber return registry; } - /** Stop subscribing from this configurer */ - public void shutdown() { - subscriber.close(); - } - private static void createProfile(QueryProfilesConfig.Queryprofile config, QueryProfileRegistry registry) { QueryProfile profile = new QueryProfile(new ComponentId(config.id()), config.id(), registry); try { diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index 7d1a62310a3..4cced8d7923 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -36,11 +36,6 @@ public class QueryProperties extends Properties { private final CompiledQueryProfileRegistry profileRegistry; private final Map<String, Embedder> embedders; - @Deprecated // TODO: Remove on Vespa 8 - public QueryProperties(Query query, CompiledQueryProfileRegistry profileRegistry, Embedder embedder) { - this(query, profileRegistry, Map.of(Embedder.defaultEmbedderId, embedder)); - } - public QueryProperties(Query query, CompiledQueryProfileRegistry profileRegistry, Map<String, Embedder> embedders) { this.query = query; this.profileRegistry = profileRegistry; @@ -379,7 +374,7 @@ public class QueryProperties extends Properties { try { query.getRanking().getFeatures().put(key, Double.parseDouble(valueString)); } - catch (IllegalArgumentException e) { // TODO: Throw instead on Vespa 8 + catch (IllegalArgumentException e) { query.getRanking().getFeatures().put(key, valueString); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryPropertyAliases.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryPropertyAliases.java index 4207080b3e5..6ea371f12a3 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryPropertyAliases.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryPropertyAliases.java @@ -24,8 +24,7 @@ public class QueryPropertyAliases extends PropertyAliases { @Override protected CompoundName unalias(CompoundName nameOrAlias) { if (nameOrAlias.first().equalsIgnoreCase("rankfeature") - || nameOrAlias.first().equalsIgnoreCase("input") - || nameOrAlias.first().equalsIgnoreCase("featureoverride")) { // deprecated: TODO Remove on Vespa 8 + || nameOrAlias.first().equalsIgnoreCase("input")) { if (nameOrAlias.size() > 1) // only alias the prefix return nameOrAlias.rest().prepend("ranking", "features"); else diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java b/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java index 4e92c3696fd..5fed5f13c64 100644 --- a/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java @@ -3,9 +3,7 @@ package com.yahoo.search.query.ranking; import com.yahoo.fs4.MapEncoder; import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.Query; import com.yahoo.search.query.Ranking; -import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.tensor.Tensor; import com.yahoo.text.JSON; @@ -27,12 +25,6 @@ public class RankFeatures implements Cloneable { private final Ranking parent; private final Map<String, Object> features; - /** @deprecated pass the parent */ - @Deprecated // TODO: Remove on Vespa 8 - public RankFeatures() { - this(new Ranking(new Query())); - } - public RankFeatures(Ranking parent) { this(parent, new LinkedHashMap<>()); } @@ -69,18 +61,6 @@ public class RankFeatures implements Cloneable { features.put(name, value); } - /** - * Returns a rank feature as a string by full name or null if not set - * - * @deprecated use getTensor (or getDouble) instead - */ - @Deprecated // TODO: Remove on Vespa 8 - public String get(String name) { - Object value = features.get(name); - if (value == null) return null; - return value.toString(); - } - /** Returns this value as either a Double, Tensor or String. Returns null if the value is not set. */ public Object getObject(String name) { return features.get(name); diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/QueryTreeUtil.java b/container-search/src/main/java/com/yahoo/search/querytransform/QueryTreeUtil.java deleted file mode 100644 index 1cf6396468e..00000000000 --- a/container-search/src/main/java/com/yahoo/search/querytransform/QueryTreeUtil.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.querytransform; - -import com.yahoo.prelude.query.AndItem; -import com.yahoo.prelude.query.Item; -import com.yahoo.prelude.query.QueryCanonicalizer; -import com.yahoo.search.Query; -import com.yahoo.search.query.QueryTree; - -/** - * Utility class for manipulating a QueryTree. - * - * @author geirst - * @deprecated use QueryTree.and instead // TODO: Remove on Vespa 8 - */ -@Deprecated -public class QueryTreeUtil { - - /** - * Adds the given item to this query - * - * @return the new root of the query tree - */ - static public Item andQueryItemWithRoot(Query query, Item item) { - return andQueryItemWithRoot(query.getModel().getQueryTree(), item); - } - - /** - * Adds the given item to this query - * - * @return the new root of the query tree - */ - static public Item andQueryItemWithRoot(QueryTree tree, Item item) { - return tree.and(item); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index d5303d205a0..a9195aaafa7 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -57,7 +57,6 @@ import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.util.ArrayDeque; import java.util.Arrays; -import java.util.Collections; import java.util.Deque; import java.util.Map; import java.util.Optional; @@ -127,18 +126,18 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private Deque<Integer> renderedChildren; static class FieldConsumerSettings { boolean debugRendering = false; - boolean jsonDeepMaps = false; - boolean jsonWsets = false; - boolean jsonMapsAll = false; + boolean jsonDeepMaps = true; + boolean jsonWsets = true; + boolean jsonMapsAll = true; boolean jsonWsetsAll = false; boolean tensorShortForm = false; boolean convertDeep() { return (jsonDeepMaps || jsonWsets); } void init() { this.debugRendering = false; - this.jsonDeepMaps = false; - this.jsonWsets = false; - this.jsonMapsAll = false; - this.jsonWsetsAll = false; + this.jsonDeepMaps = true; + this.jsonWsets = true; + this.jsonMapsAll = true; + this.jsonWsetsAll = true; this.tensorShortForm = false; } void getSettings(Query q) { @@ -148,11 +147,11 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { } var props = q.properties(); this.debugRendering = props.getBoolean(DEBUG_RENDERING_KEY, false); - this.jsonDeepMaps = props.getBoolean(WRAP_DEEP_MAPS, false); - this.jsonWsets = props.getBoolean(WRAP_WSETS, false); + this.jsonDeepMaps = props.getBoolean(WRAP_DEEP_MAPS, true); + this.jsonWsets = props.getBoolean(WRAP_WSETS, true); // we may need more fine tuning, but for now use the same query parameters here: - this.jsonMapsAll = props.getBoolean(WRAP_DEEP_MAPS, false); - this.jsonWsetsAll = props.getBoolean(WRAP_WSETS, false); + this.jsonMapsAll = props.getBoolean(WRAP_DEEP_MAPS, true); + this.jsonWsetsAll = props.getBoolean(WRAP_WSETS, true); this.tensorShortForm = q.getPresentation().getTensorShortForm(); } } @@ -174,18 +173,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { generatorFactory.setCodec(createJsonCodec()); } - /** - * Create the codec used for rendering instances of {@link TreeNode}. This - * method will be invoked when creating the first renderer instance, but not - * for each fresh clone used by individual results. - * - * @deprecated Will be removed in Vespa 8. Override the individual render methods of {@link JsonRenderer} to alter - * rendering behaviour. Override {@link #createFieldConsumer(boolean)} and sub-class {@link FieldConsumer} - * to alter rendering of hit fields. - * @return an object mapper for the internal JsonFactory - */ - @Deprecated(forRemoval = true, since = "7") // TODO Vespa 8 make private - protected static ObjectMapper createJsonCodec() { + private static ObjectMapper createJsonCodec() { return new ObjectMapper().disable(FLUSH_AFTER_WRITE_VALUE); } @@ -545,13 +533,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { return createFieldConsumer(generator, fieldConsumerSettings); } - /** @deprecated Will be removed in Vespa 8. Use {@link #createFieldConsumer(boolean)} instead. */ - @Deprecated(forRemoval = true, since = "7") // TODO Vespa 8 remove method - protected FieldConsumer createFieldConsumer(JsonGenerator generator, boolean debugRendering) { - fieldConsumerSettings.debugRendering = debugRendering; - return createFieldConsumer(generator, fieldConsumerSettings); - } - private FieldConsumer createFieldConsumer(JsonGenerator generator, FieldConsumerSettings settings) { return new FieldConsumer(generator, settings); } @@ -574,26 +555,12 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private final FieldConsumerSettings settings; private MutableBoolean hasFieldsField; - /** @deprecated Will be removed in Vespa 8. Use {@link #FieldConsumer(boolean, boolean, boolean)} instead. */ - @Deprecated(forRemoval = true, since = "7") // TODO Vespa 8 Remove - public FieldConsumer(JsonGenerator generator, boolean debugRendering) { - this(generator, debugRendering, false); - } - - /** @deprecated Will be removed in Vespa 8. Use {@link #FieldConsumer(boolean, boolean, boolean)} instead. */ - @Deprecated(forRemoval = true, since = "7") // TODO Vespa 8 Remove - public FieldConsumer(JsonGenerator generator, boolean debugRendering, boolean tensorShortForm) { - this(generator, debugRendering, tensorShortForm, false); - } - /** Invoke this from your constructor when sub-classing {@link FieldConsumer} */ protected FieldConsumer(boolean debugRendering, boolean tensorShortForm, boolean jsonMaps) { this(null, debugRendering, tensorShortForm, jsonMaps); } - /** @deprecated Will be removed in Vespa 8. Use {@link #FieldConsumer(boolean, boolean, boolean)} instead. */ - @Deprecated(forRemoval = true, since = "7") // TODO Vespa 8 remove - public FieldConsumer(JsonGenerator generator, boolean debugRendering, boolean tensorShortForm, boolean jsonMaps) { + private FieldConsumer(JsonGenerator generator, boolean debugRendering, boolean tensorShortForm, boolean jsonMaps) { this.generator = generator; this.settings = new FieldConsumerSettings(); this.settings.debugRendering = debugRendering; diff --git a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java index 7096d937959..9699e9a7448 100644 --- a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java +++ b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java @@ -3,9 +3,7 @@ package com.yahoo.search.result; import com.google.common.base.Predicate; import com.google.common.collect.Iterables; -import com.google.common.util.concurrent.ListenableFuture; import com.yahoo.collections.ListenableArrayList; -import com.yahoo.concurrent.CompletableFutures; import com.yahoo.net.URI; import com.yahoo.prelude.fastsearch.SortDataHitSorter; import com.yahoo.processing.response.ArrayDataList; @@ -964,13 +962,6 @@ public class HitGroup extends Hit implements DataList<Hit>, Cloneable, Iterable< @Override public IncomingData<Hit> incoming() { return incomingHits; } - @Override - @SuppressWarnings("removal") - @Deprecated(forRemoval = true, since = "7") - public ListenableFuture<DataList<Hit>> complete() { - return CompletableFutures.toGuavaListenableFuture(completedFuture); - } - @Override public CompletableFuture<DataList<Hit>> completeFuture() { return completedFuture; } @Override diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index 94ff9745ea6..0a85c636213 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -41,10 +41,6 @@ import java.util.concurrent.Executors; */ public class Execution extends com.yahoo.processing.execution.Execution { - /** @deprecated - applications should define their own summary class instead */ - @Deprecated(since = "7", forRemoval = true) - public static final String ATTRIBUTEPREFETCH = "attributeprefetch"; - /** * The execution context is the search chain's current view of the indexes, * search chain registrys, etc. Searcher instances may set values here to @@ -142,13 +138,6 @@ public class Execution extends com.yahoo.processing.execution.Execution { this(searchChainRegistry, indexFacts, SchemaInfo.empty(), tokenRegistry, rendererRegistry, linguistics, executor); } - /** @deprecated pass an executor */ - @Deprecated // TODO: Remove on Vespa 8 - public Context(SearchChainRegistry searchChainRegistry, IndexFacts indexFacts, - SpecialTokenRegistry tokenRegistry, RendererRegistry rendererRegistry, Linguistics linguistics) { - this(searchChainRegistry, indexFacts, SchemaInfo.empty(), tokenRegistry, rendererRegistry, linguistics, Runnable::run); - } - /** Creates a Context instance where everything except the given arguments is empty. This is for unit testing.*/ public static Context createContextStub() { return createContextStub(null, null, SchemaInfo.empty(), null); diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java index 69229c38818..e99dafef886 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java @@ -104,18 +104,6 @@ public class ExecutionFactory extends AbstractComponent { this(chainsConfig, indexInfo, SchemaInfo.empty(), clusters, searchers, specialTokens, linguistics, renderers, executor); } - /** @deprecated pass the container threadpool */ - @Deprecated // TODO: Remove on Vespa 8 - public ExecutionFactory(ChainsConfig chainsConfig, - IndexInfoConfig indexInfo, - QrSearchersConfig clusters, - ComponentRegistry<Searcher> searchers, - SpecialtokensConfig specialTokens, - Linguistics linguistics, - ComponentRegistry<Renderer> renderers) { - this(chainsConfig, indexInfo, SchemaInfo.empty(), clusters, searchers, specialTokens, linguistics, renderers, null); - } - private SearchChainRegistry createSearchChainRegistry(ComponentRegistry<Searcher> searchers, ChainsConfig chainsConfig) { SearchChainRegistry searchChainRegistry = new SearchChainRegistry(searchers); diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/FederationOptions.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/FederationOptions.java index 57953a915b6..bc9de4ebc2d 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/FederationOptions.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/FederationOptions.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.searchchain.model.federation; -import net.jcip.annotations.Immutable; - import java.util.Objects; /** @@ -11,7 +9,6 @@ import java.util.Objects; * * @author Tony Vaagenes */ -@Immutable public class FederationOptions implements Cloneable { private final Boolean optional; diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java index a12456f5354..3c2767430c7 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java @@ -12,14 +12,11 @@ import java.util.Collection; import java.util.List; import java.util.Objects; -import net.jcip.annotations.Immutable; - /** * Specifies how a local provider is to be set up. * * @author Tony Vaagenes */ -@Immutable public class LocalProviderSpec { public static final Collection<ChainedComponentModel> searcherModels = diff --git a/container-search/src/main/java/com/yahoo/search/statistics/PeakQpsSearcher.java b/container-search/src/main/java/com/yahoo/search/statistics/PeakQpsSearcher.java deleted file mode 100644 index 2823a7d74e1..00000000000 --- a/container-search/src/main/java/com/yahoo/search/statistics/PeakQpsSearcher.java +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.statistics; - -import com.yahoo.collections.Tuple2; -import com.yahoo.concurrent.ThreadLocalDirectory; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Callback; -import com.yahoo.statistics.Handle; -import com.yahoo.statistics.Statistics; -import com.yahoo.statistics.Value; - -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Deque; -import java.util.List; -import java.util.ListIterator; - -/** - * Aggregate peak qps and expose through meta hits and/or log events. - * - * @author Steinar Knutsen - * @deprecated Will be removed on Vespa 8 - */ -@Deprecated -public class PeakQpsSearcher extends Searcher { - - private final ThreadLocalDirectory<Deque<QueryRatePerSecond>, Long> directory; - private final Value qpsStatistics; - private final CompoundName propertyName; - private final boolean useMetaHit; - - /** - * Meta hit which carries the peak qps and mean qps since the last time this - * data was requested. The URI is always "meta:qps". The data is stored as - * Number subclasses in the fields named by the fields PEAK_QPS and MEAN_QPS - * in the QpsHit class. - */ - public static class QpsHit extends Hit { - - /** The name of the field containing mean QPS since the last measurement. */ - public static final String MEAN_QPS = "mean_qps"; - - /** The name of the field containing peak QPS since the last measurement. */ - public static final String PEAK_QPS = "peak_qps"; - public static final String SCHEME = "meta"; - - public QpsHit(Integer peakQps, Double meanQps) { - super(SCHEME + ":qps"); - setField(PEAK_QPS, peakQps); - setField(MEAN_QPS, meanQps); - } - - public boolean isMeta() { - return true; - } - - @Override - public String toString() { - return "QPS hit: Peak QPS " + getField(PEAK_QPS) + ", mean QPS " + getField(MEAN_QPS) + "."; - } - - } - - static class QueryRatePerSecond { - long when; - int howMany; - - QueryRatePerSecond(long when) { - this.when = when; - this.howMany = 0; - } - - void add(int x) { - howMany += x; - } - - void increment() { - howMany += 1; - } - - @Override - public String toString() { - return "QueryRatePerSecond(" + when + ": " + howMany + ")"; - } - } - - static class QueryRate implements - ThreadLocalDirectory.Updater<Deque<QueryRatePerSecond>, Long> { - @Override - public Deque<QueryRatePerSecond> update(Deque<QueryRatePerSecond> current, Long when) { - QueryRatePerSecond last = current.peekLast(); - if (last == null || last.when != when) { - last = new QueryRatePerSecond(when); - current.addLast(last); - } - last.increment(); - return current; - } - - @Override - public Deque<QueryRatePerSecond> createGenerationInstance(Deque<QueryRatePerSecond> previous) { - if (previous == null) { - return new ArrayDeque<>(); - } else { - return new ArrayDeque<>(previous.size()); - } - } - } - - private class Fetcher implements Callback { - @Override - public void run(Handle h, boolean firstRun) { - List<Deque<QueryRatePerSecond>> data = directory.fetch(); - List<QueryRatePerSecond> chewed = merge(data); - for (QueryRatePerSecond qps : chewed) { - qpsStatistics.put(qps.howMany); - } - } - } - - public PeakQpsSearcher(MeasureQpsConfig config, Statistics manager) { - directory = createDirectory(); - MeasureQpsConfig.Outputmethod.Enum method = config.outputmethod(); - if (method == MeasureQpsConfig.Outputmethod.METAHIT) { - useMetaHit = true; - propertyName = new CompoundName(config.queryproperty()); - qpsStatistics = null; - } else if (method == MeasureQpsConfig.Outputmethod.STATISTICS) { - String event = config.eventname(); - if (event == null || event.isEmpty()) { - event = getId().getName(); - event = event.replace('.', '_'); - } - qpsStatistics = new Value(event, manager, new Value.Parameters() - .setAppendChar('_').setLogMax(true).setLogMean(true) - .setLogMin(false).setLogRaw(false).setNameExtension(true) - .setCallback(new Fetcher())); - useMetaHit = false; - propertyName = null; - } else { - throw new IllegalStateException("Config definition out of sync with implementation." + - " No way to create output for method " + method + "."); - } - } - - static ThreadLocalDirectory<Deque<QueryRatePerSecond>, Long> createDirectory() { - return new ThreadLocalDirectory<>(new QueryRate()); - } - - static List<QueryRatePerSecond> merge(List<Deque<QueryRatePerSecond>> measurements) { - List<QueryRatePerSecond> rates = new ArrayList<>(); - while (measurements.size() > 0) { - Deque<Deque<QueryRatePerSecond>> consumeFrom = new ArrayDeque<>(measurements.size()); - long current = Long.MAX_VALUE; - for (ListIterator<Deque<QueryRatePerSecond>> i = measurements.listIterator(); i.hasNext();) { - Deque<QueryRatePerSecond> deck = i.next(); - if (deck.size() == 0) { - i.remove(); - continue; - } - QueryRatePerSecond threadData = deck.peekFirst(); - if (threadData.when < current) { - consumeFrom.clear(); - current = threadData.when; - consumeFrom.add(deck); - } else if (threadData.when == current) { - consumeFrom.add(deck); - } - } - if (consumeFrom.size() > 0) { - rates.add(consume(consumeFrom)); - } - } - return rates; - } - - private static QueryRatePerSecond consume(Deque<Deque<QueryRatePerSecond>> consumeFrom) { - Deque<QueryRatePerSecond> valueQueue = consumeFrom.pop(); - QueryRatePerSecond value = valueQueue.pop(); - QueryRatePerSecond thisSecond = new QueryRatePerSecond(value.when); - thisSecond.add(value.howMany); - while (consumeFrom.size() > 0) { - valueQueue = consumeFrom.pop(); - value = valueQueue.pop(); - thisSecond.add(value.howMany); - } - return thisSecond; - - } - - @Override - public Result search(Query query, Execution execution) { - Result r; - long when = query.getStartTime() / 1000L; - Hit meta = null; - directory.update(when); - if (useMetaHit) { - if (query.properties().getBoolean(propertyName, false)) { - List<QueryRatePerSecond> l = merge(directory.fetch()); - Tuple2<Integer, Double> maxAndMean = maxAndMean(l); - meta = new QpsHit(maxAndMean.first, maxAndMean.second); - } - } - r = execution.search(query); - if (meta != null) { - r.hits().add(meta); - } - return r; - } - - private Tuple2<Integer, Double> maxAndMean(List<QueryRatePerSecond> l) { - int max = Integer.MIN_VALUE; - double sum = 0.0d; - if (l.size() == 0) { - return new Tuple2<>(0, 0.0); - } - for (QueryRatePerSecond qps : l) { - sum += qps.howMany; - if (qps.howMany > max) { - max = qps.howMany; - } - } - return new Tuple2<>(max, sum / (double) l.size()); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/statistics/TimingSearcher.java b/container-search/src/main/java/com/yahoo/search/statistics/TimingSearcher.java deleted file mode 100644 index 5d036b8fa20..00000000000 --- a/container-search/src/main/java/com/yahoo/search/statistics/TimingSearcher.java +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.statistics; - -import com.yahoo.component.ComponentId; -import com.yahoo.component.chain.dependencies.Before; -import com.yahoo.search.statistics.TimingSearcherConfig.Timer; -import com.yahoo.prelude.Ping; -import com.yahoo.prelude.Pong; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.cluster.PingableSearcher; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.search.statistics.TimeTracker.Activity; -import com.yahoo.statistics.Statistics; -import com.yahoo.statistics.Value; - -/** - * A searcher which is intended to be useful as a general probe for - * measuring time consumption a search chain. - * - * @author Steinar Knutsen - * @deprecated Will be removed on Vespa 8 - */ -@Before("rawQuery") -@Deprecated -public class TimingSearcher extends PingableSearcher { - - private Value measurements; - private final boolean measurePing; - private final boolean measureSearch; - private final boolean measureFill; - private static final Parameters defaultParameters = new Parameters(null, Activity.SEARCH); - - public static class Parameters { - final String eventName; - final Activity pathToSample; - - public Parameters(String eventName, Activity pathToSample) { - super(); - this.eventName = eventName; - this.pathToSample = pathToSample; - } - } - - TimingSearcher(ComponentId id, Parameters setUp, Statistics manager) { - super(id); - if (setUp == null) { - setUp = defaultParameters; - } - String eventName = setUp.eventName; - if (eventName == null || "".equals(eventName)) { - eventName = id.getName(); - } - measurements = new Value(eventName, manager, new Value.Parameters() - .setNameExtension(true).setLogMax(true).setLogMin(true) - .setLogMean(true).setLogSum(true).setLogInsertions(true) - .setAppendChar('_')); - - measurePing = setUp.pathToSample == Activity.PING; - measureSearch = setUp.pathToSample == Activity.SEARCH; - measureFill = setUp.pathToSample == Activity.FILL; - } - - public TimingSearcher(ComponentId id, TimingSearcherConfig config, Statistics manager) { - this(id, buildParameters(config, id.getName()), manager); - } - - private static Parameters buildParameters( - TimingSearcherConfig config, String searcherName) { - for (int i = 0; i < config.timer().size(); ++i) { - Timer t = config.timer(i); - if (t.name().equals(searcherName)) { - return buildParameters(t); - } - } - return null; - } - - private static Parameters buildParameters(Timer t) { - Activity m; - Timer.Measure.Enum toSample = t.measure(); - if (toSample == Timer.Measure.FILL) { - m = Activity.FILL; - } else if (toSample == Timer.Measure.PING) { - m = Activity.PING; - } else { - m = Activity.SEARCH; - } - return new Parameters(t.eventname(), m); - } - - private long preMeasure(boolean doIt) { - if (doIt) { - return System.currentTimeMillis(); - } else { - return 0L; - } - } - - private void postMeasure(boolean doIt, long start) { - if (doIt) { - long elapsed = System.currentTimeMillis() - start; - measurements.put(elapsed); - } - } - - @Override - public void fill(Result result, String summaryClass, Execution execution) { - long start = preMeasure(measureFill); - super.fill(result, summaryClass, execution); - postMeasure(measureFill, start); - } - - @Override - public Pong ping(Ping ping, Execution execution) { - long start = preMeasure(measurePing); - Pong pong = execution.ping(ping); - postMeasure(measurePing, start); - return pong; - } - - @Override - public Result search(Query query, Execution execution) { - long start = preMeasure(measureSearch); - Result result = execution.search(query); - postMeasure(measureSearch, start); - return result; - } - - /** - * This method is only included for testing. - */ - public void setMeasurements(Value measurements) { - this.measurements = measurements; - } - - @Override - public void deconstruct() { - // avoid dangling, duplicate loggers - measurements.cancel(); - super.deconstruct(); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java index 66ad7b84608..e1400e4f860 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java +++ b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java @@ -60,13 +60,13 @@ public class MinimalQueryInserter extends Searcher { } private static boolean warmup(Linguistics linguistics) { - Query query = new Query("search/?yql=select%20*%20from%20sources%20where%20title%20contains%20'xyz';"); + Query query = new Query("search/?yql=select%20*%20from%20sources%20where%20title%20contains%20'xyz'"); Result result = insertQuery(query, new ParserEnvironment().setLinguistics(linguistics)); if (result != null) { log.warning("Warmup code trigger an error. Error = " + result.toString()); return false; } - if ( ! "select * from sources where title contains \"xyz\";".equals(query.yqlRepresentation())) { + if ( ! "select * from sources where title contains \"xyz\"".equals(query.yqlRepresentation())) { log.warning("Warmup code generated unexpected yql: " + query.yqlRepresentation()); return false; } diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index 194863e3129..96f86354ba5 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -1042,7 +1042,7 @@ public class VespaSerializer { } private boolean needsAnnotationBlock(WeakAndItem item) { - return nonDefaultScoreThreshold(item) || nonDefaultTargetNumHits(item); + return nonDefaultTargetNumHits(item); } @Override @@ -1050,14 +1050,9 @@ public class VespaSerializer { if (needsAnnotationBlock(item)) { destination.append("([{"); } - int lengthBeforeAnnotations = destination.length(); if (nonDefaultTargetNumHits(item)) { destination.append(TARGET_NUM_HITS).append(": ").append(item.getN()); } - if (nonDefaultScoreThreshold(item)) { - comma(destination, lengthBeforeAnnotations); - destination.append(SCORE_THRESHOLD).append(": ").append(item.getScoreThreshold()); - } if (needsAnnotationBlock(item)) { destination.append("}]"); } @@ -1065,10 +1060,6 @@ public class VespaSerializer { return true; } - private boolean nonDefaultScoreThreshold(WeakAndItem w) { - return w.getScoreThreshold() > 0; - } - private boolean nonDefaultTargetNumHits(WeakAndItem w) { return w.getN() != WeakAndItem.defaultN; } diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 19d4e6d41ba..15e3b7aab14 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -1139,11 +1139,6 @@ public class YqlParser implements Parser { if (targetNumHits != null) { weakAnd.setN(targetNumHits); } - Integer scoreThreshold = getAnnotation(spec, SCORE_THRESHOLD, - Integer.class, null, "min dot product score for hit inclusion"); - if (scoreThreshold != null) { - weakAnd.setScoreThreshold(scoreThreshold); - } return convertVarArgs(spec, 1, weakAnd); } diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java index f20fc85b448..ab9da8ccee5 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java @@ -20,7 +20,6 @@ import static com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher.STREAMING_S /** * Generates mail-specific query metrics. */ -@SuppressWarnings("removal") // TODO Vespa 8: remove (com.yahoo.log.event) public class MetricsSearcher extends Searcher { private static final CompoundName metricsearcherId=new CompoundName("metricsearcher.id"); diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java index 71a93607b4b..7f4de2e43f5 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java @@ -7,8 +7,6 @@ import com.yahoo.document.select.parser.ParseException; import com.yahoo.document.select.parser.TokenMgrException; import com.yahoo.documentapi.VisitorParameters; import com.yahoo.documentapi.VisitorSession; -import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.fs4.DocsumPacket; import com.yahoo.messagebus.routing.Route; import com.yahoo.prelude.Ping; @@ -90,12 +88,6 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { } @Override - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - public LoadTypeSet getLoadTypeSet() { - return ((MessageBusDocumentAccess) access.delegate()).getParams().getLoadTypes(); - } - - @Override public Visitor createVisitor(Query query, String searchCluster, Route route, String documentType, int traceLevelOverride) { return new VdsVisitor(query, searchCluster, route, documentType, this, traceLevelOverride); } diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java index 9330e43eaf7..22f77e6f19d 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java @@ -1,14 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.streamingvisitors; +import com.yahoo.document.fieldset.AllFields; import com.yahoo.document.select.parser.ParseException; import com.yahoo.documentapi.AckToken; import com.yahoo.documentapi.VisitorControlHandler; import com.yahoo.documentapi.VisitorDataHandler; import com.yahoo.documentapi.VisitorParameters; import com.yahoo.documentapi.VisitorSession; -import com.yahoo.documentapi.messagebus.loadtypes.LoadType; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.messagebus.protocol.DocumentSummaryMessage; import com.yahoo.documentapi.messagebus.protocol.QueryResultMessage; @@ -76,13 +75,6 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { public interface VisitorSessionFactory { VisitorSession createVisitorSession(VisitorParameters params) throws ParseException; - - /** - * @deprecated load types are deprecated - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - LoadTypeSet getLoadTypeSet(); } public VdsVisitor(Query query, String searchCluster, Route route, @@ -125,7 +117,6 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { return query.properties().getString(streamingSelection); } - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 private void setVisitorParameters(String searchCluster, Route route, String documentType) { params.setDocumentSelection(createSelectionString(documentType, createQuerySelectionString())); params.setTimeoutMs(query.getTimeout()); // Per bucket visitor timeout @@ -138,18 +129,10 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { if (query.properties().getDouble(streamingTotimestamp) != null) { params.setToTimestamp(query.properties().getDouble(streamingTotimestamp).longValue()); } + params.setFieldSet(AllFields.NAME); // Streaming searches need to look at _all_ fields by default. params.visitInconsistentBuckets(true); params.setPriority(DocumentProtocol.Priority.VERY_HIGH); - // TODO remove on Vespa 8 - if (query.properties().getString(streamingLoadtype) != null) { - LoadType loadType = visitorSessionFactory.getLoadTypeSet().getNameMap().get(query.properties().getString(streamingLoadtype)); - if (loadType != null) { - params.setLoadType(loadType); - params.setPriority(loadType.getPriority()); - } - } - if (query.properties().getString(streamingPriority) != null) { params.setPriority(DocumentProtocol.getPriorityByName( query.properties().getString(streamingPriority))); diff --git a/container-search/src/main/resources/configdefinitions/search.federation.strict-contracts.def b/container-search/src/main/resources/configdefinitions/search.federation.strict-contracts.def deleted file mode 100644 index 9d0e70f208a..00000000000 --- a/container-search/src/main/resources/configdefinitions/search.federation.strict-contracts.def +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=search.federation - -## DEPRECATED: This config will be removed on Vespa 8 -## A config to control whether to activate strict adherence to public contracts -## in the container. Usually, the container tries to do a best effort of hiding -## some undesirable effects of the the public contracts. Modifying this config -## signals the basic contract is sufficient, and allows the container to -## activate optimizations based on this knowledge. - -## Strict contracts for search chains, do not clone the query if it at all -## can be construed to be unnecessary. -searchchains bool default=false - -# EVERY, // Propagate any property starting by source.[sourceName] and provider.[providerName] -# NATIVE, // Propagate native properties only -# ALL, // Deprecated synonym of NATIVE -# OFFSET_HITS, // Propagate offset ands hits only -# NONE // propagate no properties -propagateSourceProperties enum {EVERY, NATIVE, ALL, OFFSET_HITS, NONE} default=EVERY diff --git a/container-search/src/main/resources/configdefinitions/search.statistics.measure-qps.def b/container-search/src/main/resources/configdefinitions/search.statistics.measure-qps.def deleted file mode 100644 index 0cc615d0601..00000000000 --- a/container-search/src/main/resources/configdefinitions/search.statistics.measure-qps.def +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=search.statistics - -## Configure measurements of peak QPS rates. - -## Control whether com.yahoo.search.statistics.PeakQpsSearcher -## should emit data through statistics framework or as a meta hit. -outputmethod enum { STATISTICS, METAHIT } default=STATISTICS - -## If using meta hits, which query property should trigger sampling -queryproperty string default="fetchpeakqps" - -## The base name for the statistics event. Default is the component name -## of the searcher, as defined in vespa-services, with dots replaced by -## underscore. -eventname string default="" diff --git a/container-search/src/main/resources/configdefinitions/search.statistics.timing-searcher.def b/container-search/src/main/resources/configdefinitions/search.statistics.timing-searcher.def deleted file mode 100644 index 16f83e31013..00000000000 --- a/container-search/src/main/resources/configdefinitions/search.statistics.timing-searcher.def +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=search.statistics - -timer[].name string -timer[].eventname string -timer[].measure enum { FILL, PING, SEARCH } default=SEARCH - diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index 434f4bcc8e0..a09c2ff9b79 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -8,7 +8,6 @@ import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.handler.ClustersStatus; import com.yahoo.container.handler.VipStatus; -import com.yahoo.container.protect.Error; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; @@ -428,81 +427,6 @@ public class ClusterSearcherTestCase { assertResult(6, new ArrayList<>(), getResult(6, 2, extra, ex)); } - @Test - public void testRequireThatSearchFailsForUndefinedRankProfileWithOneDocType() { - Execution execution = createExecution(Arrays.asList("type1"), false); - - // "default" rank profile - Query query = new Query("?query=hello"); - com.yahoo.search.Result result = execution.search(query); - assertEquals(3, result.getTotalHitCount()); - - // specified "default" rank profile - query = new Query("?query=hello&ranking.profile=default"); - result = execution.search(query); - assertEquals(3, result.getTotalHitCount()); - - // empty rank profile, should fail - query = new Query("?query=hello&ranking.profile="); - result = execution.search(query); - assertEquals(0, result.getTotalHitCount()); - assertEquals(result.hits().getError().getCode(), Error.INVALID_QUERY_PARAMETER.code); - - // invalid rank profile - query = new Query("?query=hello&ranking.profile=undefined"); - result = execution.search(query); - assertEquals(0, result.getTotalHitCount()); - assertEquals(result.hits().getError().getCode(), Error.INVALID_QUERY_PARAMETER.code); - - // valid rank profile for type1 - query = new Query("?query=hello&ranking.profile=testprofile"); - result = execution.search(query); - assertEquals(3, result.getTotalHitCount()); - } - - @Test - public void testRequireThatSearchFailsForUndefinedRankProfileWithMultipleDocTypes() { - Execution execution = createExecution(Arrays.asList("type1", "type2", "type3"), false); - - // "default" rank profile - Query query = new Query("?query=hello"); - com.yahoo.search.Result result = execution.search(query); - assertEquals(9, result.getTotalHitCount()); - - // specified "default" rank profile - query = new Query("?query=hello&ranking.profile=default"); - result = execution.search(query); - assertEquals(9, result.getTotalHitCount()); - - // empty rank profile, should fail - query = new Query("?query=hello&ranking.profile="); - result = execution.search(query); - assertEquals(0, result.getTotalHitCount()); - assertEquals(result.hits().getError().getCode(), Error.INVALID_QUERY_PARAMETER.code); - - // invalid rank profile - query = new Query("?query=hello&ranking.profile=undefined"); - result = execution.search(query); - assertEquals(0, result.getTotalHitCount()); - assertEquals(result.hits().getError().getCode(), Error.INVALID_QUERY_PARAMETER.code); - - // testprofile is only defined for type1, but should pass as it exists in at least one document type - query = new Query("?query=hello&ranking.profile=testprofile"); - result = execution.search(query); - assertEquals(9, result.getTotalHitCount()); - - // testprofile is only defined for type1, but should fail when restricting doc types - query = new Query("?query=hello&ranking.profile=testprofile&restrict=type1,type3"); - result = execution.search(query); - assertEquals(0, result.getTotalHitCount()); - assertEquals(result.hits().getError().getCode(), Error.INVALID_QUERY_PARAMETER.code); - - // testprofile is only defined for type1, ok if restricted to type1 - query = new Query("?query=hello&ranking.profile=testprofile&restrict=type1"); - result = execution.search(query); - assertEquals(3, result.getTotalHitCount()); - } - private static ClusterSearcher createSearcher(String clusterName, Double maxQueryTimeout, Double maxQueryCacheTimeout, boolean streamingMode, VipStatus vipStatus) { QrSearchersConfig.Builder qrSearchersConfig = new QrSearchersConfig.Builder(); diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java index a174bde3902..c39efe21e70 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java @@ -2,7 +2,6 @@ package com.yahoo.prelude.fastsearch; import com.google.common.collect.ImmutableSet; -import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.data.access.slime.SlimeAdapter; import com.yahoo.prelude.hitfield.JSONString; import com.yahoo.prelude.hitfield.RawData; @@ -101,7 +100,7 @@ public class SlimeSummaryTestCase { if (hit.getField("jsonstring_field") instanceof JSONString) { JSONString jstr = (JSONString) hit.getField("jsonstring_field"); assertEquals("{\"foo\":1,\"bar\":2}", jstr.getContent()); - assertNotNull(getParsedJSON(jstr)); + assertNotNull(jstr.getContent()); com.yahoo.data.access.Inspector value = jstr.inspect(); assertEquals(1L, value.field("foo").asLong()); @@ -125,8 +124,6 @@ public class SlimeSummaryTestCase { assertEquals(tensor2, featureData.getTensor("tensor2_feature")); } - @SuppressWarnings("removal") private static Object getParsedJSON(JSONString jstr) { return jstr.getParsedJSON(); } - @Test public void testFieldAccessAPI() { DocsumDefinitionSet partialDocsum1 = createPartialDocsumDefinitionSet1(); diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index c9193d08381..271b932dff5 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -16,6 +16,7 @@ import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; +import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.grouping.GroupingRequest; @@ -41,6 +42,7 @@ import java.util.logging.Logger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -51,9 +53,6 @@ import static org.junit.Assert.assertTrue; */ public class FastSearcherTestCase { - private final static DocumentdbInfoConfig documentdbInfoConfig = new DocumentdbInfoConfig(new DocumentdbInfoConfig.Builder()); - - @Test public void testNullQuery() { Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL); @@ -61,11 +60,11 @@ public class FastSearcherTestCase { MockDispatcher.create(Collections.emptyList()), new SummaryParameters(null), new ClusterParams("testhittype"), - documentdbInfoConfig, - SchemaInfo.empty()); + documentdbInfoConfig("test"), + schemaInfo("test")); String query = "?junkparam=ignored"; - Result result = doSearch(fastSearcher,new Query(query), 0, 10); + Result result = doSearch(fastSearcher, new Query(query), 0, 10); ErrorMessage message = result.hits().getError(); assertNotNull("Got error", message); @@ -93,11 +92,11 @@ public class FastSearcherTestCase { @Test public void testSinglePassGroupingIsForcedWithSingleNodeGroups() { FastSearcher fastSearcher = new FastSearcher("container.0", - MockDispatcher.create(Collections.singletonList(new Node(0, "host0", 0))), + MockDispatcher.create(List.of(new Node(0, "host0", 0))), new SummaryParameters(null), new ClusterParams("testhittype"), - documentdbInfoConfig, - SchemaInfo.empty()); + documentdbInfoConfig("test"), + schemaInfo("test")); Query q = new Query("?query=foo"); GroupingRequest request1 = GroupingRequest.newInstance(q); request1.setRootOperation(new AllOperation()); @@ -114,6 +113,19 @@ public class FastSearcherTestCase { } @Test + public void testRankProfileValidation() { + FastSearcher fastSearcher = new FastSearcher("container.0", + MockDispatcher.create(List.of(new Node(0, "host0", 0))), + new SummaryParameters(null), + new ClusterParams("testhittype"), + documentdbInfoConfig("test"), + schemaInfo("test")); + assertFalse(searchError("?query=q", fastSearcher).contains("does not contain requested rank profile")); + assertFalse(searchError("?query=q&ranking.profile=default", fastSearcher).contains("does not contain requested rank profile")); + assertTrue(searchError("?query=q&ranking.profile=nosuch", fastSearcher).contains("does not contain requested rank profile")); + } + + @Test public void testSummaryNeedsQuery() { var documentDb = new DocumentdbInfoConfig(new DocumentdbInfoConfig.Builder().documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name("test"))); var schema = new Schema.Builder("test") @@ -145,8 +157,8 @@ public class FastSearcherTestCase { dispatcher, new SummaryParameters(null), new ClusterParams("testhittype"), - documentdbInfoConfig, - SchemaInfo.empty()); + documentdbInfoConfig("test"), + schemaInfo("test")); Query q = new Query("?query=foo"); GroupingRequest request1 = GroupingRequest.newInstance(q); request1.setRootOperation(new AllOperation()); @@ -192,4 +204,24 @@ public class FastSearcherTestCase { assertTrue(vipStatus.isInRotation()); //Verify that deconstruct does not touch vipstatus } + private String searchError(String query, Searcher searcher) { + return search(query, searcher).hits().getError().getDetailedMessage(); + } + + private Result search(String query, Searcher searcher) { + return searcher.search(new Query(query), new Execution(Execution.Context.createContextStub())); + } + + private DocumentdbInfoConfig documentdbInfoConfig(String schemaName) { + var db = new DocumentdbInfoConfig.Documentdb.Builder().name(schemaName); + db.rankprofile(new DocumentdbInfoConfig.Documentdb.Rankprofile.Builder().name("default")); + return new DocumentdbInfoConfig.Builder().documentdb(db).build(); + } + + private SchemaInfo schemaInfo(String schemaName) { + var schema = new Schema.Builder(schemaName); + schema.add(new RankProfile.Builder("default").build()); + return new SchemaInfo(List.of(schema.build()), Map.of()); + } + } diff --git a/container-search/src/test/java/com/yahoo/prelude/hitfield/test/JSONStringTestCase.java b/container-search/src/test/java/com/yahoo/prelude/hitfield/test/JSONStringTestCase.java index 2e05e2e6675..256014b7df4 100644 --- a/container-search/src/test/java/com/yahoo/prelude/hitfield/test/JSONStringTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/hitfield/test/JSONStringTestCase.java @@ -155,27 +155,6 @@ public class JSONStringTestCase { @Test public void testStruct() { { - String json = "{\"as1\":[\"per\",\"paal\"],\"l1\":1122334455667788991,\"d1\":81.790001,\"i1\":1,\"al1\":[11223344556677881,11223344556677883],\"s1\":\"string\\n" - + "espa\u00F1a ssf1.s1\"}"; - JSONString js = new JSONString(json); - String[] renderedFields = { - " <struct-field name=\"l1\">1122334455667788991</struct-field>\n", - " <struct-field name=\"al1\">\n" - + " <item>11223344556677881</item>\n" - + " <item>11223344556677883</item>\n" - + " </struct-field>\n", - " <struct-field name=\"i1\">1</struct-field>\n", - " <struct-field name=\"d1\">81.790001</struct-field>\n", - " <struct-field name=\"as1\">\n" - + " <item>per</item>\n" - + " <item>paal</item>\n" - + " </struct-field>\n", - " <struct-field name=\"s1\">string\n" - + "españa ssf1.s1</struct-field>\n" }; - String rendered = js.toString(); - checkSubstrings(renderedFields, rendered); - } - { Value.ObjectValue top = new Value.ObjectValue(); top.put("d1", new Value.DoubleValue(81.790001)) .put("s1", @@ -211,24 +190,6 @@ public class JSONStringTestCase { checkSubstrings(renderedFields, rendered); } { - String json = "{\"as1\":[\"per\",\"paal\"],\"d1\":84.790001,\"i1\":4,\"al1\":[11223344556677881,11223344556677883]}"; - JSONString js = new JSONString(json); - String[] renderedFields = { - " <struct-field name=\"al1\">\n" - + " <item>11223344556677881</item>\n" - + " <item>11223344556677883</item>\n" - + " </struct-field>\n", - " <struct-field name=\"i1\">4</struct-field>\n", - " <struct-field name=\"d1\">84.790001</struct-field>\n", - " <struct-field name=\"as1\">\n" - + " <item>per</item>\n" - + " <item>paal</item>\n" - + " </struct-field>\n " }; - String rendered = js.toString(); - - checkSubstrings(renderedFields, rendered); - } - { Value.ObjectValue top = new Value.ObjectValue(); top.put("d1", new Value.DoubleValue(84.790001)) .put("al1", @@ -259,36 +220,6 @@ public class JSONStringTestCase { } { - String json = "{\"s2\":\"string espa\u00F1a\\n" - + "ssf5.s2\",\"nss1\":{\"as1\":[\"per\",\"paal\"],\"l1\":1122334455667788995,\"d1\":85.790001,\"i1\":5,\"al1\":[11223344556677881,11223344556677883],\"s1\":\"string\\n" - + "espa\u00F1a ssf5.nss1.s1\"}}"; - JSONString js = new JSONString(json); - String[] renderedFields = { - " <struct-field name=\"nss1\">\n", - " <struct-field name=\"s1\">string\n" - + "españa ssf5.nss1.s1</struct-field>\n", - " <struct-field name=\"s2\">string españa\n" - + "ssf5.s2</struct-field>\n " }; - String nss1Fields[] = { - " <struct-field name=\"l1\">1122334455667788995</struct-field>\n", - " <struct-field name=\"al1\">\n" - + " <item>11223344556677881</item>\n" - + " <item>11223344556677883</item>\n" - + " </struct-field>\n", - " <struct-field name=\"i1\">5</struct-field>\n", - " <struct-field name=\"d1\">85.790001</struct-field>\n", - " <struct-field name=\"as1\">\n" - + " <item>per</item>\n" - + " <item>paal</item>\n" - + " </struct-field>\n" }; - - String rendered = js.toString(); - checkSubstrings(renderedFields, rendered); - int nss1Offset = rendered.indexOf(renderedFields[0]) - + renderedFields[0].length(); - checkSubstrings(nss1Fields, rendered, nss1Offset); - } - { Value.ObjectValue top = new Value.ObjectValue(); top.put("s2", "string espa\u00F1a\nssf5.s2").put( "nss1", @@ -347,34 +278,6 @@ public class JSONStringTestCase { expectedEnd)); } { - String json = "{\"s2\":\"string espa\u00F1a\\n" - + "ssf8.s2\",\"nss1\":{\"as1\":[\"per\",\"paal\"],\"d1\":88.790001,\"i1\":8,\"al1\":[11223344556677881,11223344556677883]}}"; - JSONString js = new JSONString(json); - - String[] renderedFields = { - " <struct-field name=\"nss1\">\n", - " <struct-field name=\"s2\">string españa\n" - + "ssf8.s2</struct-field>\n " }; - String nss1Fields[] = { - " <struct-field name=\"al1\">\n" - + " <item>11223344556677881</item>\n" - + " <item>11223344556677883</item>\n" - + " </struct-field>\n", - " <struct-field name=\"i1\">8</struct-field>\n", - " <struct-field name=\"d1\">88.790001</struct-field>\n", - " <struct-field name=\"as1\">\n" - + " <item>per</item>\n" - + " <item>paal</item>\n" - + " </struct-field>\n" }; - - String rendered = js.toString(); - checkSubstrings(renderedFields, rendered); - int nss1Offset = rendered.indexOf(renderedFields[0]) - + renderedFields[0].length(); - checkSubstrings(nss1Fields, rendered, nss1Offset); - - } - { Value.ObjectValue top = new Value.ObjectValue(); top.put("s2", "string espa\u00F1a\nssf8.s2").put( "nss1", diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java index 7dad3c9b52b..4fffc450e74 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java @@ -12,7 +12,6 @@ import com.yahoo.component.chain.Chain; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.search.federation.FederationConfig; import com.yahoo.container.QrSearchersConfig; -import com.yahoo.search.federation.StrictContractsConfig; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.prelude.fastsearch.FastHit; @@ -113,10 +112,8 @@ public class BlendingSearcherTestCase { entry.getValue())); } - StrictContractsConfig contracts = new StrictContractsConfig.Builder().build(); - FederationSearcher fedSearcher = - new FederationSearcher(new FederationConfig(builder), contracts, new ComponentRegistry<>()); + new FederationSearcher(new FederationConfig(builder), new ComponentRegistry<>()); BlendingSearcher blendingSearcher = new BlendingSearcher(blendingField); blendingChain = new SearchChain(ComponentId.createAnonymousComponentId("blendingChain"), blendingSearcher, fedSearcher); return true; diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index 02e2ac87f1c..a076b929ad7 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -115,7 +115,6 @@ public class MockSearchCluster extends SearchCluster { public static DispatchConfig createDispatchConfig(double minSearchCoverage, List<Node> nodes) { DispatchConfig.Builder builder = new DispatchConfig.Builder(); builder.minActivedocsPercentage(88.0); - builder.minGroupCoverage(99.0); builder.minSearchCoverage(minSearchCoverage); builder.distributionPolicy(DispatchConfig.DistributionPolicy.Enum.ROUNDROBIN); if (minSearchCoverage < 100.0) { diff --git a/container-search/src/test/java/com/yahoo/search/federation/FederationSearcherTest.java b/container-search/src/test/java/com/yahoo/search/federation/FederationSearcherTest.java index ef9e0311e59..4bb6db40c66 100644 --- a/container-search/src/test/java/com/yahoo/search/federation/FederationSearcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/federation/FederationSearcherTest.java @@ -20,8 +20,6 @@ import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.Execution.Context; import com.yahoo.search.searchchain.model.federation.FederationOptions; -import org.junit.After; -import org.junit.Before; import org.junit.Test; import java.util.Arrays; @@ -173,7 +171,6 @@ public class FederationSearcherTest { FederationSearcher searcher = new FederationSearcher( new FederationConfig(new FederationConfig.Builder().targetSelector(targetSelectorId.toString())), - new StrictContractsConfig(new StrictContractsConfig.Builder()), targetSelectors); Query query = new Query(); @@ -192,7 +189,6 @@ public class FederationSearcherTest { FederationSearcher searcher = new FederationSearcher( new FederationConfig(new FederationConfig.Builder().targetSelector(targetSelectorId.toString())), - new StrictContractsConfig(new StrictContractsConfig.Builder()), targetSelectors); Query query = new Query(); diff --git a/container-search/src/test/java/com/yahoo/search/federation/FederationSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/FederationSearcherTestCase.java index 8c7e5fd59c6..c2feb16df04 100644 --- a/container-search/src/test/java/com/yahoo/search/federation/FederationSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/federation/FederationSearcherTestCase.java @@ -11,7 +11,6 @@ import com.yahoo.search.federation.sourceref.SearchChainResolver; import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.SearchChain; import com.yahoo.search.searchchain.SearchChainRegistry; @@ -26,12 +25,10 @@ import org.junit.Test; import java.util.List; -import static com.yahoo.search.federation.StrictContractsConfig.PropagateSourceProperties; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; /** @@ -88,21 +85,7 @@ public class FederationSearcherTestCase { } private Searcher createFederationSearcher() { - return buildFederation(new StrictContractsConfig(new StrictContractsConfig.Builder())); - } - - private Searcher createFederationSearcher(PropagateSourceProperties.Enum propagateSourceProperties) { - return buildFederation(new StrictContractsConfig(new StrictContractsConfig.Builder().propagateSourceProperties(propagateSourceProperties))); - } - - private Searcher createStrictFederationSearcher() { - StrictContractsConfig.Builder builder = new StrictContractsConfig.Builder(); - builder.searchchains(true); - return buildFederation(new StrictContractsConfig(builder)); - } - - private Searcher buildFederation(StrictContractsConfig contracts) throws RuntimeException { - return new FederationSearcher(new FederationConfig(builder), contracts, new ComponentRegistry<>()); + return new FederationSearcher(new FederationConfig(builder), new ComponentRegistry<>()); } private SearchChain createSearchChain(ComponentId chainId,Searcher searcher) { @@ -161,10 +144,7 @@ public class FederationSearcherTestCase { }, SOURCE2); return new Chain<>("default", - new FederationSearcher(new FederationConfig(builder), - new StrictContractsConfig( - new StrictContractsConfig.Builder().searchchains(strictContracts)), - new ComponentRegistry<>())); + new FederationSearcher(new FederationConfig(builder), new ComponentRegistry<>())); } @Test @@ -199,19 +179,8 @@ public class FederationSearcherTestCase { } @Test - public void testPropertyPropagation_native() { - Result result = searchWithPropertyPropagation(PropagateSourceProperties.NATIVE); - - assertEquals("source:mySource1", result.hits().get(0).getId().stringValue()); - assertEquals("source:mySource2", result.hits().get(1).getId().stringValue()); - assertEquals("nalle", result.hits().get(0).getQuery().getPresentation().getSummary()); - assertNull(result.hits().get(1).getQuery().getPresentation().getSummary()); - assertNull(result.hits().get(0).getQuery().properties().get("custom")); - } - - @Test - public void testPropertyPropagation_every() { - Result result = searchWithPropertyPropagation(PropagateSourceProperties.EVERY); + public void testPropertyPropagation() { + Result result = searchWithPropertyPropagation(); assertEquals("source:mySource1", result.hits().get(0).getId().stringValue()); assertEquals("source:mySource2", result.hits().get(1).getId().stringValue()); @@ -228,10 +197,10 @@ public class FederationSearcherTestCase { assertNull(result.hits().get(1).getQuery().getPresentation().getSummary()); } - private Result searchWithPropertyPropagation(PropagateSourceProperties.Enum propagateSourceProperties) { + private Result searchWithPropertyPropagation() { addChained(new MockSearcher(), "mySource1"); addChained(new MockSearcher(), "mySource2"); - Chain<Searcher> mainChain = new Chain<>("default", createFederationSearcher(propagateSourceProperties)); + Chain<Searcher> mainChain = new Chain<>("default", createFederationSearcher()); Query q = new Query(QueryTestCase.httpEncode("?query=test&source.mySource1.presentation.summary=nalle&source.mySource1.customSourceProperty=foo&source.mySource2.custom.source.property=bar&source.mySource1.hits=13&source.mySource1.offset=1")); @@ -241,39 +210,6 @@ public class FederationSearcherTestCase { } @Test - public void testDisablePropertyPropagation() { - Result result = searchWithPropertyPropagation(PropagateSourceProperties.NONE); - - assertNull(result.hits().get(0).getQuery().getPresentation().getSummary()); - } - - @Test - public void testNoCloning() { - String sourceName = "cloningcheck"; - Query query = new Query(QueryTestCase.httpEncode("?query=test&sources=" + sourceName)); - addChained(new QueryCheckSearcher(query), sourceName); - addChained(new MockSearcher(), "mySource1"); - Chain<Searcher> mainChain = new Chain<>("default", createStrictFederationSearcher()); - Result result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry)).search(query); - HitGroup h = (HitGroup) result.hits().get(0); - assertNull(h.getErrorHit()); - assertSame(QueryCheckSearcher.OK, h.get(0).getField(QueryCheckSearcher.STATUS)); - - mainChain = new Chain<>("default", createFederationSearcher()); - result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry)).search(query); - h = (HitGroup) result.hits().get(0); - assertSame(QueryCheckSearcher.FEDERATION_SEARCHER_HAS_CLONED_THE_QUERY, h.getError().getDetailedMessage()); - - query = new Query(QueryTestCase.httpEncode("?query=test&sources=" + sourceName + ",mySource1")); - addChained(new QueryCheckSearcher(query), sourceName); - result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry)).search(query); - h = (HitGroup) result.hits().get(0); - assertEquals("source:" + sourceName, h.getId().stringValue()); - assertSame(QueryCheckSearcher.FEDERATION_SEARCHER_HAS_CLONED_THE_QUERY, h.getError().getDetailedMessage()); - assertEquals("source:mySource1", result.hits().get(1).getId().stringValue()); - } - - @Test public void testTopLevelHitGroupFieldPropagation() { addChained(new MockSearcher(), "mySource1"); addChained(new AnotherMockSearcher(), "mySource2"); diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java index 7811336823d..dd78cfefe28 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java @@ -348,7 +348,7 @@ public class JSONSearchHandlerTestCase { // Run query String result = driver.sendRequest(uri + "searchChain=echoingQuery", com.yahoo.jdisc.http.HttpRequest.Method.POST, root.toString(), JSON_CONTENT_TYPE).readAll(); - assertEquals("{\"root\":{\"id\":\"toplevel\",\"relevance\":1.0,\"fields\":{\"totalCount\":0},\"children\":[{\"id\":\"Query\",\"relevance\":1.0,\"fields\":{\"query\":\"select * from sources * where default contains \\\"bad\\\";\"}}]}}", + assertEquals("{\"root\":{\"id\":\"toplevel\",\"relevance\":1.0,\"fields\":{\"totalCount\":0},\"children\":[{\"id\":\"Query\",\"relevance\":1.0,\"fields\":{\"query\":\"select * from sources * where default contains \\\"bad\\\"\"}}]}}", result); } @@ -373,7 +373,7 @@ public class JSONSearchHandlerTestCase { "}\n"; String result = driver.sendRequest(uri + "searchChain=echoingQuery", com.yahoo.jdisc.http.HttpRequest.Method.POST, query, JSON_CONTENT_TYPE).readAll(); - String expected = "{\"root\":{\"id\":\"toplevel\",\"relevance\":1.0,\"fields\":{\"totalCount\":0},\"children\":[{\"id\":\"Query\",\"relevance\":1.0,\"fields\":{\"query\":\"select * from sources * where field contains \\\"term\\\" | all(output(count()));\"}}]}}"; + String expected = "{\"root\":{\"id\":\"toplevel\",\"relevance\":1.0,\"fields\":{\"totalCount\":0},\"children\":[{\"id\":\"Query\",\"relevance\":1.0,\"fields\":{\"query\":\"select * from sources * where field contains \\\"term\\\" | all(output(count()))\"}}]}}"; assertEquals(expected, result); } @@ -396,7 +396,7 @@ public class JSONSearchHandlerTestCase { "}\n"; String result = driver.sendRequest(uri + "searchChain=echoingQuery", com.yahoo.jdisc.http.HttpRequest.Method.POST, query, JSON_CONTENT_TYPE).readAll(); - String expected = "{\"root\":{\"id\":\"toplevel\",\"relevance\":1.0,\"fields\":{\"totalCount\":0},\"children\":[{\"id\":\"Query\",\"relevance\":1.0,\"fields\":{\"query\":\"select * from sources * where field contains \\\"term\\\" | all(output(count()));\"}}]}}"; + String expected = "{\"root\":{\"id\":\"toplevel\",\"relevance\":1.0,\"fields\":{\"totalCount\":0},\"children\":[{\"id\":\"Query\",\"relevance\":1.0,\"fields\":{\"query\":\"select * from sources * where field contains \\\"term\\\" | all(output(count()))\"}}]}}"; assertEquals(expected, result); } @@ -407,14 +407,14 @@ public class JSONSearchHandlerTestCase { // Run query String result = driver.sendRequest(uri + "searchChain=echoingQuery", com.yahoo.jdisc.http.HttpRequest.Method.POST, root.toString(), JSON_CONTENT_TYPE).readAll(); - assertEquals("{\"root\":{\"id\":\"toplevel\",\"relevance\":1.0,\"fields\":{\"totalCount\":0},\"children\":[{\"id\":\"Query\",\"relevance\":1.0,\"fields\":{\"query\":\"select * from sources * where default contains \\\"bad\\\";\"}}]}}", + assertEquals("{\"root\":{\"id\":\"toplevel\",\"relevance\":1.0,\"fields\":{\"totalCount\":0},\"children\":[{\"id\":\"Query\",\"relevance\":1.0,\"fields\":{\"query\":\"select * from sources * where default contains \\\"bad\\\"\"}}]}}", result); } @Test public void testRequestMapping() { ObjectNode json = jsonMapper.createObjectNode(); - json.put("yql", "select * from sources * where sddocname contains \"blog_post\" limit 0 | all(group(date) max(3) order(-count())each(output(count())));"); + json.put("yql", "select * from sources * where sddocname contains \"blog_post\" limit 0 | all(group(date) max(3) order(-count())each(output(count())))"); json.put("hits", 10); json.put("offset", 5); json.put("queryProfile", "foo"); @@ -518,7 +518,7 @@ public class JSONSearchHandlerTestCase { "&nocache=false&model.type=yql&collapse.summary=default&ranking.matchPhase.diversity.minGroups=1&ranking.location=123789.89123N%3B128123W&ranking.queryCache=false&offset=5&streaming.groupname=abc&groupingSessionCache=false" + "&presentation.template=json&trace.rules=none&rules.off=false&ranking.properties=default&searchChain=exceptionInPlugin&pos.ll=1263123N%3B1231.9W&ranking.sorting=desc&ranking.matchPhase.ascending=true&ranking.features=none&hitcountestimate=true" + "&model.filter=default&metrics.ignore=_all&collapse.field=none&ranking.profile=1&rules.rulebase=default&model.defaultIndex=1&trace.level=1&ranking.listFeatures=false&timeout=0&presentation.format=json" + - "&yql=select+%2A+from+sources+%2A+where+sddocname+contains+%22blog_post%22+limit+0+%7C+all%28group%28date%29+max%283%29+order%28-count%28%29%29each%28output%28count%28%29%29%29%29%3B&recall=none&streaming.maxbucketspervisitor=5" + + "&yql=select+%2A+from+sources+%2A+where+sddocname+contains+%22blog_post%22+limit+0+%7C+all%28group%28date%29+max%283%29+order%28-count%28%29%29each%28output%28count%28%29%29%29%29&recall=none&streaming.maxbucketspervisitor=5" + "&queryProfile=foo&presentation.bolding=true&model.encoding=json&model.queryString=abc&streaming.selection=none&trace.timestamps=false&collapse.size=2&streaming.priority=10&ranking.matchPhase.diversity.attribute=title" + "&ranking.matchPhase.attribute=title&hits=10&streaming.userid=123&pos.bb=1237123W%3B123218N&model.restrict=_doc%2Cjson%2Cxml&ranking.freshness=0.05&user=123"; diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java index 84bb82f3851..29a8969233b 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java @@ -5,10 +5,7 @@ import com.yahoo.search.Query; import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileProperties; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; -import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileConfigurer; -import com.yahoo.search.query.profile.config.QueryProfilesConfig; -import com.yahoo.search.query.profile.config.QueryProfilesConfig.Queryprofile; import com.yahoo.search.test.QueryTestCase; import org.junit.Test; @@ -29,27 +26,27 @@ public class QueryProfileConfigurationTestCase { @Test public void testConfiguration() { - QueryProfileConfigurer configurer= - new QueryProfileConfigurer("file:" + CONFIG_DIR + "query-profiles-configuration.cfg"); - QueryProfile profile=configurer.getCurrentRegistry().getComponent("default"); + var profile = QueryProfileConfigurer.createFromConfigId("file:" + CONFIG_DIR + "query-profiles-configuration.cfg") + .getComponent("default"); - assertEquals("a-value",profile.get("a")); - assertEquals("b-value",profile.get("b")); - assertEquals("c.d-value",profile.get("c.d")); + assertEquals("a-value", profile.get("a")); + assertEquals("b-value", profile.get("b")); + assertEquals("c.d-value", profile.get("c.d")); assertFalse(profile.isDeclaredOverridable("c.d", null)); - assertEquals("e-value-inherited1",profile.get("e")); - assertEquals("g.d2-value-inherited1",profile.get("g.d2")); // Even though we make an explicit reference to one not having this value, we still inherit it - assertEquals("a-value-subprofile1",profile.get("sub1.a")); - assertEquals("c.d-value-subprofile1",profile.get("sub1.c.d")); - assertEquals("a-value-subprofile2",profile.get("sub2.a")); - assertEquals("c.d-value-subprofile2",profile.get("sub2.c.d")); - assertEquals("e-value-subprofile3",profile.get("g.e")); + assertEquals("e-value-inherited1", profile.get("e")); + assertEquals("g.d2-value-inherited1", profile.get("g.d2")); // Even though we make an explicit reference to one not having this value, we still inherit it + assertEquals("a-value-subprofile1", profile.get("sub1.a")); + assertEquals("c.d-value-subprofile1", profile.get("sub1.c.d")); + assertEquals("a-value-subprofile2", profile.get("sub2.a")); + assertEquals("c.d-value-subprofile2", profile.get("sub2.c.d")); + assertEquals("e-value-subprofile3", profile.get("g.e")); } @Test public void testBug3197426() { - QueryProfileConfigurer configurer = new QueryProfileConfigurer("file:" + CONFIG_DIR + "bug3197426.cfg"); - CompiledQueryProfile profile = configurer.getCurrentRegistry().getComponent("default").compile(null); + var profile = QueryProfileConfigurer.createFromConfigId("file:" + CONFIG_DIR + "bug3197426.cfg") + .getComponent("default").compile(null); + Map<String, Object> properties = new QueryProfileProperties(profile).listProperties("source.image"); assertEquals("yes", properties.get("mlr")); assertEquals("zh-Hant", properties.get("language")); @@ -67,44 +64,42 @@ public class QueryProfileConfigurationTestCase { @Test public void testVariantConfiguration() { - QueryProfileConfigurer configurer= - new QueryProfileConfigurer("file:" + CONFIG_DIR + "query-profile-variants-configuration.cfg"); + var registry = QueryProfileConfigurer.createFromConfigId("file:" + CONFIG_DIR + "query-profile-variants-configuration.cfg"); // Variant 1 - QueryProfile variants1 =configurer.getCurrentRegistry().getComponent("variants1"); - assertGet("x1.y1.a","a",new String[] { "x1","y1" }, variants1); - assertGet("x1.y1.b","b",new String[] { "x1","y1" }, variants1); - assertGet("x1.y?.a","a",new String[] { "x1","zz" }, variants1); - assertGet("x?.y1.a","a",new String[] { "zz","y1" }, variants1); - assertGet("a-deflt","a",new String[] { "z1","z2" }, variants1); + QueryProfile variants1 = registry.getComponent("variants1"); + assertGet("x1.y1.a", "a", new String[] { "x1","y1" }, variants1); + assertGet("x1.y1.b", "b", new String[] { "x1","y1" }, variants1); + assertGet("x1.y?.a", "a", new String[] { "x1","zz" }, variants1); + assertGet("x?.y1.a", "a", new String[] { "zz","y1" }, variants1); + assertGet("a-deflt", "a", new String[] { "z1","z2" }, variants1); // ...inherited - assertGet("parent1-value","parent1",new String[] { "x1","y1" }, variants1); - assertGet("parent2-value","parent2",new String[] { "x1","y1" }, variants1); - assertGet(null,"parent1",new String[] { "x1","y2" }, variants1); - assertGet(null,"parent2",new String[] { "x1","y2" }, variants1); + assertGet("parent1-value", "parent1", new String[] { "x1","y1" }, variants1); + assertGet("parent2-value", "parent2", new String[] { "x1","y1" }, variants1); + assertGet(null, "parent1", new String[] { "x1","y2" }, variants1); + assertGet(null, "parent2", new String[] { "x1","y2" }, variants1); // Variant 2 - QueryProfile variants2 =configurer.getCurrentRegistry().getComponent("variants2"); - assertGet("variant2:y1.c","c",new String[] { "*","y1" }, variants2); - assertGet("variant2:y2.c","c",new String[] { "*","y2" }, variants2); - assertGet("variant2:c-df","c",new String[] { "*","z1" }, variants2); - assertGet("variant2:c-df","c",new String[] { }, variants2); - assertGet("variant2:c-df","c",new String[] { "*" }, variants2); - assertGet(null, "d",new String[] { "*","y1" }, variants2); + QueryProfile variants2 = registry.getComponent("variants2"); + assertGet("variant2:y1.c", "c", new String[] { "*","y1" }, variants2); + assertGet("variant2:y2.c", "c", new String[] { "*","y2" }, variants2); + assertGet("variant2:c-df", "c", new String[] { "*","z1" }, variants2); + assertGet("variant2:c-df", "c", new String[] { }, variants2); + assertGet("variant2:c-df", "c", new String[] { "*" }, variants2); + assertGet(null, "d", new String[] { "*","y1" }, variants2); // Reference following from variant 1 - assertGet("variant2:y1.c","toVariants.c",new String[] { "**", "y1" } , variants1); - assertGet("variant3:c-df","toVariants.c",new String[] { "x1", "**" } , variants1); - assertGet("variant3:y1.c","toVariants.c",new String[] { "x1", "y1" } , variants1); // variant3 by order priority - assertGet("variant3:y2.c","toVariants.c",new String[] { "x1", "y2" } , variants1); + assertGet("variant2:y1.c", "toVariants.c", new String[] { "**", "y1" } , variants1); + assertGet("variant3:c-df", "toVariants.c", new String[] { "x1", "**" } , variants1); + assertGet("variant3:y1.c", "toVariants.c", new String[] { "x1", "y1" } , variants1); // variant3 by order priority + assertGet("variant3:y2.c", "toVariants.c", new String[] { "x1", "y2" } , variants1); } @Test public void testVariantConfigurationThroughQueryLookup() { - QueryProfileConfigurer configurer= - new QueryProfileConfigurer("file:" + CONFIG_DIR + "query-profile-variants-configuration.cfg"); + var registry = QueryProfileConfigurer.createFromConfigId("file:" + CONFIG_DIR + "query-profile-variants-configuration.cfg") + .compile(); - CompiledQueryProfileRegistry registry = configurer.getCurrentRegistry().compile(); CompiledQueryProfile variants1 = registry.getComponent("variants1"); // Variant 1 @@ -138,10 +133,9 @@ public class QueryProfileConfigurationTestCase { public void testVariant2ConfigurationThroughQueryLookup() { final double delta = 0.0000001; - QueryProfileConfigurer configurer= - new QueryProfileConfigurer("file:" + CONFIG_DIR + "query-profile-variants2.cfg"); + var registry = QueryProfileConfigurer.createFromConfigId("file:" + CONFIG_DIR + "query-profile-variants2.cfg") + .compile(); - CompiledQueryProfileRegistry registry = configurer.getCurrentRegistry().compile(); Query query = new Query(QueryTestCase.httpEncode("?query=heh&queryProfile=multi&myindex=default&myquery=lo ve&tracelevel=5"), registry.findQueryProfile("multi")); assertEquals("love", query.properties().get("model.queryString")); @@ -154,11 +148,12 @@ public class QueryProfileConfigurationTestCase { assertEquals(-30, query.getRanking().getFeatures().getDouble("query(scorelimit)").getAsDouble(), delta); } - private void assertGet(String expectedValue,String parameter,String[] dimensionValues,QueryProfile profile) { - Map<String,String> context=new HashMap<>(); - for (int i=0; i<dimensionValues.length; i++) - context.put(profile.getVariants().getDimensions().get(i),dimensionValues[i]); // Lookup dim. names to ease test... - assertEquals("Looking up '" + parameter + "' for '" + Arrays.toString(dimensionValues) + "'",expectedValue,profile.get(parameter, context, null)); + private void assertGet(String expectedValue, String parameter, String[] dimensionValues, QueryProfile profile) { + Map<String, String> context = new HashMap<>(); + for (int i = 0; i<dimensionValues.length; i++) + context.put(profile.getVariants().getDimensions().get(i), dimensionValues[i]); // Lookup dim. names to ease test... + assertEquals("Looking up '" + parameter + "' for '" + Arrays.toString(dimensionValues) + "'", + expectedValue, profile.get(parameter, context, null)); } } diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/TypedProfilesConfigurationTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/TypedProfilesConfigurationTestCase.java index 0531d847c9e..944ee6d79ef 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/TypedProfilesConfigurationTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/TypedProfilesConfigurationTestCase.java @@ -1,11 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.config.test; -import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileConfigurer; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; -import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -20,45 +18,43 @@ public class TypedProfilesConfigurationTestCase { /** Asserts that everything is read correctly from this configuration */ @Test public void testIt() { - QueryProfileConfigurer configurer= - new QueryProfileConfigurer("file:src/test/java/com/yahoo/search/query/profile/config/test/typed-profiles.cfg"); - QueryProfileRegistry registry=configurer.getCurrentRegistry(); - QueryProfileTypeRegistry types=registry.getTypeRegistry(); + var registry = QueryProfileConfigurer.createFromConfigId("file:src/test/java/com/yahoo/search/query/profile/config/test/typed-profiles.cfg"); + var types = registry.getTypeRegistry(); // Assert that each type was read correctly - QueryProfileType testType=types.getComponent("testtype"); - assertEquals("testtype",testType.getId().getName()); + QueryProfileType testType = types.getComponent("testtype"); + assertEquals("testtype", testType.getId().getName()); assertFalse(testType.isStrict()); assertFalse(testType.getMatchAsPath()); - assertEquals(7,testType.fields().size()); - assertEquals("myString",testType.getField("myString").getName()); + assertEquals(7, testType.fields().size()); + assertEquals("myString", testType.getField("myString").getName()); assertTrue(testType.getField("myString").isMandatory()); assertTrue(testType.getField("myString").isOverridable()); assertFalse(testType.getField("myInteger").isMandatory()); assertFalse(testType.getField("myInteger").isOverridable()); FieldDescription field= testType.getField("myUserQueryProfile"); - assertEquals("reference to a query profile of type 'user'",field.getType().toInstanceDescription()); + assertEquals("reference to a query profile of type 'user'", field.getType().toInstanceDescription()); assertTrue(field.getAliases().contains("myqp")); assertTrue(field.getAliases().contains("user-profile")); QueryProfileType testTypeStrict=types.getComponent("testtypestrict"); assertTrue(testTypeStrict.isStrict()); assertTrue(testTypeStrict.getMatchAsPath()); - assertEquals(7,testTypeStrict.fields().size()); + assertEquals(7, testTypeStrict.fields().size()); assertEquals("reference to a query profile of type 'userstrict'", testTypeStrict.getField("myUserQueryProfile").getType().toInstanceDescription()); QueryProfileType user=types.getComponent("user"); assertFalse(user.isStrict()); assertFalse(user.getMatchAsPath()); - assertEquals(2,user.fields().size()); - assertEquals(String.class,user.getField("myUserString").getType().getValueClass()); + assertEquals(2, user.fields().size()); + assertEquals(String.class, user.getField("myUserString").getType().getValueClass()); - QueryProfileType userStrict=types.getComponent("userstrict"); + QueryProfileType userStrict = types.getComponent("userstrict"); assertTrue(userStrict.isStrict()); assertFalse(userStrict.getMatchAsPath()); - assertEquals(2,userStrict.fields().size()); + assertEquals(2, userStrict.fields().size()); } } diff --git a/container-search/src/test/java/com/yahoo/search/query/test/RankingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/test/RankingTestCase.java index 09dbfb8e5c8..7a9080f46db 100644 --- a/container-search/src/test/java/com/yahoo/search/query/test/RankingTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/test/RankingTestCase.java @@ -80,13 +80,6 @@ public class RankingTestCase { q.getRanking().setSorting((String)null); } - /** Tests deprecated naming */ - @Test - public void testFeatureOverride() { - Query query = new Query("?query=abc&featureoverride.something=2"); - assertEquals(2, query.getRanking().getFeatures().getDouble("something").getAsDouble(), 0.0000001); - } - @Test public void testStructuredRankProperty() { Query query = new Query("?query=abc&rankproperty.distanceToPath(gps_position).path=(0,0,10,0,10,5,20,5)"); diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index 3236cbd5bbc..570b1d006de 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.search.rendering; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.util.concurrent.ListenableFuture; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.Chain; import com.yahoo.container.QrSearchersConfig; @@ -57,7 +56,9 @@ import com.yahoo.text.Utf8; import com.yahoo.yolean.Exceptions; import com.yahoo.yolean.trace.TraceNode; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.Timeout; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -83,6 +84,9 @@ public class JsonRendererTestCase { private static final ObjectMapper jsonMapper = new ObjectMapper(); + @Rule + public Timeout globalTimeout = Timeout.seconds(300); + private final JsonRenderer originalRenderer; private JsonRenderer renderer; @@ -205,14 +209,14 @@ public class JsonRendererTestCase { + " \"object\": \"thingie\"," + " \"string\": \"stuff\"," + " \"predicate\": \"a in [b]\"," - + " \"tensor1\": { \"cells\": [ { \"address\": {\"x\": \"a\"}, \"value\":2.0 } ] }," + + " \"tensor1\": { \"type\": \"tensor(x{})\", \"cells\": { \"a\":2.0 } }," + " \"tensor2\": { \"cells\": [] }," - + " \"tensor3\": { \"cells\": [ { \"address\": {\"x\": \"a\", \"y\": \"0\"}, \"value\":2.0 }, { \"address\": {\"x\": \"a\", \"y\": \"1\"}, \"value\":-1.0 } ] }," + + " \"tensor3\": { \"type\": \"tensor(x{},y{})\", \"cells\": [ { \"address\": {\"x\": \"a\", \"y\": \"0\"}, \"value\":2.0 }, { \"address\": {\"x\": \"a\", \"y\": \"1\"}, \"value\":-1.0 } ] }," + " \"summaryfeatures\": {" + " \"scalar1\":1.5," + " \"scalar2\":2.5," - + " \"tensor1\":{\"type\":\"tensor(x[3])\",\"cells\":[{\"address\":{\"x\":\"0\"},\"value\":1.5},{\"address\":{\"x\":\"1\"},\"value\":2.0},{\"address\":{\"x\":\"2\"},\"value\":2.5}]}," - + " \"tensor2\":{\"type\":\"tensor()\",\"cells\":[{\"address\":{},\"value\":0.5}]}" + + " \"tensor1\":{\"type\":\"tensor(x[3])\", \"values\":[1.5, 2.0, 2.5] }," + + " \"tensor2\":{\"type\":\"tensor()\", \"values\":[0.5] }" + " }," + " \"data\": \"Data \\\\xc3\\\\xa6 \\\\xc3\\\\xa5\"" + " }," @@ -409,7 +413,7 @@ public class JsonRendererTestCase { assertEqualJson(expected, summary); } - @SuppressWarnings({"unchecked", "removal"}) + @SuppressWarnings({"unchecked"}) @Test public void testTracingWithEmptySubtree() throws IOException, InterruptedException, ExecutionException { String expected = "{" @@ -1310,6 +1314,11 @@ public class JsonRendererTestCase { r.setTotalHitCount(1L); String summary = render(r); assertEqualJson(expected.toString(), summary); + r = new Result(new Query("/?")); + r.hits().add(h); + r.setTotalHitCount(1L); + summary = render(r); + assertEqualJson(expected.toString(), summary); r = new Result(new Query("/?renderer.json.jsonMaps=false")); expected = dataFromSimplified( @@ -1438,7 +1447,6 @@ public class JsonRendererTestCase { return render(execution, r); } - @SuppressWarnings("removal") private String render(Execution execution, Result r) throws InterruptedException, ExecutionException { ByteArrayOutputStream bs = new ByteArrayOutputStream(); CompletableFuture<Boolean> f = renderer.renderResponse(bs, r, execution, null); diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/FutureDataTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/FutureDataTestCase.java index 2426b18f018..15517270cb9 100644 --- a/container-search/src/test/java/com/yahoo/search/searchchain/test/FutureDataTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/FutureDataTestCase.java @@ -32,7 +32,6 @@ import static org.junit.Assert.assertTrue; */ public class FutureDataTestCase { - @SuppressWarnings("removal") @Test public void testAsyncFederation() throws InterruptedException, ExecutionException { // Setup environment @@ -73,13 +72,12 @@ public class FutureDataTestCase { asyncProviderSearcher.simulateOneHitIOComplete(new Hit("async:1")); asyncProviderSearcher.simulateAllHitsIOComplete(); assertEquals("Got no async data yet, as we haven't pulled it", 0, asyncGroup.size()); - asyncGroup.complete().get(); + asyncGroup.completeFuture().get(); assertEquals("Completed, so we have the data", 2, asyncGroup.size()); assertEquals("async:0", asyncGroup.get(0).getId().toString()); assertEquals("async:1", asyncGroup.get(1).getId().toString()); } - @SuppressWarnings("removal") @Test public void testFutureData() throws InterruptedException, ExecutionException, TimeoutException { // Set up @@ -104,7 +102,7 @@ public class FutureDataTestCase { // Results with future hit groups will be passed to rendering directly and start rendering immediately. // For this test we block and wait for the data instead: - result.hits().complete().get(1000, TimeUnit.MILLISECONDS); + result.hits().completeFuture().get(1000, TimeUnit.MILLISECONDS); assertEquals(2, result.hits().getConcreteSize()); } diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index b3aeda65f4b..bddba51f17a 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -561,9 +561,9 @@ public class QueryTestCase { Set<String> traces = new HashSet<>(); for (String trace : q.getContext(true).getTrace().traceNode().descendants(String.class)) traces.add(trace); - assertTrue(traces.contains("trace1: [select * from sources * where default contains \"foo\";]")); + assertTrue(traces.contains("trace1: [select * from sources * where default contains \"foo\"]")); assertTrue(traces.contains("trace2")); - assertTrue(traces.contains("trace3-1, trace3-2: [select * from sources * where default contains \"foo\";]")); + assertTrue(traces.contains("trace3-1, trace3-2: [select * from sources * where default contains \"foo\"]")); assertTrue(traces.contains("trace4-1, trace4-2")); } diff --git a/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java index bd666edcba3..1cc810aee7d 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java @@ -229,7 +229,7 @@ public class MinimalQueryInserterTestCase { execution.search(query); assertEquals(7, query.getOffset()); assertEquals(24, query.getHits()); - assertEquals("select * from sources * where title contains \"madonna\" limit 31 offset 7;", + assertEquals("select * from sources * where title contains \"madonna\" limit 31 offset 7", query.yqlRepresentation()); } @@ -258,7 +258,7 @@ public class MinimalQueryInserterTestCase { Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20timeout%2051"); execution.search(query); assertEquals(51L, query.getTimeout()); - assertEquals("select * from sources * where title contains \"madonna\" timeout 51;", query.yqlRepresentation()); + assertEquals("select * from sources * where title contains \"madonna\" timeout 51", query.yqlRepresentation()); } @Test @@ -266,7 +266,7 @@ public class MinimalQueryInserterTestCase { var select = "select x, y, z from sources * " + "where (x > 0 AND y contains \"foo\" AND z contains \"bar\") " + "order by x limit 20 offset 10 timeout 30 " - + "| all(group(y) max(3) each(output(count())));"; + + "| all(group(y) max(3) each(output(count())))"; Query query = new Query("search/?yql="+ URLEncoder.encode(select, Charsets.UTF_8)); execution.search(query); @@ -294,7 +294,7 @@ public class MinimalQueryInserterTestCase { .fieldOrders().get(1).getFieldName()); assertEquals(Order.DESCENDING, query.getRanking().getSorting() .fieldOrders().get(1).getSortOrder()); - assertEquals("select ignoredfield from ignoredsource where title contains \"madonna\" order by something, shoesize desc limit 300 timeout 3;", query.yqlRepresentation()); + assertEquals("select ignoredfield from ignoredsource where title contains \"madonna\" order by something, shoesize desc limit 300 timeout 3", query.yqlRepresentation()); } { String yql = "select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20order%20by%20other%20limit%20300%20timeout%203"; @@ -304,7 +304,7 @@ public class MinimalQueryInserterTestCase { .get(0).getFieldName()); assertEquals(Order.ASCENDING, query.getRanking().getSorting() .fieldOrders().get(0).getSortOrder()); - assertEquals("select ignoredfield from ignoredsource where title contains \"madonna\" order by other limit 300 timeout 3;", query.yqlRepresentation()); + assertEquals("select ignoredfield from ignoredsource where title contains \"madonna\" order by other limit 300 timeout 3", query.yqlRepresentation()); } { String yql = "select%20foo%20from%20bar%20where%20title%20contains%20%22madonna%22%20order%20by%20%5B%7B%22function%22%3A%20%22uca%22%2C%20%22locale%22%3A%20%22en_US%22%2C%20%22strength%22%3A%20%22IDENTICAL%22%7D%5Dother%20desc%2C%20%5B%7B%22function%22%3A%20%22lowercase%22%7D%5Dsomething%20limit%20300%20timeout%203"; @@ -327,7 +327,7 @@ public class MinimalQueryInserterTestCase { AttributeSorter sorter = fieldOrder.getSorter(); assertEquals(LowerCaseSorter.class, sorter.getClass()); } - assertEquals("select foo from bar where title contains \"madonna\" order by [{\"function\": \"uca\", \"locale\": \"en_US\", \"strength\": \"IDENTICAL\"}]other desc, [{\"function\": \"lowercase\"}]something limit 300 timeout 3;", + assertEquals("select foo from bar where title contains \"madonna\" order by [{\"function\": \"uca\", \"locale\": \"en_US\", \"strength\": \"IDENTICAL\"}]other desc, [{\"function\": \"lowercase\"}]something limit 300 timeout 3", query.yqlRepresentation()); } } @@ -337,7 +337,7 @@ public class MinimalQueryInserterTestCase { String yql = "select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20order%20by%20something%2C%20shoesize%20desc%20limit%20300%20timeout%203"; Query query = new Query("search/?yql=" + yql); execution.search(query); - assertEquals("select ignoredfield from ignoredsource where title contains \"madonna\" order by something, shoesize desc limit 300 timeout 3;", + assertEquals("select ignoredfield from ignoredsource where title contains \"madonna\" order by something, shoesize desc limit 300 timeout 3", query.yqlRepresentation()); } @@ -345,7 +345,7 @@ public class MinimalQueryInserterTestCase { public void testAndSegmenting() { Query query = new Query("?yql=select%20%2A%20from%20sources%20%2A%20where%20%5B%7B%22defaultIndex%22%3A%20%22default%22%2C%22grammar%22%3A%20%22web%22%2C%22stem%22%3A%20true%2C%22allowEmpty%22%3A%20true%7D%5DuserInput%28%40animal%29%3B&animal=m%26m%27s&tracelevel=3"); execution.search(query); - assertEquals("select * from sources * where (default contains \"m\" AND default contains ([{origin: {original: \"m\\'s\", offset: 0, length: 3}, andSegmenting: true}]phrase(\"m\", \"s\")));", + assertEquals("select * from sources * where (default contains \"m\" AND default contains ([{origin: {original: \"m\\'s\", offset: 0, length: 3}, andSegmenting: true}]phrase(\"m\", \"s\")))", query.yqlRepresentation()); } diff --git a/container-search/src/test/java/com/yahoo/search/yql/TermListTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/TermListTestCase.java index efaaaa5fca7..30254aa5738 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/TermListTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/TermListTestCase.java @@ -26,7 +26,7 @@ public class TermListTestCase { builder.setParameter("myTerms", "{'1':1, '2':1, 3:1}"); builder.setParameter("yql", "select * from sources * where weightedSet(user_id, @myTerms)"); Query query = searchAndAssertNoErrors(builder); - assertEquals("select * from sources * where weightedSet(user_id, {\"1\": 1, \"2\": 1, \"3\": 1});", + assertEquals("select * from sources * where weightedSet(user_id, {\"1\": 1, \"2\": 1, \"3\": 1})", query.yqlRepresentation()); } @@ -36,7 +36,7 @@ public class TermListTestCase { builder.setParameter("myTerms", "{'1':1, 2:1, '3':1}"); builder.setParameter("yql", "select * from sources * where wand(user_id, @myTerms)"); Query query = searchAndAssertNoErrors(builder); - assertEquals("select * from sources * where wand(user_id, {\"1\": 1, \"2\": 1, \"3\": 1});", + assertEquals("select * from sources * where wand(user_id, {\"1\": 1, \"2\": 1, \"3\": 1})", query.yqlRepresentation()); } @@ -46,7 +46,7 @@ public class TermListTestCase { builder.setParameter("myTerms", "{'1':1, '2':1, '3':1}"); builder.setParameter("yql", "select * from sources * where dotProduct(user_id, @myTerms)"); Query query = searchAndAssertNoErrors(builder); - assertEquals("select * from sources * where dotProduct(user_id, {\"1\": 1, \"2\": 1, \"3\": 1});", + assertEquals("select * from sources * where dotProduct(user_id, {\"1\": 1, \"2\": 1, \"3\": 1})", query.yqlRepresentation()); } diff --git a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java index c41de3a73f1..1e36bc3ad45 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java @@ -50,14 +50,14 @@ public class UserInputTestCase { URIBuilder builder = searchUri(); builder.setParameter("yql", "select * from sources * where userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); - assertEquals("select * from sources * where default contains \"nalle\";", query.yqlRepresentation()); + assertEquals("select * from sources * where default contains \"nalle\"", query.yqlRepresentation()); } { URIBuilder builder = searchUri(); builder.setParameter("nalle", "bamse"); builder.setParameter("yql", "select * from sources * where userInput(@nalle)"); Query query = searchAndAssertNoErrors(builder); - assertEquals("select * from sources * where default contains \"bamse\";", query.yqlRepresentation()); + assertEquals("select * from sources * where default contains \"bamse\"", query.yqlRepresentation()); } { URIBuilder builder = searchUri(); @@ -74,7 +74,7 @@ public class UserInputTestCase { URIBuilder builder = searchUri(); builder.setParameter("yql", "select * from sources * where {grammar: \"raw\"}userInput(\"nal le\")"); Query query = searchAndAssertNoErrors(builder); - assertEquals("select * from sources * where default contains \"nal le\";", query.yqlRepresentation()); + assertEquals("select * from sources * where default contains \"nal le\"", query.yqlRepresentation()); } @Test @@ -83,7 +83,7 @@ public class UserInputTestCase { builder.setParameter("yql", "select * from sources * where {grammar: \"segment\"}userInput(\"nal le\")"); Query query = searchAndAssertNoErrors(builder); - assertEquals("select * from sources * where default contains ([{origin: {original: \"nal le\", offset: 0, length: 6}}]phrase(\"nal\", \"le\"));", query.yqlRepresentation()); + assertEquals("select * from sources * where default contains ([{origin: {original: \"nal le\", offset: 0, length: 6}}]phrase(\"nal\", \"le\"))", query.yqlRepresentation()); } @Test @@ -92,7 +92,7 @@ public class UserInputTestCase { builder.setParameter("yql", "select * from sources * where {grammar: \"segment\"}userInput(\"^^^^^^^^\")"); Query query = searchAndAssertNoErrors(builder); - assertEquals("select * from sources * where default contains \"^^^^^^^^\";", query.yqlRepresentation()); + assertEquals("select * from sources * where default contains \"^^^^^^^^\"", query.yqlRepresentation()); } @Test @@ -100,7 +100,7 @@ public class UserInputTestCase { URIBuilder builder = searchUri(); builder.setParameter("yql", "select * from sources * where {grammar: \"any\"}userInput('foo bar')"); Query query = searchAndAssertNoErrors(builder); - assertEquals("select * from sources * where (default contains \"foo\" OR default contains \"bar\");", + assertEquals("select * from sources * where (default contains \"foo\" OR default contains \"bar\")", query.yqlRepresentation()); } @@ -109,7 +109,7 @@ public class UserInputTestCase { URIBuilder builder = searchUri(); builder.setParameter("yql", "select * from sources * where {grammar: \"all\"}userInput('foo bar')"); Query query = searchAndAssertNoErrors(builder); - assertEquals("select * from sources * where (default contains \"foo\" AND default contains \"bar\");", + assertEquals("select * from sources * where (default contains \"foo\" AND default contains \"bar\")", query.yqlRepresentation()); } @@ -118,7 +118,7 @@ public class UserInputTestCase { URIBuilder builder = searchUri(); builder.setParameter("yql", "select * from sources * where {grammar: \"weakAnd\"}userInput('foo bar')"); Query query = searchAndAssertNoErrors(builder); - assertEquals("select * from sources * where weakAnd(default contains \"foo\", default contains \"bar\");", + assertEquals("select * from sources * where weakAnd(default contains \"foo\", default contains \"bar\")", query.yqlRepresentation()); } @@ -139,7 +139,7 @@ public class UserInputTestCase { builder.setParameter("yql", "select * from sources * where [{defaultIndex: \"glompf\"}]userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); - assertEquals("select * from sources * where glompf contains \"nalle\";", query.yqlRepresentation()); + assertEquals("select * from sources * where glompf contains \"nalle\"", query.yqlRepresentation()); } @Test @@ -149,7 +149,7 @@ public class UserInputTestCase { "select * from sources * where [{stem: false}]userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); assertEquals( - "select * from sources * where default contains ([{stem: false}]\"nalle\");", + "select * from sources * where default contains ([{stem: false}]\"nalle\")", query.yqlRepresentation()); } @@ -160,7 +160,7 @@ public class UserInputTestCase { builder.setParameter("yql", "select * from ecitem where rank(([{defaultIndex:\"myfield\"}](userInput(@myinput))))"); Query query = searchAndAssertNoErrors(builder); - assertEquals("select * from ecitem where rank(myfield = (-5));", query.yqlRepresentation()); + assertEquals("select * from ecitem where rank(myfield = (-5))", query.yqlRepresentation()); assertEquals("RANK myfield:-5", query.getModel().getQueryTree().getRoot().toString()); } @@ -171,7 +171,7 @@ public class UserInputTestCase { "select * from sources * where [{ranked: false}]userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); assertEquals( - "select * from sources * where default contains ([{ranked: false}]\"nalle\");", + "select * from sources * where default contains ([{ranked: false}]\"nalle\")", query.yqlRepresentation()); } @@ -182,7 +182,7 @@ public class UserInputTestCase { "select * from sources * where [{filter: true}]userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); assertEquals( - "select * from sources * where default contains ([{filter: true}]\"nalle\");", + "select * from sources * where default contains ([{filter: true}]\"nalle\")", query.yqlRepresentation()); } @@ -194,7 +194,7 @@ public class UserInputTestCase { "select * from sources * where [{normalizeCase: false}]userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); assertEquals( - "select * from sources * where default contains ([{normalizeCase: false}]\"nalle\");", + "select * from sources * where default contains ([{normalizeCase: false}]\"nalle\")", query.yqlRepresentation()); } @@ -205,7 +205,7 @@ public class UserInputTestCase { "select * from sources * where [{accentDrop: false}]userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); assertEquals( - "select * from sources * where default contains ([{accentDrop: false}]\"nalle\");", + "select * from sources * where default contains ([{accentDrop: false}]\"nalle\")", query.yqlRepresentation()); } @@ -216,7 +216,7 @@ public class UserInputTestCase { "select * from sources * where [{usePositionData: false}]userInput(\"nalle\")"); Query query = searchAndAssertNoErrors(builder); assertEquals( - "select * from sources * where default contains ([{usePositionData: false}]\"nalle\");", + "select * from sources * where default contains ([{usePositionData: false}]\"nalle\")", query.yqlRepresentation()); } @@ -228,7 +228,7 @@ public class UserInputTestCase { builder.setParameter("yql", "select * from sources * where foo contains @nalle and foo contains phrase(@nalle, @meta, @nalle)"); Query query = searchAndAssertNoErrors(builder); - assertEquals("select * from sources * where (foo contains \"bamse\" AND foo contains phrase(\"bamse\", \"syntactic\", \"bamse\"));", query.yqlRepresentation()); + assertEquals("select * from sources * where (foo contains \"bamse\" AND foo contains phrase(\"bamse\", \"syntactic\", \"bamse\"))", query.yqlRepresentation()); } @Test @@ -237,7 +237,7 @@ public class UserInputTestCase { builder.setParameter("varref", "1980"); builder.setParameter("yql", "select * from sources * where year > @varref"); Query query = searchAndAssertNoErrors(builder); - assertEquals("select * from sources * where year > 1980;", query.yqlRepresentation()); + assertEquals("select * from sources * where year > 1980", query.yqlRepresentation()); } @Test @@ -248,7 +248,7 @@ public class UserInputTestCase { "select * from sources * where myfield contains 'token'" + "| [{'continuations':[@continuation, 'BCBKCBACBKCCK'] }] all(group(f) each(output(count())))"); Query query = searchAndAssertNoErrors(builder); - assertEquals("select * from sources * where myfield contains \"token\" | [{ continuations:['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]all(group(f) each(output(count())));", query.yqlRepresentation()); + assertEquals("select * from sources * where myfield contains \"token\" | [{ continuations:['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]all(group(f) each(output(count())))", query.yqlRepresentation()); } @Test @@ -258,7 +258,7 @@ public class UserInputTestCase { builder.setParameter("yql", "select foo from bar where fieldName contains equiv(@term,'B')"); Query query = searchAndAssertNoErrors(builder); - assertEquals("select foo from bar where fieldName contains equiv(\"A\", \"B\");", query.yqlRepresentation()); + assertEquals("select foo from bar where fieldName contains equiv(\"A\", \"B\")", query.yqlRepresentation()); } private Query searchAndAssertNoErrors(URIBuilder builder) { diff --git a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java index 8a90d224003..5c37f590619 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java @@ -297,9 +297,6 @@ public class VespaSerializerTestCase { @Test public void testAnnotatedWeakAnd() { parseAndConfirm("([{" + YqlParser.TARGET_NUM_HITS + ": 10}]weakAnd(a contains \"A\", b contains \"B\"))"); - parseAndConfirm("([{" + YqlParser.SCORE_THRESHOLD + ": 10}]weakAnd(a contains \"A\", b contains \"B\"))"); - parseAndConfirm("([{" + YqlParser.TARGET_NUM_HITS + ": 10, " + YqlParser.SCORE_THRESHOLD - + ": 20}]weakAnd(a contains \"A\", b contains \"B\"))"); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 93b0ea60de2..c6811fdc4d4 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -181,7 +181,7 @@ public class YqlParserTestCase { assertParse("select foo from bar where title contains \"madonna\" and title contains \"saint\"", "AND title:madonna title:saint"); assertParse("select foo from bar where title contains \"madonna\" and title contains \"saint\" and title " + - "contains \"angel\";", + "contains \"angel\"", "AND title:madonna title:saint title:angel"); } @@ -307,7 +307,7 @@ public class YqlParserTestCase { public void testTermAnnotations() { assertEquals("merkelapp", getRootWord("select foo from bar where baz contains " + - "({label: \"merkelapp\"}\"colors\");").getLabel()); + "({label: \"merkelapp\"}\"colors\")").getLabel()); assertEquals("another", getRootWord("select foo from bar where baz contains " + "({annotations: {cox: \"another\"}}\"colors\")").getAnnotation("cox")); @@ -336,14 +336,14 @@ public class YqlParserTestCase { public void testAnnotationsCanBeInBrackets() { assertEquals("merkelapp", getRootWord("select foo from bar where baz contains " + - "([ {label: \"merkelapp\"} ]\"colors\");").getLabel()); + "([ {label: \"merkelapp\"} ]\"colors\")").getLabel()); } @Test public void testValuesCanBeQuoted() { assertEquals("merkelapp", getRootWord("select foo from bar where baz contains " + - "( {label: \"merkelapp\"} \"colors\");").getLabel()); + "( {label: \"merkelapp\"} \"colors\")").getLabel()); } @Test @@ -411,7 +411,7 @@ public class YqlParserTestCase { assertFalse(getRootWord("select foo from bar where baz contains " + "([ {stem: true} ]\"colors\")").isStemmed()); assertFalse(getRootWord("select foo from bar where baz contains " + - "\"colors\";").isStemmed()); + "\"colors\"").isStemmed()); } @Test @@ -614,7 +614,7 @@ public class YqlParserTestCase { //This test is order dependent. Fix it! @Test public void testDotProduct() { - assertParse("select foo from bar where dotProduct(description, {\"a\":1, \"b\":2});", + assertParse("select foo from bar where dotProduct(description, {\"a\":1, \"b\":2})", "DOTPRODUCT description{[1]:\"a\",[2]:\"b\"}"); assertParse("select foo from bar where dotProduct(description, {\"a\":2})", "DOTPRODUCT description{[2]:\"a\"}"); @@ -705,11 +705,10 @@ public class YqlParserTestCase { "b contains \"B\")", "WEAKAND(37) a:A b:B"); - QueryTree tree = parse("select foo from bar where {scoreThreshold: 41}weakAnd(a " + + QueryTree tree = parse("select foo from bar where weakAnd(a " + "contains \"A\", b contains \"B\")"); assertEquals("WEAKAND(100) a:A b:B", tree.toString()); assertEquals(WeakAndItem.class, tree.getRoot().getClass()); - assertEquals(41, ((WeakAndItem)tree.getRoot()).getScoreThreshold()); } @Test @@ -1166,7 +1165,6 @@ public class YqlParserTestCase { expectedYql.append("\"google com\")"); // but not with the simple syntax if (hasAnnotations) expectedYql.append(")"); - expectedYql.append(";"); assertEquals(expectedYql.toString(), query.yqlRepresentation()); assertTrue(query.getModel().getQueryTree().getRoot() instanceof PhraseItem); diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java index 9b867be1484..8ede243d3ca 100644 --- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java +++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java @@ -560,10 +560,9 @@ public class SelectTestCase { assertParse("{ \"weakAnd\": { \"children\" : [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ], \"attributes\" : {\"targetHits\": 37} }}", "WEAKAND(37) a:A b:B"); - QueryTree tree = parseWhere("{ \"weakAnd\": { \"children\" : [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ], \"attributes\" : {\"scoreThreshold\": 41}}}"); + QueryTree tree = parseWhere("{ \"weakAnd\": { \"children\" : [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ] }}"); assertEquals("WEAKAND(100) a:A b:B", tree.toString()); assertEquals(WeakAndItem.class, tree.getRoot().getClass()); - assertEquals(41, ((WeakAndItem)tree.getRoot()).getScoreThreshold()); } @Test diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java index b1bc926daed..19fa8c50424 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java @@ -1,10 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.streamingvisitors; +import com.yahoo.document.fieldset.AllFields; import com.yahoo.document.select.parser.ParseException; import com.yahoo.documentapi.*; -import com.yahoo.documentapi.messagebus.loadtypes.LoadType; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.messagebus.protocol.DocumentSummaryMessage; import com.yahoo.documentapi.messagebus.protocol.QueryResultMessage; @@ -31,14 +30,7 @@ import static org.junit.Assert.*; /** * @author <a href="mailto:ulf@yahoo-inc.com">Ulf Carlin</a> */ -@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public class VdsVisitorTestCase { - private LoadTypeSet loadTypeSet = new LoadTypeSet(); // TODO remove on Vespa 8 - - public VdsVisitorTestCase() { - loadTypeSet.addLoadType(1, "low", DocumentProtocol.Priority.LOW_1); - loadTypeSet.addLoadType(2, "normal", DocumentProtocol.Priority.NORMAL_1); - } private SearchResult createSR(String docId, double rank) { BufferSerializer serializer = new BufferSerializer(); @@ -116,7 +108,6 @@ public class VdsVisitorTestCase { String selection = null; long from = 0; long to = 0; - String loadTypeName = null; DocumentProtocol.Priority priority = null; int maxBucketsPerVisitor = 0; @@ -140,7 +131,6 @@ public class VdsVisitorTestCase { selection = null; from = 123; to = 456; - loadTypeName = "low"; priority = DocumentProtocol.Priority.HIGH_2; maxBucketsPerVisitor = 2; @@ -194,9 +184,6 @@ public class VdsVisitorTestCase { if (qa.to != 0) { queryString.append("&streaming.totimestamp=").append(qa.to); } - if (qa.loadTypeName != null) { - queryString.append("&streaming.loadtype=").append(qa.loadTypeName); - } if (qa.priority != null) { queryString.append("&streaming.priority=").append(qa.priority); } @@ -228,28 +215,16 @@ public class VdsVisitorTestCase { } assertEquals(qa.from, params.getFromTimestamp()); assertEquals(qa.to, params.getToTimestamp()); - if (qa.loadTypeName != null && loadTypeSet.getNameMap().get(qa.loadTypeName) != null) { - LoadType expectedLoadType = loadTypeSet.getNameMap().get(qa.loadTypeName); - assertEquals(expectedLoadType, params.getLoadType()); - if (qa.priority != null) { - assertEquals(qa.priority, params.getPriority()); - } else { - assertEquals(expectedLoadType.getPriority(), params.getPriority()); - } + if (qa.priority != null) { + assertEquals(qa.priority, params.getPriority()); } else { - assertEquals(LoadType.DEFAULT, params.getLoadType()); - if (qa.priority != null) { - assertEquals(qa.priority, params.getPriority()); - } else { - assertEquals(DocumentProtocol.Priority.VERY_HIGH, params.getPriority()); - } + assertEquals(DocumentProtocol.Priority.VERY_HIGH, params.getPriority()); } if (qa.maxBucketsPerVisitor != 0) { assertEquals(qa.maxBucketsPerVisitor, params.getMaxBucketsPerVisitor()); } else { assertEquals(VdsVisitor.MAX_BUCKETS_PER_VISITOR, params.getMaxBucketsPerVisitor()); } - assertEquals(false, params.getDynamicallyIncreaseMaxBucketsPerVisitor()); // Verify parameters based only on query assertEquals(qa.timeout*1000, params.getTimeoutMs(),0.0000001); @@ -257,6 +232,7 @@ public class VdsVisitorTestCase { assertEquals("searchvisitor", params.getVisitorLibrary()); assertEquals(Integer.MAX_VALUE, params.getMaxPending()); assertEquals(qa.traceLevel, params.getTraceLevel()); + assertEquals(AllFields.NAME, params.getFieldSet()); // Verify library parameters //System.err.println("query="+new String(params.getLibraryParameters().get("query"))); @@ -322,7 +298,7 @@ public class VdsVisitorTestCase { public void testBasics() throws Exception { Route route = Route.parse("storageClusterRouteSpec"); String searchCluster = "searchClusterConfigId"; - MockVisitorSessionFactory factory = new MockVisitorSessionFactory(loadTypeSet); + MockVisitorSessionFactory factory = new MockVisitorSessionFactory(); // Default values and no selection QueryArguments qa = new QueryArguments(); @@ -331,10 +307,6 @@ public class VdsVisitorTestCase { // Groupdoc qa.groupName = "group"; qa.maxBucketsPerVisitor = 2; // non-default maxBucketsPerVisitor - qa.loadTypeName = "normal"; // non-default loadTypeName, default priority - verifyVisitorOk(factory, qa, route, searchCluster); - - qa.loadTypeName = "unknown"; // unknown loadTypeName, default priority verifyVisitorOk(factory, qa, route, searchCluster); qa.priority = DocumentProtocol.Priority.NORMAL_2; // unknown loadTypeName, non-default priority @@ -349,7 +321,7 @@ public class VdsVisitorTestCase { public void testFailures() throws Exception { Route route = Route.parse("storageClusterRouteSpec"); String searchCluster = "searchClusterConfigId"; - MockVisitorSessionFactory factory = new MockVisitorSessionFactory(loadTypeSet); + MockVisitorSessionFactory factory = new MockVisitorSessionFactory(); // Default values and no selection QueryArguments qa = new QueryArguments(); @@ -490,13 +462,10 @@ public class VdsVisitorTestCase { private static class MockVisitorSessionFactory implements VdsVisitor.VisitorSessionFactory { private VisitorParameters params; - private LoadTypeSet loadTypeSet; // TODO remove on Vespa 8 private boolean timeoutQuery = false; private boolean failQuery = false; - private MockVisitorSessionFactory(LoadTypeSet loadTypeSet) { - this.loadTypeSet = loadTypeSet; - } + private MockVisitorSessionFactory() {} @Override public VisitorSession createVisitorSession(VisitorParameters params) throws ParseException { @@ -504,12 +473,6 @@ public class VdsVisitorTestCase { return new MockVisitorSession(params, timeoutQuery, failQuery); } - @Override - // TODO: Remove on Vespa 8 - public LoadTypeSet getLoadTypeSet() { - return loadTypeSet; - } - public VisitorParameters getParams() { return params; } diff --git a/container-spifly/.gitignore b/container-spifly/.gitignore new file mode 100644 index 00000000000..916e17c097a --- /dev/null +++ b/container-spifly/.gitignore @@ -0,0 +1 @@ +dependency-reduced-pom.xml diff --git a/statistics/CMakeLists.txt b/container-spifly/CMakeLists.txt index f8d34754c41..cd4b4a43488 100644 --- a/statistics/CMakeLists.txt +++ b/container-spifly/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_config_definitions() +install_jar(container-spifly.jar) diff --git a/container-spifly/README.md b/container-spifly/README.md new file mode 100644 index 00000000000..4477c6dabb9 --- /dev/null +++ b/container-spifly/README.md @@ -0,0 +1 @@ +Repackaging of SPIFly with ASM embedded diff --git a/container-spifly/pom.xml b/container-spifly/pom.xml new file mode 100644 index 00000000000..09a58a6fc34 --- /dev/null +++ b/container-spifly/pom.xml @@ -0,0 +1,105 @@ +<?xml version="1.0"?> +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <artifactId>container-spifly</artifactId> + <packaging>jar</packaging> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>8-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + + <properties> + <relocation-pkg-prefix>com.yahoo.vespa.spifly.repackaged</relocation-pkg-prefix> + </properties> + + <dependencies> + <!-- Required for ServiceLoader to function in OSGi environment. ServiceLoader is used by Jetty --> + <dependency> + <groupId>org.apache.aries.spifly</groupId> + <artifactId>org.apache.aries.spifly.dynamic.bundle</artifactId> + <scope>compile</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <minimizeJar>false</minimizeJar> + <filters> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>module-info.class</exclude> + <exclude>META-INF/*</exclude> + </excludes> + </filter> + </filters> + <relocations> + <relocation> + <pattern>org.apache.aries.spifly</pattern> + <shadedPattern>${relocation-pkg-prefix}.spifly</shadedPattern> + </relocation> + <relocation> + <pattern>aQute</pattern> + <shadedPattern>${relocation-pkg-prefix}.aQute</shadedPattern> + </relocation> + <relocation> + <pattern>org.objectweb.asm</pattern> + <shadedPattern>${relocation-pkg-prefix}.asm</shadedPattern> + </relocation> + </relocations> + <transformers> + <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> + <manifestEntries> + <Bundle-Name>container-spifly</Bundle-Name> + <Bundle-SymbolicName>container-spifly</Bundle-SymbolicName> + <Bundle-Version>${spifly.version}</Bundle-Version> + <Export-Package> + ${relocation-pkg-prefix}.spifly;version="${spifly.version}", + ${relocation-pkg-prefix}.spifly.dynamic;version="${spifly.version}", + ${relocation-pkg-prefix}.spifly.weaver;version="${spifly.version}" + </Export-Package> + <Import-Package> + org.osgi.framework;version="[1.7,2)", + org.osgi.framework.hooks.weaving;version="[1.0,2)", + org.osgi.framework.wiring;version="[1.1,2)", + org.osgi.util.tracker;version="[1.5,2)" + </Import-Package> + <Require-Capability>osgi.ee</Require-Capability> + <Provide-Capability> + osgi.extender;osgi.extender="osgi.serviceloader.registrar";version:Version="1.0", + osgi.extender;osgi.extender="osgi.serviceloader.processor";version:Version="1.0";uses:="${relocation-pkg-prefix}.spifly" + </Provide-Capability> + <Bundle-Activator> + ${relocation-pkg-prefix}.spifly.dynamic.DynamicWeavingActivator + </Bundle-Activator> + </manifestEntries> + </transformer> + </transformers> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + +</project> diff --git a/container-test/pom.xml b/container-test/pom.xml index 961d827a390..a938329dd72 100644 --- a/container-test/pom.xml +++ b/container-test/pom.xml @@ -10,11 +10,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-test</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>jar</packaging> <dependencies> <dependency> @@ -97,5 +97,63 @@ <artifactId>commons-compress</artifactId> <scope>compile</scope> </dependency> + <dependency> + <groupId>org.bouncycastle</groupId> + <artifactId>bcpkix-jdk15on</artifactId> + </dependency> + <dependency> + <groupId>org.bouncycastle</groupId> + <artifactId>bcprov-jdk15on</artifactId> + </dependency> + <dependency> + <!-- required for container-search code using org.json --> + <groupId>org.json</groupId> + <artifactId>json</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <!-- TODO: this, and probably others, could be removed from here if we make the fat jar the default artifact for jdisc_core --> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> + </dependency> + + <!-- START JETTY embedded jars --> + <dependency> + <groupId>org.eclipse.jetty.alpn</groupId> + <artifactId>alpn-api</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty.http2</groupId> + <artifactId>http2-server</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-alpn-java-server</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-client</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-continuation</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-jmx</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlet</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlets</artifactId> + </dependency> + <!-- END JETTY embedded jars --> </dependencies> </project> diff --git a/container/pom.xml b/container/pom.xml index 0b6bef77282..b2a3a1baef8 100644 --- a/container/pom.xml +++ b/container/pom.xml @@ -10,11 +10,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>jar</packaging> <dependencies> <dependency> diff --git a/controller-api/pom.xml b/controller-api/pom.xml index 42e96e0331a..f7057c93561 100644 --- a/controller-api/pom.xml +++ b/controller-api/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>controller-api</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> @@ -67,6 +67,13 @@ <scope>provided</scope> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>security-utils</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <!-- compile --> <dependency> diff --git a/controller-server/pom.xml b/controller-server/pom.xml index ff7367e03b2..773d63202b6 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>controller-server</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index c0829adc7ae..5ff15307ac3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -413,6 +413,7 @@ public class RoutingController { /** Create a common name based on a hash of given application. This must be less than 64 characters long. */ private static String commonNameHashOf(ApplicationId application, SystemName system) { + @SuppressWarnings("deprecation") // for Hashing.sha1() HashCode sha1 = Hashing.sha1().hashString(application.serializedForm(), StandardCharsets.UTF_8); String base32 = BaseEncoding.base32().omitPadding().lowerCase().encode(sha1.asBytes()); return 'v' + base32 + Endpoint.internalDnsSuffix(system); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java index 7b3b90d1458..4bc9aeb00e4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java @@ -20,7 +20,6 @@ import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; -import com.yahoo.slime.Type; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.deployment.ZipBuilder; import com.yahoo.yolean.Exceptions; @@ -98,6 +97,7 @@ public class ApplicationPackage { * it must not be further changed by the caller. * If 'requireFiles' is true, files needed by deployment orchestration must be present. */ + @SuppressWarnings("deprecation") // for Hashing.sha1() public ApplicationPackage(byte[] zippedContent, boolean requireFiles) { this.zippedContent = Objects.requireNonNull(zippedContent, "The application package content cannot be null"); this.contentHash = Hashing.sha1().hashBytes(zippedContent).toString(); @@ -240,6 +240,7 @@ public class ApplicationPackage { } // Hashes all files and settings that require a deployment to be forwarded to configservers + @SuppressWarnings("deprecation") // for Hashing.sha1() private String calculateBundleHash(byte[] zippedContent) { Predicate<String> entryMatcher = name -> ! name.endsWith(deploymentFile) && ! name.endsWith(buildMetaFile); SortedMap<String, Long> crcByEntry = new TreeMap<>(); @@ -261,6 +262,7 @@ public class ApplicationPackage { .hash().toString(); } + @SuppressWarnings("deprecation") // for Hashing.sha1() public static String calculateHash(byte[] bytes) { return Hashing.sha1().newHasher() .putBytes(bytes) @@ -274,14 +276,6 @@ public class ApplicationPackage { /** Max size of each extracted file */ private static final int maxSize = 10 << 20; // 10 Mb - // TODO: Vespa 8: Remove application/ directory support - private static final String applicationDir = "application/"; - - private static String withoutLegacyDir(String name) { - if (name.startsWith(applicationDir)) return name.substring(applicationDir.length()); - return name; - } - private final byte[] zip; private final Map<Path, Optional<byte[]>> cache; @@ -307,11 +301,11 @@ public class ApplicationPackage { private Map<Path, Optional<byte[]>> read(Collection<String> names) { var entries = ZipEntries.from(zip, - name -> names.contains(withoutLegacyDir(name)), + name -> names.contains(name), maxSize, true) .asList().stream() - .collect(toMap(entry -> Paths.get(withoutLegacyDir(entry.name())).normalize(), + .collect(toMap(entry -> Paths.get(entry.name()).normalize(), ZipEntries.ZipEntryWithContent::content)); names.stream().map(Paths::get).forEach(path -> entries.putIfAbsent(path.normalize(), Optional.empty())); return entries; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java index 270cfba00c0..33aeda5e011 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java @@ -67,12 +67,12 @@ public class EndpointCertificates { if(metadata.isPresent()) { var m = metadata.get(); GcpSecretStore gcpSecretStore = controller.serviceRegistry().gcpSecretStore(); - String mangledCertName = "endpointCert_" + m.certName().replace('.', '_'); // Google cloud does not accept dots in secrets, but they accept underscores - String mangledKeyName = "endpointCert_" + m.keyName().replace('.', '_'); // Google cloud does not accept dots in secrets, but they accept underscores + String mangledCertName = "endpointCert_" + m.certName().replace('.', '_') + "-v" + m.version(); // Google cloud does not accept dots in secrets, but they accept underscores + String mangledKeyName = "endpointCert_" + m.keyName().replace('.', '_') + "-v" + m.version(); // Google cloud does not accept dots in secrets, but they accept underscores if (gcpSecretStore.getSecret(mangledCertName, m.version()) == null) - gcpSecretStore.createSecret(mangledCertName + "-v" + m.version(), controller.secretStore().getSecret(m.certName(), m.version())); + gcpSecretStore.createSecret(mangledCertName, controller.secretStore().getSecret(m.certName(), m.version())); if (gcpSecretStore.getSecret(mangledKeyName, m.version()) == null) - gcpSecretStore.createSecret(mangledKeyName + "-v" + m.version(), controller.secretStore().getSecret(m.keyName(), m.version())); + gcpSecretStore.createSecret(mangledKeyName, controller.secretStore().getSecret(m.keyName(), m.version())); return Optional.of(m.withVersion(1).withKeyName(mangledKeyName).withCertName(mangledCertName)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 795f14bbde1..f7368aab143 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; +import com.yahoo.component.annotation.Inject; import com.yahoo.concurrent.maintenance.Maintainer; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; @@ -60,7 +60,6 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new CostReportMaintainer(controller, intervals.costReportMaintainer, controller.serviceRegistry().costReportConsumer())); maintainers.add(new ResourceMeterMaintainer(controller, intervals.resourceMeterMaintainer, metric, controller.serviceRegistry().meteringService())); maintainers.add(new ResourceTagMaintainer(controller, intervals.resourceTagMaintainer, controller.serviceRegistry().resourceTagger())); - maintainers.add(new SystemRoutingPolicyMaintainer(controller, intervals.systemRoutingPolicyMaintainer)); maintainers.add(new ApplicationMetaDataGarbageCollector(controller, intervals.applicationMetaDataGarbageCollector)); maintainers.add(new ArtifactExpirer(controller, intervals.containerImageExpirer)); maintainers.add(new HostInfoUpdater(controller, intervals.hostInfoUpdater)); @@ -117,9 +116,7 @@ public class ControllerMaintenance extends AbstractComponent { private final Duration nameServiceDispatcher; private final Duration costReportMaintainer; private final Duration resourceMeterMaintainer; - private final Duration cloudEventReporter; private final Duration resourceTagMaintainer; - private final Duration systemRoutingPolicyMaintainer; private final Duration applicationMetaDataGarbageCollector; private final Duration containerImageExpirer; private final Duration hostInfoUpdater; @@ -152,9 +149,7 @@ public class ControllerMaintenance extends AbstractComponent { this.nameServiceDispatcher = duration(10, SECONDS); this.costReportMaintainer = duration(2, HOURS); this.resourceMeterMaintainer = duration(3, MINUTES); - this.cloudEventReporter = duration(30, MINUTES); this.resourceTagMaintainer = duration(30, MINUTES); - this.systemRoutingPolicyMaintainer = duration(15, MINUTES); this.applicationMetaDataGarbageCollector = duration(12, HOURS); this.containerImageExpirer = duration(12, HOURS); this.hostInfoUpdater = duration(12, HOURS); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java deleted file mode 100644 index 5acb21917eb..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; - -import java.time.Duration; - -/** - * This maintains {@link RoutingPolicy}'s for {@link SystemApplication}s. In contrast to regular applications, this - * refreshes policies at an interval, not on deployment. - * - * @author mpolden - */ -public class SystemRoutingPolicyMaintainer extends ControllerMaintainer { - - public SystemRoutingPolicyMaintainer(Controller controller, Duration interval) { - super(controller, interval); - } - - @Override - protected double maintain() { - for (var zone : controller().zoneRegistry().zones().reachable().ids()) { - for (var application : SystemApplication.values()) { - if (!application.hasEndpoint()) continue; - DeploymentId deployment = new DeploymentId(application.id(), zone); - controller().routing().of(deployment).configure(DeploymentSpec.empty); - } - } - return 1.0; - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 75abf94bf0e..53557bafcb0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -811,14 +811,6 @@ public class ControllerTest { } @Test - public void testDeployApplicationPackageWithApplicationDir() { - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .region("us-west-1") - .build(true); - tester.newDeploymentContext().submit(applicationPackage); - } - - @Test public void testDeployApplicationWithWarnings() { var context = tester.newDeploymentContext(); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java index 99e22302c73..0458f77fc00 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageTest.java @@ -81,23 +81,6 @@ public class ApplicationPackageTest { } @Test - public void testMetaDataWithLegacyApplicationDirectory() { - byte[] zip = ApplicationPackage.filesZip(Map.of("application/deployment.xml", deploymentXml.getBytes(UTF_8), - "application/services.xml", servicesXml.getBytes(UTF_8), - "application/jdisc.xml", jdiscXml.getBytes(UTF_8), - "application/content/content.xml", contentXml.getBytes(UTF_8), - "application/content/nodes.xml", nodesXml.getBytes(UTF_8), - "application/gurba", "gurba".getBytes(UTF_8))); - - assertEquals(Map.of("deployment.xml", deploymentXml, - "services.xml", servicesXml, - "jdisc.xml", jdiscXml, - "content/content.xml", contentXml, - "content/nodes.xml", nodesXml), - unzip(new ApplicationPackage(zip, false).metaDataZip())); - } - - @Test public void testMetaDataWithMissingFiles() { byte[] zip = ApplicationPackage.filesZip(Map.of("services.xml", servicesXml.getBytes(UTF_8))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 490731ca06a..ea8f4db0346 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -337,23 +337,15 @@ public class ApplicationPackageBuilder { } public ApplicationPackage build() { - return build(false); - } - - public ApplicationPackage build(boolean useApplicationDir) { - String dir = ""; - if (useApplicationDir) { - dir = "application/"; - } ByteArrayOutputStream zip = new ByteArrayOutputStream(); try (ZipOutputStream out = new ZipOutputStream(zip)) { out.setLevel(Deflater.NO_COMPRESSION); // This is for testing purposes so we skip compression for performance - writeZipEntry(out, dir + "deployment.xml", deploymentSpec()); - writeZipEntry(out, dir + "validation-overrides.xml", validationOverrides()); - writeZipEntry(out, dir + "search-definitions/test.sd", searchDefinition()); - writeZipEntry(out, dir + "build-meta.json", buildMeta(compileVersion)); + writeZipEntry(out, "deployment.xml", deploymentSpec()); + writeZipEntry(out, "validation-overrides.xml", validationOverrides()); + writeZipEntry(out, "schemas/test.sd", searchDefinition()); + writeZipEntry(out, "build-meta.json", buildMeta(compileVersion)); if (!trustedCertificates.isEmpty()) { - writeZipEntry(out, dir + "security/clients.pem", X509CertificateUtils.toPem(trustedCertificates).getBytes(UTF_8)); + writeZipEntry(out, "security/clients.pem", X509CertificateUtils.toPem(trustedCertificates).getBytes(UTF_8)); } } catch (IOException e) { throw new UncheckedIOException(e); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainerTest.java deleted file mode 100644 index 8b2bfe8ee95..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainerTest.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; -import org.junit.Test; - -import java.time.Duration; -import java.util.List; -import java.util.Optional; -import java.util.Set; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertSame; - -/** - * @author mpolden - */ -public class SystemRoutingPolicyMaintainerTest { - - @Test - public void maintain() { - var tester = new ControllerTester(); - var updater = new SystemRoutingPolicyMaintainer(tester.controller(), Duration.ofDays(1)); - var dispatcher = new NameServiceDispatcher(tester.controller(), Duration.ofSeconds(Integer.MAX_VALUE)); - - var zone = ZoneId.from("prod", "us-west-1"); - tester.zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(zone)); - tester.configServer().putLoadBalancers(zone, List.of(new LoadBalancer("lb1", - SystemApplication.configServer.id(), - ClusterSpec.Id.from("config"), - Optional.of(HostName.of("lb1.example.com")), - LoadBalancer.State.active, - Optional.of("dns-zone-1")))); - - // Record is created - updater.run(); - dispatcher.run(); - Set<Record> records = tester.nameService().records(); - assertEquals(1, records.size()); - Record record = records.iterator().next(); - assertSame(Record.Type.CNAME, record.type()); - assertEquals("cfg.prod.us-west-1.test.vip", record.name().asString()); - assertEquals("lb1.example.com.", record.data().asString()); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ApplicationRequestToDiscFilterRequestWrapper.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ApplicationRequestToDiscFilterRequestWrapper.java index 5f580b6f6b3..52fd7393c4d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ApplicationRequestToDiscFilterRequestWrapper.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ApplicationRequestToDiscFilterRequestWrapper.java @@ -62,12 +62,6 @@ public class ApplicationRequestToDiscFilterRequestWrapper extends DiscFilterRequ } @Override - @Deprecated - public void setUri(URI uri) { - throw new UnsupportedOperationException(); - } - - @Override public String getParameter(String name) { throw new UnsupportedOperationException(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 6bfbb044944..d9f0f010104 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -305,7 +305,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/myuser/job/dev-us-east-1/diff/1", GET).userIdentity(HOSTED_VESPA_OPERATOR), (response) -> assertTrue(response.getBodyAsString(), - response.getBodyAsString().contains("--- search-definitions/test.sd\n" + + response.getBodyAsString().contains("--- schemas/test.sd\n" + "@@ -1,0 +1,1 @@\n" + "+ search test { }\n")), 200); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 7110cbbd738..be28d88abaa 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -85,9 +85,6 @@ "name": "RetriggerMaintainer" }, { - "name": "SystemRoutingPolicyMaintainer" - }, - { "name": "SystemUpgrader" }, { diff --git a/controller-server/src/test/resources/application-packages/changed-deployment-xml.zip b/controller-server/src/test/resources/application-packages/changed-deployment-xml.zip Binary files differindex e4ec61c50ab..e6482904b22 100644 --- a/controller-server/src/test/resources/application-packages/changed-deployment-xml.zip +++ b/controller-server/src/test/resources/application-packages/changed-deployment-xml.zip diff --git a/controller-server/src/test/resources/application-packages/changed-services-xml.zip b/controller-server/src/test/resources/application-packages/changed-services-xml.zip Binary files differindex daaa1bd9e3c..e11b1ef162e 100644 --- a/controller-server/src/test/resources/application-packages/changed-services-xml.zip +++ b/controller-server/src/test/resources/application-packages/changed-services-xml.zip diff --git a/controller-server/src/test/resources/application-packages/include-absolute.zip b/controller-server/src/test/resources/application-packages/include-absolute.zip Binary files differindex 3b30cd8265a..49c99ff5da9 100644 --- a/controller-server/src/test/resources/application-packages/include-absolute.zip +++ b/controller-server/src/test/resources/application-packages/include-absolute.zip diff --git a/controller-server/src/test/resources/application-packages/include-parent.zip b/controller-server/src/test/resources/application-packages/include-parent.zip Binary files differindex 18c1b0f5e37..8702b512c98 100644 --- a/controller-server/src/test/resources/application-packages/include-parent.zip +++ b/controller-server/src/test/resources/application-packages/include-parent.zip diff --git a/controller-server/src/test/resources/application-packages/original.zip b/controller-server/src/test/resources/application-packages/original.zip Binary files differindex 3963527a6cd..cabac1999c3 100644 --- a/controller-server/src/test/resources/application-packages/original.zip +++ b/controller-server/src/test/resources/application-packages/original.zip diff --git a/controller-server/src/test/resources/application-packages/similar-deployment-xml.zip b/controller-server/src/test/resources/application-packages/similar-deployment-xml.zip Binary files differindex 4075ee08ce3..67c38c344c0 100644 --- a/controller-server/src/test/resources/application-packages/similar-deployment-xml.zip +++ b/controller-server/src/test/resources/application-packages/similar-deployment-xml.zip diff --git a/defaults/pom.xml b/defaults/pom.xml index 86872b6d700..ffafc683fd6 100644 --- a/defaults/pom.xml +++ b/defaults/pom.xml @@ -7,13 +7,13 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <groupId>com.yahoo.vespa</groupId> <artifactId>defaults</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>defaults</name> <description> Library with the defaults for Vespa. diff --git a/dist/vespa.spec b/dist/vespa.spec index 54fe37aad2d..d278204be2e 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -226,9 +226,9 @@ BuildRequires: zlib-devel BuildRequires: libicu-devel %endif %if 0%{?el7} && 0%{?amzn2} -BuildRequires: java-11-amazon-corretto +BuildRequires: java-17-amazon-corretto %else -BuildRequires: java-11-openjdk-devel +BuildRequires: java-17-openjdk-devel %endif BuildRequires: rpm-build BuildRequires: make @@ -365,9 +365,9 @@ Vespa - The open big data serving engine Summary: Vespa - The open big data serving engine - base %if 0%{?el7} && 0%{?amzn2} -Requires: java-11-amazon-corretto +Requires: java-17-amazon-corretto %else -Requires: java-11-openjdk-devel +Requires: java-17-openjdk-devel %endif Requires: perl Requires: perl-Getopt-Long @@ -597,7 +597,7 @@ source %{_rhgit227_enable} || true %if 0%{?_java_home:1} export JAVA_HOME=%{?_java_home} %else -export JAVA_HOME=/usr/lib/jvm/java-11-openjdk +export JAVA_HOME=/usr/lib/jvm/java-17-openjdk %endif export PATH="$JAVA_HOME/bin:$PATH" export FACTORY_VESPA_VERSION=%{version} @@ -648,7 +648,7 @@ cp %{buildroot}/%{_prefix}/etc/systemd/system/vespa.service %{buildroot}/usr/lib cp %{buildroot}/%{_prefix}/etc/systemd/system/vespa-configserver.service %{buildroot}/usr/lib/systemd/system %endif -ln -s /usr/lib/jvm/jre-11-openjdk %{buildroot}/%{_prefix}/jdk +ln -s /usr/lib/jvm/jre-17-openjdk %{buildroot}/%{_prefix}/jdk %clean rm -rf $RPM_BUILD_ROOT @@ -757,7 +757,6 @@ fi %{_prefix}/lib/jars/document.jar %{_prefix}/lib/jars/filedistribution-jar-with-dependencies.jar %{_prefix}/lib/jars/http-client-jar-with-dependencies.jar -%{_prefix}/lib/jars/jdisc_jetty.jar %{_prefix}/lib/jars/logserver-jar-with-dependencies.jar %{_prefix}/lib/jars/metrics-proxy-jar-with-dependencies.jar %{_prefix}/lib/jars/node-repository-jar-with-dependencies.jar @@ -860,7 +859,6 @@ fi %{_prefix}/bin/vespa %{_prefix}/bin/vespa-feed-client %{_prefix}/conf/vespa-feed-client/logging.properties -%{_prefix}/lib/jars/vespa-http-client-jar-with-dependencies.jar %{_prefix}/lib/jars/vespa-feed-client-cli-jar-with-dependencies.jar %docdir /usr/share/man /usr/share/man @@ -897,8 +895,6 @@ fi %dir %{_prefix} %dir %{_prefix}/lib %dir %{_prefix}/lib/jars -%{_prefix}/lib/jars/asm-*.jar -%{_prefix}/lib/jars/aopalliance-repackaged-*.jar %{_prefix}/lib/jars/application-model-jar-with-dependencies.jar %{_prefix}/lib/jars/bcpkix-jdk15on-*.jar %{_prefix}/lib/jars/bcprov-jdk15on-*.jar @@ -911,28 +907,20 @@ fi %{_prefix}/lib/jars/container-disc-jar-with-dependencies.jar %{_prefix}/lib/jars/container-search-and-docproc-jar-with-dependencies.jar %{_prefix}/lib/jars/container-search-gui-jar-with-dependencies.jar +%{_prefix}/lib/jars/container-spifly.jar %{_prefix}/lib/jars/docprocs-jar-with-dependencies.jar %{_prefix}/lib/jars/flags-jar-with-dependencies.jar -%{_prefix}/lib/jars/hk2-*.jar %{_prefix}/lib/jars/hosted-zone-api-jar-with-dependencies.jar %{_prefix}/lib/jars/jackson-*.jar -%{_prefix}/lib/jars/javassist-*.jar %{_prefix}/lib/jars/javax.*.jar %{_prefix}/lib/jars/jdisc-cloud-aws-jar-with-dependencies.jar %{_prefix}/lib/jars/jdisc_core-jar-with-dependencies.jar %{_prefix}/lib/jars/jdisc-security-filters-jar-with-dependencies.jar -%{_prefix}/lib/jars/jersey-*.jar %{_prefix}/lib/jars/linguistics-components-jar-with-dependencies.jar -%{_prefix}/lib/jars/alpn-*.jar -%{_prefix}/lib/jars/http2-*.jar -%{_prefix}/lib/jars/jetty-*.jar %{_prefix}/lib/jars/model-evaluation-jar-with-dependencies.jar %{_prefix}/lib/jars/model-integration-jar-with-dependencies.jar -%{_prefix}/lib/jars/org.apache.aries.spifly.dynamic.bundle-*.jar -%{_prefix}/lib/jars/osgi-resource-locator-*.jar %{_prefix}/lib/jars/security-utils.jar %{_prefix}/lib/jars/standalone-container-jar-with-dependencies.jar -%{_prefix}/lib/jars/validation-api-*.jar %{_prefix}/lib/jars/vespa-athenz-jar-with-dependencies.jar %{_prefix}/lib/jars/vespaclient-container-plugin-jar-with-dependencies.jar %{_prefix}/lib/jars/vespajlib.jar diff --git a/docproc/abi-spec.json b/docproc/abi-spec.json index 5ad2c6d454d..84a31bbbfa2 100644 --- a/docproc/abi-spec.json +++ b/docproc/abi-spec.json @@ -80,8 +80,6 @@ "methods": [ "public void <init>(com.yahoo.docproc.DocumentProcessor)", "public void <init>(com.yahoo.docproc.DocumentProcessor, com.yahoo.jdisc.Metric)", - "public void <init>(com.yahoo.docproc.DocumentProcessor, com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric)", - "public void <init>(com.yahoo.docproc.DocumentProcessor, java.lang.String, com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric)", "public void <init>(com.yahoo.docproc.DocumentProcessor, java.lang.String, com.yahoo.jdisc.Metric)", "public java.lang.Object clone()", "public com.yahoo.docproc.DocumentProcessor getDocumentProcessor()", @@ -102,12 +100,9 @@ "public void <init>()", "public void <init>(java.lang.String)", "public void <init>(com.yahoo.jdisc.Metric)", - "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric)", - "public void <init>(java.lang.String, com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric)", "public void <init>(java.lang.String, com.yahoo.jdisc.Metric)", "public void <init>(com.yahoo.docproc.CallStack)", "public void <init>(java.lang.String, java.util.Collection, com.yahoo.jdisc.Metric)", - "public void <init>(java.lang.String, java.util.Collection, com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric)", "public java.lang.String getName()", "public void setName(java.lang.String)", "public com.yahoo.docproc.CallStack addNext(com.yahoo.docproc.Call)", @@ -133,77 +128,10 @@ "public java.util.ListIterator iterator()", "public int size()", "public java.lang.String toString()", - "public com.yahoo.statistics.Statistics getStatistics()", "public com.yahoo.jdisc.Metric getMetric()" ], "fields": [] }, - "com.yahoo.docproc.DocprocExecutor": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String, com.yahoo.docproc.CallStack)", - "public void <init>(com.yahoo.docproc.DocprocExecutor, com.yahoo.docproc.CallStack)", - "public com.yahoo.docproc.CallStack getCallStack()", - "public java.lang.String getName()", - "public com.yahoo.docproc.DocumentProcessor$Progress process(com.yahoo.docproc.Processing)", - "public com.yahoo.docproc.DocumentProcessor$Progress processUntilDone(com.yahoo.docproc.Processing)" - ], - "fields": [] - }, - "com.yahoo.docproc.DocprocService": { - "superClass": "com.yahoo.component.AbstractComponent", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.component.ComponentId)", - "public void <init>(com.yahoo.component.ComponentId, com.yahoo.docproc.CallStack, com.yahoo.document.DocumentTypeManager, int)", - "public void <init>(com.yahoo.component.ComponentId, com.yahoo.docproc.CallStack, com.yahoo.document.DocumentTypeManager)", - "public void <init>(java.lang.String)", - "public void deconstruct()", - "public com.yahoo.document.DocumentTypeManager getDocumentTypeManager()", - "public void setDocumentTypeManager(com.yahoo.document.DocumentTypeManager)", - "public int getQueueSize()", - "public com.yahoo.docproc.DocprocExecutor getExecutor()", - "public java.util.concurrent.ThreadPoolExecutor getThreadPoolExecutor()", - "public void setInService(boolean)", - "public boolean isInService()", - "public boolean isAcceptingNewProcessings()", - "public void setAcceptingNewProcessings(boolean)", - "public java.lang.String getName()", - "public com.yahoo.docproc.CallStack getCallStack()", - "public void setCallStack(com.yahoo.docproc.CallStack)", - "public void process(com.yahoo.docproc.Processing, com.yahoo.docproc.ProcessingEndpoint)", - "public void process(com.yahoo.docproc.Processing)", - "public void process(com.yahoo.document.DocumentOperation, com.yahoo.docproc.ProcessingEndpoint)", - "public void process(com.yahoo.document.DocumentOperation)", - "public void processDocumentOperations(java.util.List, com.yahoo.docproc.ProcessingEndpoint)", - "public void processDocumentOperations(java.util.List)", - "public boolean doWork()", - "public boolean doWorkBlocking()" - ], - "fields": [ - "public static com.yahoo.docproc.proxy.SchemaMap schemaMap" - ] - }, - "com.yahoo.docproc.DocumentOperationWrapper": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract com.yahoo.document.DocumentOperation getWrappedDocumentOperation()" - ], - "fields": [] - }, "com.yahoo.docproc.DocumentProcessor$LaterProgress": { "superClass": "com.yahoo.docproc.DocumentProcessor$Progress", "interfaces": [], @@ -258,33 +186,20 @@ ], "fields": [] }, - "com.yahoo.docproc.HandledProcessingException": { - "superClass": "java.lang.RuntimeException", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String)" - ], - "fields": [] - }, "com.yahoo.docproc.Processing": { - "superClass": "java.lang.Object", + "superClass": "com.yahoo.docproc.impl.ProcessingAccess", "interfaces": [], "attributes": [ - "public" + "public", + "final" ], "methods": [ "public void <init>()", "public static com.yahoo.docproc.Processing of(com.yahoo.document.DocumentOperation)", "public void <init>(java.lang.String, com.yahoo.document.DocumentOperation, com.yahoo.docproc.CallStack)", "public static com.yahoo.docproc.Processing createProcessingFromDocumentOperations(java.lang.String, java.util.List, com.yahoo.docproc.CallStack)", - "public com.yahoo.component.provider.ComponentRegistry getDocprocServiceRegistry()", - "public void setDocprocServiceRegistry(com.yahoo.component.provider.ComponentRegistry)", "public java.lang.String getServiceName()", "public void setServiceName(java.lang.String)", - "public com.yahoo.docproc.DocprocService getService()", "public java.lang.Object getVariable(java.lang.String)", "public java.util.Iterator getVariableAndNameIterator()", "public void clearVariables()", @@ -298,20 +213,6 @@ ], "fields": [] }, - "com.yahoo.docproc.ProcessingEndpoint": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void processingDone(com.yahoo.docproc.Processing)", - "public abstract void processingFailed(com.yahoo.docproc.Processing, java.lang.Exception)" - ], - "fields": [] - }, "com.yahoo.docproc.SimpleDocumentProcessor": { "superClass": "com.yahoo.docproc.DocumentProcessor", "interfaces": [], @@ -326,40 +227,5 @@ "public final com.yahoo.docproc.DocumentProcessor$Progress process(com.yahoo.docproc.Processing)" ], "fields": [] - }, - "com.yahoo.docproc.TransientFailureException": { - "superClass": "java.lang.RuntimeException", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.docproc.util.JoinerDocumentProcessor": { - "superClass": "com.yahoo.docproc.DocumentProcessor", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.config.docproc.SplitterJoinerDocumentProcessorConfig, com.yahoo.document.config.DocumentmanagerConfig)", - "public com.yahoo.docproc.DocumentProcessor$Progress process(com.yahoo.docproc.Processing)" - ], - "fields": [] - }, - "com.yahoo.docproc.util.SplitterDocumentProcessor": { - "superClass": "com.yahoo.docproc.DocumentProcessor", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.config.docproc.SplitterJoinerDocumentProcessorConfig, com.yahoo.document.config.DocumentmanagerConfig)", - "public com.yahoo.docproc.DocumentProcessor$Progress process(com.yahoo.docproc.Processing)" - ], - "fields": [] } }
\ No newline at end of file diff --git a/docproc/pom.xml b/docproc/pom.xml index f3202973abc..e7d8d6e3a3e 100644 --- a/docproc/pom.xml +++ b/docproc/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>docproc</artifactId> <packaging>jar</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <!-- PROVIDED scope --> <dependency> @@ -78,12 +78,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>statistics</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>vespajlib</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/docproc/src/main/java/com/yahoo/docproc/Call.java b/docproc/src/main/java/com/yahoo/docproc/Call.java index bb277e5ebfe..440935ef494 100644 --- a/docproc/src/main/java/com/yahoo/docproc/Call.java +++ b/docproc/src/main/java/com/yahoo/docproc/Call.java @@ -2,6 +2,7 @@ package com.yahoo.docproc; import com.yahoo.component.ComponentId; +import com.yahoo.docproc.impl.DocumentOperationWrapper; import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.docproc.proxy.ProxyDocument; import com.yahoo.docproc.proxy.ProxyDocumentUpdate; @@ -11,7 +12,6 @@ import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentUpdate; import com.yahoo.jdisc.Metric; import com.yahoo.concurrent.SystemTimer; -import com.yahoo.statistics.Statistics; import java.util.List; @@ -40,14 +40,6 @@ public class Call implements Cloneable { public Call(DocumentProcessor processor, Metric metric) { this(processor, "", metric); } - @Deprecated - public Call(DocumentProcessor processor, Statistics manager, Metric metric) { - this(processor, "", metric); - } - @Deprecated - public Call(DocumentProcessor processor, String chainName, Statistics manager, Metric metric) { - this(processor, chainName, metric); - } public Call(DocumentProcessor processor, String chainName, Metric metric) { this.processor = processor; @@ -64,8 +56,7 @@ public class Call implements Cloneable { @Override public Object clone() { try { - Call clone = (Call) super.clone(); - return clone; + return super.clone(); } catch (CloneNotSupportedException e) { throw new RuntimeException("Will not happen"); } @@ -132,7 +123,6 @@ public class Call implements Cloneable { } - @SuppressWarnings("removal") // TODO Vespa 8: remove private void unwrapSchemaMapping(Processing processing) { final List<DocumentOperation> documentOperations = processing.getDocumentOperations(); diff --git a/docproc/src/main/java/com/yahoo/docproc/CallStack.java b/docproc/src/main/java/com/yahoo/docproc/CallStack.java index 9f59d9e869b..9c7251ee54e 100644 --- a/docproc/src/main/java/com/yahoo/docproc/CallStack.java +++ b/docproc/src/main/java/com/yahoo/docproc/CallStack.java @@ -4,7 +4,6 @@ package com.yahoo.docproc; import com.yahoo.component.ComponentId; import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.jdisc.Metric; -import com.yahoo.statistics.Statistics; import java.util.Collection; import java.util.Iterator; @@ -47,14 +46,6 @@ public class CallStack { this.name = null; this.metric = metric; } - @Deprecated - public CallStack(Statistics statistics, Metric metric) { - this(metric); - } - @Deprecated - public CallStack(String name, Statistics manager, Metric metric) { - this(name, metric); - } /** Creates an empty stack with a name */ public CallStack(String name, Metric metric) { this.name = name; @@ -87,10 +78,6 @@ public class CallStack { addLast(docproc); } } - @Deprecated - public CallStack(String name, Collection<DocumentProcessor> docprocs, Statistics manager, Metric metric) { - this(name, docprocs, metric); - } /** Returns the name of this stack, or null if it is not named */ public String getName() { @@ -384,11 +371,6 @@ public class CallStack { return b.toString(); } - @Deprecated - public Statistics getStatistics() { - return null; - } - public Metric getMetric() { return metric; } diff --git a/docproc/src/main/java/com/yahoo/docproc/DocumentProcessor.java b/docproc/src/main/java/com/yahoo/docproc/DocumentProcessor.java index ae7c1358a4f..7b8f07373a5 100644 --- a/docproc/src/main/java/com/yahoo/docproc/DocumentProcessor.java +++ b/docproc/src/main/java/com/yahoo/docproc/DocumentProcessor.java @@ -3,6 +3,7 @@ package com.yahoo.docproc; import com.yahoo.collections.Pair; import com.yahoo.component.chain.ChainedComponent; +import com.yahoo.docproc.impl.DocprocService; import java.util.HashMap; import java.util.Map; @@ -42,7 +43,6 @@ import java.util.logging.Logger; */ public abstract class DocumentProcessor extends ChainedComponent { - @SuppressWarnings("removal") // TODO Vespa 8: remove static Logger log = Logger.getLogger(DocprocService.class.getName()); /** Schema map for doctype-fieldnames */ diff --git a/docproc/src/main/java/com/yahoo/docproc/Processing.java b/docproc/src/main/java/com/yahoo/docproc/Processing.java index 834d63c5a86..24eb4f42762 100644 --- a/docproc/src/main/java/com/yahoo/docproc/Processing.java +++ b/docproc/src/main/java/com/yahoo/docproc/Processing.java @@ -1,7 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.docproc; -import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.docproc.impl.ProcessingAccess; +import com.yahoo.docproc.impl.ProcessingEndpoint; import com.yahoo.document.DocumentOperation; import java.util.ArrayList; @@ -18,7 +19,7 @@ import java.util.Map; * * @author bratseth */ -public class Processing { +public final class Processing extends ProcessingAccess { /** The name of the service which owns this processing. Null is the same as "default". */ private String service = null; @@ -40,13 +41,8 @@ public class Processing { private Map<String, Object> context = null; /** The endpoint of this processing. */ - @SuppressWarnings("removal") // TODO Vespa 8: remove private ProcessingEndpoint endpoint = null; - /** The registry of docproc services. */ - @SuppressWarnings("removal") // TODO Vespa 8: remove - private ComponentRegistry<DocprocService> docprocServiceRegistry = null; - private boolean operationsGotten = false; /** @@ -79,8 +75,7 @@ public class Processing { * @param callStack the document processors to call in this processing. * @param endp the endpoint of this processing */ - @SuppressWarnings("removal") // TODO Vespa 8: remove - Processing(String service, DocumentOperation documentOperation, CallStack callStack, ProcessingEndpoint endp) { + private Processing(String service, DocumentOperation documentOperation, CallStack callStack, ProcessingEndpoint endp) { this.service = service; this.documentOperations = new ArrayList<>(1); documentOperations.add(documentOperation); @@ -102,8 +97,7 @@ public class Processing { this(service, documentOperation, callStack, null); } - // TODO Vespa 8: remove "removal" - @SuppressWarnings({"unused", "removal"}) + @SuppressWarnings({"unused"}) private Processing(String service, List<DocumentOperation> documentOpsAndUpdates, CallStack callStack, ProcessingEndpoint endp, boolean unused) { this.service = service; this.documentOperations = new ArrayList<>(documentOpsAndUpdates.size()); @@ -112,11 +106,6 @@ public class Processing { this.endpoint = endp; } - @SuppressWarnings("removal") // TODO Vespa 8: remove - static Processing createProcessingFromDocumentOperations(String service, List<DocumentOperation> documentOpsAndUpdates, CallStack callStack, ProcessingEndpoint endp) { - return new Processing(service, documentOpsAndUpdates, callStack, endp, false); - } - /** * Creates a Processing from a list of operations. * @@ -133,21 +122,6 @@ public class Processing { return new Processing(service, documentsAndUpdates, callStack, null, false); } - @Deprecated(forRemoval = true, since="7") - @SuppressWarnings("removal") // TODO Vespa 8: remove - public ComponentRegistry<DocprocService> getDocprocServiceRegistry() { - return docprocServiceRegistry; - } - - /** - * @deprecated This method will be removed without replacement in Vespa 8. - */ - @Deprecated(forRemoval = true, since="7") - @SuppressWarnings("removal") // TODO Vespa 8: remove - public void setDocprocServiceRegistry(ComponentRegistry<DocprocService> docprocServiceRegistry) { - this.docprocServiceRegistry = docprocServiceRegistry; - } - /** Returns the name of the service processing this. This will never return null */ public String getServiceName() { if (service == null) return "default"; @@ -159,23 +133,6 @@ public class Processing { this.service = service; } - /** - * Convenience method for looking up and returning the service processing this. This might return null - * if #getServiceName returns a name that is not registered in {@link com.yahoo.docproc.DocprocService}. - * - * @return the service processing this, or null if unknown. - * @deprecated Formerly used to retrieve the {@link com.yahoo.document.DocumentTypeManager}, - * which can now be directly injected via your component constructor. - */ - @Deprecated(forRemoval = true, since="7") - @SuppressWarnings("removal") // TODO Vespa 8: remove - public DocprocService getService() { - if (docprocServiceRegistry != null) { - return docprocServiceRegistry.getComponent(getServiceName()); - } - return null; - } - /** Returns a context variable, or null if it is not set */ public Object getVariable(String name) { if (context == null) return null; @@ -214,23 +171,13 @@ public class Processing { return context != null && context.containsKey(name); } - /** - * Returns the ProcessingEndpoint that is called when this Processing is complete, if any. - * - * @return the ProcessingEndpoint, or null - */ - @SuppressWarnings("removal") // TODO Vespa 8: remove - ProcessingEndpoint getEndpoint() { + @Override + protected ProcessingEndpoint getEndpoint() { return endpoint; } - /** - * Sets the ProcessingEndpoint to be called when this Processing is complete. - * - * @param endpoint the ProcessingEndpoint to use - */ - @SuppressWarnings("removal") // TODO Vespa 8: remove - void setEndpoint(ProcessingEndpoint endpoint) { + @Override + protected void setEndpoint(ProcessingEndpoint endpoint) { this.endpoint = endpoint; } @@ -256,17 +203,13 @@ public class Processing { return callStack; } - /** - * Package-private method to set the callstack of this processing. Only to be used - * by DocprocService.process(Processing). - * - * @param callStack the callstack to set - */ - void setCallStack(CallStack callStack) { + @Override + protected void setCallStack(CallStack callStack) { this.callStack = callStack; } - List<DocumentOperation> getOnceOperationsToBeProcessed() { + @Override + protected List<DocumentOperation> getOnceOperationsToBeProcessed() { if (operationsGotten) return Collections.emptyList(); diff --git a/docproc/src/main/java/com/yahoo/docproc/DocprocExecutor.java b/docproc/src/main/java/com/yahoo/docproc/impl/DocprocExecutor.java index 0e7a217efbe..cfd43099cc8 100644 --- a/docproc/src/main/java/com/yahoo/docproc/DocprocExecutor.java +++ b/docproc/src/main/java/com/yahoo/docproc/impl/DocprocExecutor.java @@ -1,6 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.docproc; +package com.yahoo.docproc.impl; +import com.yahoo.docproc.Call; +import com.yahoo.docproc.CallStack; +import com.yahoo.docproc.DocumentProcessor; +import com.yahoo.docproc.Processing; import com.yahoo.document.DocumentOperation; import com.yahoo.document.DocumentPut; import com.yahoo.document.json.JsonWriter; @@ -22,9 +26,7 @@ import static java.util.stream.Collectors.groupingBy; * An executor executed incoming processings on its CallStack * * @author Einar M R Rosenvinge - * @deprecated Will be removed in Vespa 8. Only for internal use. */ -@Deprecated(forRemoval = true, since = "7") public class DocprocExecutor { private final static String METRIC_NAME_DOCUMENTS_PROCESSED = "documents_processed"; @@ -77,7 +79,7 @@ public class DocprocExecutor { } private void incrementNumDocsProcessed(Processing processing) { - List<DocumentOperation> operations = processing.getOnceOperationsToBeProcessed(); + List<DocumentOperation> operations = ((ProcessingAccess)processing).getOnceOperationsToBeProcessed(); if ( ! operations.isEmpty()) { metric.add(docCounterName, operations.size(), null); operations.stream() @@ -97,7 +99,7 @@ public class DocprocExecutor { public DocumentProcessor.Progress process(Processing processing) { processing.setServiceName(getName()); if (processing.callStack() == null) { - processing.setCallStack(new CallStack(getCallStack())); + ((ProcessingAccess)processing).setCallStack(new CallStack(getCallStack())); } DocumentProcessor.Progress progress = DocumentProcessor.Progress.DONE; diff --git a/docproc/src/main/java/com/yahoo/docproc/DocprocService.java b/docproc/src/main/java/com/yahoo/docproc/impl/DocprocService.java index c6956f44f52..62a9a31f47a 100644 --- a/docproc/src/main/java/com/yahoo/docproc/DocprocService.java +++ b/docproc/src/main/java/com/yahoo/docproc/impl/DocprocService.java @@ -1,9 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.docproc; +package com.yahoo.docproc.impl; import com.yahoo.component.AbstractComponent; import com.yahoo.component.ComponentId; import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.docproc.CallStack; +import com.yahoo.docproc.DocumentProcessor; +import com.yahoo.docproc.Processing; import com.yahoo.docproc.proxy.SchemaMap; import com.yahoo.document.DocumentOperation; import com.yahoo.document.DocumentTypeManager; @@ -29,10 +32,7 @@ import java.util.logging.Logger; * <p>This class is thread safe.</p> * * @author bratseth - * @deprecated Will be removed in Vespa 8. Only for internal use. */ -@SuppressWarnings("removal") // TODO Vespa 8: remove -@Deprecated(forRemoval = true, since = "7") public class DocprocService extends AbstractComponent { private static final Logger log = Logger.getLogger(DocprocService.class.getName()); @@ -77,11 +77,6 @@ public class DocprocService extends AbstractComponent { setInService(true); } - @Deprecated - public DocprocService(ComponentId id, CallStack stack, DocumentTypeManager mgr) { - this(id, stack, mgr, Runtime.getRuntime().availableProcessors()); - } - /** * Creates a service with a name with an unbounded input queue. If the given name is null or the empty string, * it will become the name "default". @@ -194,8 +189,8 @@ public class DocprocService extends AbstractComponent { */ public void process(Processing processing, ProcessingEndpoint endp) { processing.setServiceName(getName()); - processing.setCallStack(new CallStack(getCallStack())); - processing.setEndpoint(endp); + ((ProcessingAccess)processing).setCallStack(new CallStack(getCallStack())); + ((ProcessingAccess)processing).setEndpoint(endp); addProcessing(processing); } @@ -218,7 +213,9 @@ public class DocprocService extends AbstractComponent { * @throws IllegalStateException if this DocprocService is not accepting new incoming processings */ public void process(DocumentOperation documentOperation, ProcessingEndpoint endp) { - addProcessing(new Processing(getName(), documentOperation, new CallStack(getCallStack()), endp)); + Processing processing = new Processing(getName(), documentOperation, new CallStack(getCallStack())); + ((ProcessingAccess)processing).setEndpoint(endp); + addProcessing(processing); } /** @@ -241,7 +238,10 @@ public class DocprocService extends AbstractComponent { * @throws IllegalStateException if this DocprocService is not accepting new incoming processings */ public void processDocumentOperations(List<DocumentOperation> documentOperations, ProcessingEndpoint endp) { - addProcessing(Processing.createProcessingFromDocumentOperations(getName(), documentOperations, new CallStack(getCallStack()), endp)); + Processing processing = Processing.createProcessingFromDocumentOperations(getName(), documentOperations, new CallStack(getCallStack())); + ((ProcessingAccess)processing).setEndpoint(endp); + addProcessing(processing); + } /** @@ -359,7 +359,7 @@ public class DocprocService extends AbstractComponent { if (DocumentProcessor.Progress.DONE.equals(progress)) { //notify endpoint - ProcessingEndpoint recv = processing.getEndpoint(); + ProcessingEndpoint recv = ((ProcessingAccess)processing).getEndpoint(); if (recv != null) { recv.processingDone(processing); } @@ -392,7 +392,7 @@ public class DocprocService extends AbstractComponent { } //notify endpoint - ProcessingEndpoint recv = processing.getEndpoint(); + ProcessingEndpoint recv = ((ProcessingAccess)processing).getEndpoint(); if (recv != null) { recv.processingFailed(processing, e); } diff --git a/docproc/src/main/java/com/yahoo/docproc/DocumentOperationWrapper.java b/docproc/src/main/java/com/yahoo/docproc/impl/DocumentOperationWrapper.java index 4530af29600..32ccc11c1ab 100644 --- a/docproc/src/main/java/com/yahoo/docproc/DocumentOperationWrapper.java +++ b/docproc/src/main/java/com/yahoo/docproc/impl/DocumentOperationWrapper.java @@ -1,13 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.docproc; +package com.yahoo.docproc.impl; import com.yahoo.document.DocumentOperation; /** * @author Einar M R Rosenvinge - * @deprecated Will be removed in Vespa 8. Only for internal use. */ -@Deprecated(forRemoval = true, since = "7") public interface DocumentOperationWrapper { DocumentOperation getWrappedDocumentOperation(); diff --git a/docproc/src/main/java/com/yahoo/docproc/HandledProcessingException.java b/docproc/src/main/java/com/yahoo/docproc/impl/HandledProcessingException.java index 2872d53f558..7bf706a0b56 100644 --- a/docproc/src/main/java/com/yahoo/docproc/HandledProcessingException.java +++ b/docproc/src/main/java/com/yahoo/docproc/impl/HandledProcessingException.java @@ -1,14 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.docproc; +package com.yahoo.docproc.impl; /** * Exception generated by known bad input in a docproc. Will cause only message to be logged, * not stacktrace. * * @author Mathias Mølster Lidal - * @deprecated Will be removed in Vespa 8. Only for internal use. */ -@Deprecated(forRemoval = true, since = "7") public class HandledProcessingException extends RuntimeException { public HandledProcessingException(String message) { diff --git a/docproc/src/main/java/com/yahoo/docproc/impl/ProcessingAccess.java b/docproc/src/main/java/com/yahoo/docproc/impl/ProcessingAccess.java new file mode 100644 index 00000000000..a18257f61ec --- /dev/null +++ b/docproc/src/main/java/com/yahoo/docproc/impl/ProcessingAccess.java @@ -0,0 +1,31 @@ +package com.yahoo.docproc.impl; + +import com.yahoo.docproc.CallStack; +import com.yahoo.document.DocumentOperation; + +import java.util.List; + +/** + * Bridge to access protected (originally package private) methods in {@link com.yahoo.docproc.Processing}. + * + * @author gjoranv + */ +public abstract class ProcessingAccess { + + protected ProcessingEndpoint getEndpoint() { + throw new UnsupportedOperationException("docproc.Processing must override this method!"); + } + + protected void setEndpoint(ProcessingEndpoint endpoint) { + throw new UnsupportedOperationException("docproc.Processing must override this method!"); + } + + protected void setCallStack(CallStack callStack) { + throw new UnsupportedOperationException("docproc.Processing must override this method!"); + } + + protected List<DocumentOperation> getOnceOperationsToBeProcessed() { + throw new UnsupportedOperationException("docproc.Processing must override this method!"); + } + +} diff --git a/docproc/src/main/java/com/yahoo/docproc/ProcessingEndpoint.java b/docproc/src/main/java/com/yahoo/docproc/impl/ProcessingEndpoint.java index 00b0bc7c7c7..e88c2ef444b 100644 --- a/docproc/src/main/java/com/yahoo/docproc/ProcessingEndpoint.java +++ b/docproc/src/main/java/com/yahoo/docproc/impl/ProcessingEndpoint.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.docproc; +package com.yahoo.docproc.impl; + +import com.yahoo.docproc.Processing; /** * @author Einar M R Rosenvinge - * @deprecated Will be removed in Vespa 8. Only for internal use. */ -@Deprecated(forRemoval = true, since = "7") public interface ProcessingEndpoint { void processingDone(Processing processing); diff --git a/docproc/src/main/java/com/yahoo/docproc/TransientFailureException.java b/docproc/src/main/java/com/yahoo/docproc/impl/TransientFailureException.java index a88158cc367..b470ba4b806 100644 --- a/docproc/src/main/java/com/yahoo/docproc/TransientFailureException.java +++ b/docproc/src/main/java/com/yahoo/docproc/impl/TransientFailureException.java @@ -1,14 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.docproc; +package com.yahoo.docproc.impl; /** * Exception to be thrown by a document processor on transient failures. * Caller is welcome to try the call again later. * * @author Einar M R Rosenvinge - * @deprecated Will be removed in Vespa 8. Only for internal use. */ -@Deprecated(forRemoval = true, since = "7") public class TransientFailureException extends RuntimeException { public TransientFailureException(String s) { diff --git a/container-core/src/main/java/com/yahoo/container/xml/bind/package-info.java b/docproc/src/main/java/com/yahoo/docproc/impl/package-info.java index e6850261f2c..f77ab5d9103 100644 --- a/container-core/src/main/java/com/yahoo/container/xml/bind/package-info.java +++ b/docproc/src/main/java/com/yahoo/docproc/impl/package-info.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage -package com.yahoo.container.xml.bind; +package com.yahoo.docproc.impl; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java index 3b31b0447a3..9eda0b12069 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java @@ -12,12 +12,11 @@ import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.core.document.ContainerDocumentConfig; import com.yahoo.docproc.AbstractConcreteDocumentFactory; import com.yahoo.docproc.CallStack; -import com.yahoo.docproc.DocprocService; +import com.yahoo.docproc.impl.DocprocService; import com.yahoo.docproc.DocumentProcessor; import com.yahoo.docproc.jdisc.messagebus.MbusRequestContext; import com.yahoo.docproc.proxy.SchemaMap; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Request; import com.yahoo.jdisc.handler.AbstractRequestHandler; @@ -43,7 +42,6 @@ import static com.yahoo.component.chain.model.ChainsModelBuilder.buildFromConfig public class DocumentProcessingHandler extends AbstractRequestHandler { private static final Logger log = Logger.getLogger(DocumentProcessingHandler.class.getName()); - @SuppressWarnings("removal") // TODO Vespa 8: remove private final ComponentRegistry<DocprocService> docprocServiceRegistry; private final ComponentRegistry<AbstractConcreteDocumentFactory> docFactoryRegistry; private final ChainRegistry<DocumentProcessor> chainRegistry = new ChainRegistry<>(); @@ -52,7 +50,6 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { private final ContainerDocumentConfig containerDocConfig; private final DocumentTypeManager documentTypeManager; - @SuppressWarnings("removal") // TODO Vespa 8: remove private DocumentProcessingHandler(ComponentRegistry<DocprocService> docprocServiceRegistry, ComponentRegistry<DocumentProcessor> documentProcessorComponentRegistry, ComponentRegistry<AbstractConcreteDocumentFactory> docFactoryRegistry, @@ -85,7 +82,6 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { return (maxThreads > 0) ? maxThreads : Runtime.getRuntime().availableProcessors(); } - @SuppressWarnings("removal") // TODO Vespa 8: remove DocumentProcessingHandler(ComponentRegistry<DocprocService> docprocServiceRegistry, ComponentRegistry<DocumentProcessor> documentProcessorComponentRegistry, ComponentRegistry<AbstractConcreteDocumentFactory> docFactoryRegistry, @@ -118,13 +114,11 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { } @Override - @SuppressWarnings("removal") // TODO Vespa 8: remove protected void destroy() { laterExecutor.shutdown(); docprocServiceRegistry.allComponents().forEach(docprocService -> docprocService.deconstruct()); } - @SuppressWarnings("removal") // TODO Vespa 8: remove public ComponentRegistry<DocprocService> getDocprocServiceRegistry() { return docprocServiceRegistry; } @@ -140,7 +134,6 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { } - @SuppressWarnings("removal") // TODO Vespa 8: remove private static CallStack convertToCallStack(Chain<DocumentProcessor> chain, Metric metric) { CallStack stack = new CallStack(chain.getId().stringValue(), metric); for (DocumentProcessor processor : chain.components()) { @@ -151,7 +144,6 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { } @Override - @SuppressWarnings("removal") // TODO Vespa 8: remove public ContentChannel handleRequest(Request request, ResponseHandler handler) { RequestContext requestContext; if (request instanceof MbusRequest) { diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java index e629953ccff..f7f8d0e6a10 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java @@ -4,10 +4,10 @@ package com.yahoo.docproc.jdisc; import com.yahoo.collections.Tuple2; import com.yahoo.docproc.Call; import com.yahoo.docproc.CallStack; -import com.yahoo.docproc.DocprocExecutor; -import com.yahoo.docproc.DocprocService; +import com.yahoo.docproc.impl.DocprocExecutor; +import com.yahoo.docproc.impl.DocprocService; import com.yahoo.docproc.DocumentProcessor; -import com.yahoo.docproc.HandledProcessingException; +import com.yahoo.docproc.impl.HandledProcessingException; import com.yahoo.docproc.Processing; import java.util.logging.Level; import com.yahoo.yolean.Exceptions; @@ -19,7 +19,6 @@ import java.util.Iterator; import java.util.List; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; -import java.util.logging.Level; import java.util.logging.Logger; /** @@ -34,11 +33,9 @@ public class DocumentProcessingTask implements Runnable { private final DocumentProcessingHandler docprocHandler; private final RequestContext requestContext; - @SuppressWarnings("removal") // TODO Vespa 8: remove private final DocprocService service; private final ThreadPoolExecutor executor; - @SuppressWarnings("removal") // TODO Vespa 8: remove public DocumentProcessingTask(RequestContext requestContext, DocumentProcessingHandler docprocHandler, DocprocService service, ThreadPoolExecutor executor) { this.requestContext = requestContext; @@ -67,7 +64,6 @@ public class DocumentProcessingTask implements Runnable { return; } - @SuppressWarnings("removal") // TODO Vespa 8: remove DocprocExecutor executor = service.getExecutor(); DocumentProcessor.Progress progress = process(executor); @@ -91,7 +87,6 @@ public class DocumentProcessingTask implements Runnable { * * @param executor the DocprocService to use for processing */ - @SuppressWarnings("removal") // TODO Vespa 8: remove private DocumentProcessor.Progress process(DocprocExecutor executor) { Iterator<Processing> iterator = processings.iterator(); List<Tuple2<DocumentProcessor.Progress, Processing>> later = new ArrayList<>(); @@ -188,7 +183,6 @@ public class DocumentProcessingTask implements Runnable { '}'; } - @SuppressWarnings("removal") // TODO Vespa 8: remove private static void logProcessingFailure(Processing processing, Exception exception) { //LOGGING ONLY: String errorMsg = processing + " failed at " + processing.callStack().getLastPopped(); diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java index 50d7acb4064..caaff318cdd 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java @@ -5,10 +5,10 @@ import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.concurrent.CopyOnWriteHashMap; import com.yahoo.container.core.document.ContainerDocumentConfig; import com.yahoo.docproc.AbstractConcreteDocumentFactory; -import com.yahoo.docproc.DocprocService; -import com.yahoo.docproc.HandledProcessingException; +import com.yahoo.docproc.impl.DocprocService; +import com.yahoo.docproc.impl.HandledProcessingException; import com.yahoo.docproc.Processing; -import com.yahoo.docproc.TransientFailureException; +import com.yahoo.docproc.impl.TransientFailureException; import com.yahoo.docproc.jdisc.RequestContext; import com.yahoo.document.DocumentOperation; import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; @@ -53,7 +53,6 @@ public class MbusRequestContext implements RequestContext, ResponseHandler { public final static String internalNoThrottledSource = "internalNoThrottledSource"; private final static String internalNoThrottledSourcePath = "/" + internalNoThrottledSource; - @SuppressWarnings("removal") // TODO Vespa 8: remove public MbusRequestContext(MbusRequest request, ResponseHandler responseHandler, ComponentRegistry<DocprocService> docprocServiceComponentRegistry, ComponentRegistry<AbstractConcreteDocumentFactory> docFactoryRegistry, @@ -61,7 +60,7 @@ public class MbusRequestContext implements RequestContext, ResponseHandler { this.request = request; this.requestMsg = (DocumentMessage)request.getMessage(); this.responseHandler = responseHandler; - this.processingFactory = new ProcessingFactory(docprocServiceComponentRegistry, docFactoryRegistry, + this.processingFactory = new ProcessingFactory(docFactoryRegistry, containerDocConfig, getServiceName()); this.messageFactory = newMessageFactory(requestMsg); } @@ -112,7 +111,6 @@ public class MbusRequestContext implements RequestContext, ResponseHandler { } @Override - @SuppressWarnings("removal") // TODO Vespa 8: remove public void processingFailed(Exception exception) { ErrorCode errorCode; if (exception instanceof TransientFailureException) { diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java index 1fe6d4cc86c..013f75c317c 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java @@ -6,7 +6,6 @@ import com.yahoo.document.DocumentOperation; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentUpdate; -import com.yahoo.documentapi.messagebus.loadtypes.LoadType; import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; @@ -21,34 +20,23 @@ import java.util.logging.Logger; /** * @author Einar M R Rosenvinge */ -@SuppressWarnings("removal") // TODO: Remove on Vespa 8 class MessageFactory { private final static Logger log = Logger.getLogger(MessageFactory.class.getName()); private final Message requestMsg; - private final LoadType loadType; // TODO: Remove on Vespa 8 - private final DocumentProtocol.Priority priority; // TODO: Remove on Vespa 8 - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public MessageFactory(DocumentMessage requestMsg) { this.requestMsg = requestMsg; - loadType = requestMsg.getLoadType(); - priority = requestMsg.getPriority(); // TODO: Remove on Vespa 8 } - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public DocumentMessage fromDocumentOperation(Processing processing, DocumentOperation documentOperation) { DocumentMessage message = newMessage(documentOperation); - message.setLoadType(loadType); - message.setPriority(priority); // TODO: Remove on Vespa 8 message.setRoute(requestMsg.getRoute()); message.setTimeReceivedNow(); message.setTimeRemaining(requestMsg.getTimeRemainingNow()); message.getTrace().setLevel(requestMsg.getTrace().getLevel()); log.log(Level.FINE, () -> "Created '" + message.getClass().getName() + "', route = '" + message.getRoute() + - "', priority = '" + message.getPriority().name() + // TODO: Remove on Vespa 8 - "', load type = '" + message.getLoadType() + "', trace level = '" + message.getTrace().getLevel() + "', time remaining = '" + message.getTimeRemaining() + "'."); return message; diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java index 33cd6647ede..52f112f9b43 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java @@ -8,7 +8,6 @@ import com.yahoo.component.ComponentId; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.core.document.ContainerDocumentConfig; import com.yahoo.docproc.AbstractConcreteDocumentFactory; -import com.yahoo.docproc.DocprocService; import com.yahoo.docproc.Processing; import com.yahoo.document.Document; import com.yahoo.document.DocumentOperation; @@ -27,18 +26,13 @@ import com.yahoo.messagebus.Message; class ProcessingFactory { private final static Logger log = Logger.getLogger(ProcessingFactory.class.getName()); - @SuppressWarnings("removal") // TODO Vespa 8: remove - private final ComponentRegistry<DocprocService> docprocServiceComponentRegistry; private final ComponentRegistry<AbstractConcreteDocumentFactory> docFactoryRegistry; private final ContainerDocumentConfig containerDocConfig; private final String serviceName; - @SuppressWarnings("removal") // TODO Vespa 8: remove - public ProcessingFactory(ComponentRegistry<DocprocService> docprocServiceComponentRegistry, - ComponentRegistry<AbstractConcreteDocumentFactory> docFactoryRegistry, + public ProcessingFactory(ComponentRegistry<AbstractConcreteDocumentFactory> docFactoryRegistry, ContainerDocumentConfig containerDocConfig, String serviceName) { - this.docprocServiceComponentRegistry = docprocServiceComponentRegistry; this.docFactoryRegistry = docFactoryRegistry; this.containerDocConfig = containerDocConfig; this.serviceName = serviceName; @@ -100,15 +94,11 @@ class ProcessingFactory { return null; } - @SuppressWarnings("removal") // TODO Vespa 8: remove private Processing createProcessing(DocumentOperation documentOperation, Message message) { Processing processing = new Processing(); processing.addDocumentOperation(documentOperation); processing.setServiceName(serviceName); - // TODO Vespa 8: Remove statement (registry will be removed from Processing) - processing.setDocprocServiceRegistry(docprocServiceComponentRegistry); - processing.setVariable("route", message.getRoute()); processing.setVariable("timeout", message.getTimeRemaining()); return processing; diff --git a/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocument.java b/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocument.java index d54ab012b3b..975d396a5cb 100644 --- a/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocument.java +++ b/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocument.java @@ -2,7 +2,7 @@ package com.yahoo.docproc.proxy; import com.yahoo.docproc.Accesses; -import com.yahoo.docproc.DocumentOperationWrapper; +import com.yahoo.docproc.impl.DocumentOperationWrapper; import com.yahoo.docproc.DocumentProcessor; import com.yahoo.document.DataType; import com.yahoo.document.Document; @@ -15,7 +15,6 @@ import com.yahoo.document.FieldPath; import com.yahoo.document.datatypes.FieldPathIteratorHandler; import com.yahoo.document.datatypes.FieldPathIteratorHandler.ModificationStatus; import com.yahoo.document.datatypes.FieldValue; -import com.yahoo.document.datatypes.Struct; import com.yahoo.document.serialization.DocumentReader; import com.yahoo.document.serialization.DocumentWriter; import com.yahoo.document.serialization.FieldReader; @@ -42,7 +41,6 @@ import java.util.Set; * * @author Vegard Havdal */ -@SuppressWarnings("removal") // TODO Vespa 8: remove public class ProxyDocument extends Document implements DocumentOperationWrapper { private final Map<String, String> fieldMap; @@ -232,18 +230,6 @@ public class ProxyDocument extends Document implements DocumentOperationWrapper } @Override - @SuppressWarnings("deprecation") - public Struct getHeader() { - return doc.getHeader(); - } - - @Override - @SuppressWarnings("deprecation") - public Struct getBody() { - return null; - } - - @Override public void assign(Object o) { doc.assign(o); } diff --git a/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocumentUpdate.java b/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocumentUpdate.java index 68af456829d..642ae216687 100644 --- a/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocumentUpdate.java +++ b/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocumentUpdate.java @@ -1,6 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.docproc.proxy; -import com.yahoo.docproc.DocumentOperationWrapper; +import com.yahoo.docproc.impl.DocumentOperationWrapper; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentOperation; @@ -18,7 +18,6 @@ import java.util.Map; * * @author vegardh */ -@SuppressWarnings("removal") // TODO Vespa 8: remove public class ProxyDocumentUpdate extends DocumentUpdate implements DocumentOperationWrapper { private DocumentUpdate docU; diff --git a/docproc/src/main/java/com/yahoo/docproc/proxy/SchemaMap.java b/docproc/src/main/java/com/yahoo/docproc/proxy/SchemaMap.java index ffe89e5a405..8a544dea221 100644 --- a/docproc/src/main/java/com/yahoo/docproc/proxy/SchemaMap.java +++ b/docproc/src/main/java/com/yahoo/docproc/proxy/SchemaMap.java @@ -16,7 +16,6 @@ import java.util.Map.Entry; * * @author vegardh */ -@SuppressWarnings("removal") // TODO Vespa 8: remove public class SchemaMap implements ConfigSubscriber.SingleSubscriber<SchemamappingConfig> { /** Map key. Doctype can be null, not the others. */ diff --git a/docproc/src/main/java/com/yahoo/docproc/util/JoinerDocumentProcessor.java b/docproc/src/main/java/com/yahoo/docproc/util/JoinerDocumentProcessor.java deleted file mode 100644 index 4e0a5b3eea6..00000000000 --- a/docproc/src/main/java/com/yahoo/docproc/util/JoinerDocumentProcessor.java +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.docproc.util; - -import com.yahoo.document.DocumentOperation; -import com.yahoo.document.DocumentPut; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.config.docproc.SplitterJoinerDocumentProcessorConfig; -import com.yahoo.docproc.DocumentProcessor; -import com.yahoo.docproc.Processing; -import com.yahoo.document.Document; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.DocumentTypeManagerConfigurer; -import com.yahoo.document.datatypes.Array; -import java.util.logging.Level; - -import java.util.logging.Logger; - -/** - * @author Einar M R Rosenvinge - * @deprecated Will be removed on Vespa 8 without replacement - */ -@Deprecated(forRemoval = true, since = "7") -@SuppressWarnings("removal") -public class JoinerDocumentProcessor extends DocumentProcessor { - - private static Logger log = Logger.getLogger(JoinerDocumentProcessor.class.getName()); - private String documentTypeName; - private String arrayFieldName; - private String contextFieldName; - private DocumentTypeManager manager; - - public JoinerDocumentProcessor(SplitterJoinerDocumentProcessorConfig cfg, DocumentmanagerConfig documentmanagerConfig) { - super(); - this.documentTypeName = cfg.documentTypeName(); - this.arrayFieldName = cfg.arrayFieldName(); - this.contextFieldName = cfg.contextFieldName(); - manager = DocumentTypeManagerConfigurer.configureNewManager(documentmanagerConfig); - SplitterDocumentProcessor.validate(manager, documentTypeName, arrayFieldName); - } - - @Override - public Progress process(Processing processing) { - if ( ! SplitterDocumentProcessor.doProcessOuterDocument(processing.getVariable(contextFieldName), documentTypeName)) { - return Progress.DONE; - } - - DocumentPut outerDoc = (DocumentPut)processing.getVariable(contextFieldName); - - @SuppressWarnings("unchecked") - Array<Document> innerDocuments = (Array<Document>) outerDoc.getDocument().getFieldValue(arrayFieldName); - - if (innerDocuments == null) { - @SuppressWarnings("unchecked") - Array<Document> empty = (Array<Document>) outerDoc.getDocument().getDataType().getField(arrayFieldName).getDataType().createFieldValue(); - innerDocuments = empty; - } - - for (DocumentOperation op : processing.getDocumentOperations()) { - if (op instanceof DocumentPut) { - innerDocuments.add(((DocumentPut)op).getDocument()); - } else { - log.log(Level.FINE, () -> "Skipping: " + op); - } - } - processing.getDocumentOperations().clear(); - processing.getDocumentOperations().add(outerDoc); - processing.removeVariable(contextFieldName); - return Progress.DONE; - } - -} diff --git a/docproc/src/main/java/com/yahoo/docproc/util/SplitterDocumentProcessor.java b/docproc/src/main/java/com/yahoo/docproc/util/SplitterDocumentProcessor.java deleted file mode 100644 index 1c6e7617edb..00000000000 --- a/docproc/src/main/java/com/yahoo/docproc/util/SplitterDocumentProcessor.java +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.docproc.util; - -import com.yahoo.document.DocumentOperation; -import com.yahoo.document.DocumentPut; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.config.docproc.SplitterJoinerDocumentProcessorConfig; -import com.yahoo.docproc.DocumentProcessor; -import com.yahoo.docproc.Processing; -import com.yahoo.document.ArrayDataType; -import com.yahoo.document.Document; -import com.yahoo.document.DocumentType; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.DocumentTypeManagerConfigurer; -import com.yahoo.document.datatypes.Array; -import java.util.logging.Level; - -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * @author Einar M R Rosenvinge - * @deprecated Will be removed on Vespa 8 without replacement - */ -@Deprecated(forRemoval = true, since = "7") -public class SplitterDocumentProcessor extends DocumentProcessor { - - private static Logger log = Logger.getLogger(SplitterDocumentProcessor.class.getName()); - private String documentTypeName; - private String arrayFieldName; - private String contextFieldName; - DocumentTypeManager manager; - - public SplitterDocumentProcessor(SplitterJoinerDocumentProcessorConfig cfg, DocumentmanagerConfig documentmanagerConfig) { - super(); - this.documentTypeName = cfg.documentTypeName(); - this.arrayFieldName = cfg.arrayFieldName(); - this.contextFieldName = cfg.contextFieldName(); - this.manager = DocumentTypeManagerConfigurer.configureNewManager(documentmanagerConfig); - validate(manager, documentTypeName, arrayFieldName); - } - - @Override - public Progress process(Processing processing) { - if (processing.getDocumentOperations().size() != 1) { - //we were given more than one document, return - log.log(Level.FINE, () -> "More than one document given, returning. (Was given " - + processing.getDocumentOperations().size() + " documents)."); - return Progress.DONE; - } - - if (!doProcessOuterDocument(processing.getDocumentOperations().get(0), documentTypeName)) { - return Progress.DONE; - } - - Document outerDoc = ((DocumentPut)processing.getDocumentOperations().get(0)).getDocument();; - - @SuppressWarnings("unchecked") - Array<Document> innerDocuments = (Array<Document>) outerDoc.getFieldValue(arrayFieldName); - if (innerDocuments == null) { - //the document does not have the field, return - log.log(Level.FINE, () -> "The given Document does not have a field value for field " - + arrayFieldName + ", returning. (Was given " + outerDoc + ")."); - return Progress.DONE; - } - - if (innerDocuments.size() == 0) { - //the array is empty, return - log.log(Level.FINE, () -> "The given Document does not have any elements in array field " - + arrayFieldName + ", returning. (Was given " + outerDoc + ")."); - return Progress.DONE; - } - - split(processing, innerDocuments); - return Progress.DONE; - } - - private void split(Processing processing, Array<Document> innerDocuments) { - processing.setVariable(contextFieldName, processing.getDocumentOperations().get(0)); - processing.getDocumentOperations().clear(); - processing.getDocumentOperations().addAll(innerDocuments.stream() - .map(DocumentPut::new) - .collect(Collectors.toList())); - - innerDocuments.clear(); - } - - - static void validate(DocumentTypeManager manager, String documentTypeName, String arrayFieldName) { - DocumentType docType = manager.getDocumentType(documentTypeName); - - if (docType == null) { - //the document type does not exist, return - throw new IllegalStateException("The document type " + documentTypeName + " is not deployed."); - } - - if (docType.getField(arrayFieldName) == null) { - //the document type does not have the field, return - throw new IllegalStateException("The document type " + documentTypeName - + " does not have a field named " + arrayFieldName + "."); - } - - if (!(docType.getField(arrayFieldName).getDataType() instanceof ArrayDataType)) { - //the data type of the field is wrong, return - throw new IllegalStateException("The data type of the field named " - + arrayFieldName + " in document type " + documentTypeName - + " is not an array type"); - } - - ArrayDataType fieldDataType = (ArrayDataType) docType.getField(arrayFieldName).getDataType(); - - if (!(fieldDataType.getNestedType() instanceof DocumentType)) { - //the subtype of tye array data type of the field is wrong, return - throw new IllegalStateException("The data type of the field named " - + arrayFieldName + " in document type " + documentTypeName - + " is not an array of Document."); - } - } - - static boolean doProcessOuterDocument(Object o, String documentTypeName) { - if ( ! (o instanceof DocumentOperation)) { - if (log.isLoggable(Level.FINE)) { - log.log(Level.FINE, o + " is not a DocumentOperation."); - } - return false; - } - - DocumentOperation outerDocOp = (DocumentOperation)o; - if ( ! (outerDocOp instanceof DocumentPut)) { - //this is not a put, return - if (log.isLoggable(Level.FINE)) { - log.log(Level.FINE, "Given DocumentOperation is not a DocumentPut, returning. (Was given " - + outerDocOp + ")."); - } - return false; - } - - Document outerDoc = ((DocumentPut) outerDocOp).getDocument(); - DocumentType type = outerDoc.getDataType(); - if (!type.getName().equalsIgnoreCase(documentTypeName)) { - //this is not the right document type - if (log.isLoggable(Level.FINE)) { - log.log(Level.FINE, "Given Document is of wrong type, returning. (Was given " + outerDoc + ")."); - } - return false; - } - return true; - } - -} diff --git a/docproc/src/main/java/com/yahoo/docproc/util/package-info.java b/docproc/src/main/java/com/yahoo/docproc/util/package-info.java deleted file mode 100644 index ae68d79bb0e..00000000000 --- a/docproc/src/main/java/com/yahoo/docproc/util/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -@PublicApi // TODO Vespa 8: revoke from public api -package com.yahoo.docproc.util; - -import com.yahoo.api.annotations.PublicApi; -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/docproc/src/test/java/com/yahoo/docproc/CallbackTestCase.java b/docproc/src/test/java/com/yahoo/docproc/CallbackTestCase.java index 6f3d332a17c..53df0fbd460 100644 --- a/docproc/src/test/java/com/yahoo/docproc/CallbackTestCase.java +++ b/docproc/src/test/java/com/yahoo/docproc/CallbackTestCase.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.docproc; +import com.yahoo.docproc.impl.DocprocService; +import com.yahoo.docproc.impl.ProcessingEndpoint; import com.yahoo.document.DataType; import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentOperation; @@ -18,12 +20,11 @@ import static org.junit.Assert.assertEquals; /** * @author Einar M R Rosenvinge */ -@SuppressWarnings("removal") // TODO Vespa 8: remove public class CallbackTestCase { private DocumentPut put1; private DocumentPut put2; - private List<DocumentOperation> operations = new ArrayList<>(2); + private final List<DocumentOperation> operations = new ArrayList<>(2); DocprocService service; @Before diff --git a/docproc/src/test/java/com/yahoo/docproc/DocumentProcessingAbstractTestCase.java b/docproc/src/test/java/com/yahoo/docproc/DocumentProcessingAbstractTestCase.java index 6a0d7727393..3e017186ce1 100644 --- a/docproc/src/test/java/com/yahoo/docproc/DocumentProcessingAbstractTestCase.java +++ b/docproc/src/test/java/com/yahoo/docproc/DocumentProcessingAbstractTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.docproc; +import com.yahoo.docproc.impl.DocprocService; import com.yahoo.document.DataType; import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentOperation; @@ -15,7 +16,6 @@ import static org.junit.Assert.assertEquals; * * @author bratseth */ -@SuppressWarnings("removal") // TODO Vespa 8: remove public abstract class DocumentProcessingAbstractTestCase { /** diff --git a/docproc/src/test/java/com/yahoo/docproc/EmptyProcessingTestCase.java b/docproc/src/test/java/com/yahoo/docproc/EmptyProcessingTestCase.java index 14a6dc3c2d5..fd312acd3ff 100644 --- a/docproc/src/test/java/com/yahoo/docproc/EmptyProcessingTestCase.java +++ b/docproc/src/test/java/com/yahoo/docproc/EmptyProcessingTestCase.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.docproc; +import com.yahoo.docproc.impl.DocprocService; import org.junit.Test; /** * @author Einar M R Rosenvinge */ -@SuppressWarnings("removal") // TODO Vespa 8: remove public class EmptyProcessingTestCase { @Test diff --git a/docproc/src/test/java/com/yahoo/docproc/FailingDocumentProcessingTestCase.java b/docproc/src/test/java/com/yahoo/docproc/FailingDocumentProcessingTestCase.java index 669621bb926..ef7525aa57b 100644 --- a/docproc/src/test/java/com/yahoo/docproc/FailingDocumentProcessingTestCase.java +++ b/docproc/src/test/java/com/yahoo/docproc/FailingDocumentProcessingTestCase.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.docproc; +import com.yahoo.docproc.impl.DocprocService; +import com.yahoo.docproc.impl.HandledProcessingException; import com.yahoo.document.DataType; import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentPut; @@ -15,7 +17,6 @@ import static org.junit.Assert.assertEquals; * * @author bratseth */ -@SuppressWarnings("removal") // TODO Vespa 8: remove public class FailingDocumentProcessingTestCase { /** diff --git a/docproc/src/test/java/com/yahoo/docproc/FailingDocumentProcessingWithoutExceptionTestCase.java b/docproc/src/test/java/com/yahoo/docproc/FailingDocumentProcessingWithoutExceptionTestCase.java index 6293ac8e9f4..782d76836d1 100644 --- a/docproc/src/test/java/com/yahoo/docproc/FailingDocumentProcessingWithoutExceptionTestCase.java +++ b/docproc/src/test/java/com/yahoo/docproc/FailingDocumentProcessingWithoutExceptionTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.docproc; +import com.yahoo.docproc.impl.DocprocService; import com.yahoo.document.DataType; import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentOperation; @@ -16,7 +17,6 @@ import static org.junit.Assert.assertEquals; * * @author Einar M. R. Rosenvinge */ -@SuppressWarnings("removal") // TODO Vespa 8: remove public class FailingDocumentProcessingWithoutExceptionTestCase { /** diff --git a/docproc/src/test/java/com/yahoo/docproc/FailingPermanentlyDocumentProcessingTestCase.java b/docproc/src/test/java/com/yahoo/docproc/FailingPermanentlyDocumentProcessingTestCase.java index 4da76484c1f..a5f2bf71023 100644 --- a/docproc/src/test/java/com/yahoo/docproc/FailingPermanentlyDocumentProcessingTestCase.java +++ b/docproc/src/test/java/com/yahoo/docproc/FailingPermanentlyDocumentProcessingTestCase.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.docproc; +import com.yahoo.docproc.impl.DocprocService; import com.yahoo.document.DataType; -import com.yahoo.document.Document; import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentOperation; import com.yahoo.document.DocumentPut; @@ -20,7 +20,6 @@ import static org.junit.Assert.assertTrue; * * @author Einar M. R. Rosenvinge */ -@SuppressWarnings("removal") // TODO Vespa 8: remove public class FailingPermanentlyDocumentProcessingTestCase { /** diff --git a/docproc/src/test/java/com/yahoo/docproc/FailingWithErrorTestCase.java b/docproc/src/test/java/com/yahoo/docproc/FailingWithErrorTestCase.java index c4442fadad3..d631107fdea 100644 --- a/docproc/src/test/java/com/yahoo/docproc/FailingWithErrorTestCase.java +++ b/docproc/src/test/java/com/yahoo/docproc/FailingWithErrorTestCase.java @@ -1,10 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.docproc; +import com.yahoo.docproc.impl.DocprocService; import com.yahoo.document.DataType; -import com.yahoo.document.Document; import com.yahoo.document.DocumentId; -import com.yahoo.document.DocumentOperation; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentType; import org.junit.Test; @@ -15,7 +14,6 @@ import static org.junit.Assert.fail; /** * @author Einar M R Rosenvinge */ -@SuppressWarnings("removal") // TODO Vespa 8: remove public class FailingWithErrorTestCase { @Test diff --git a/docproc/src/test/java/com/yahoo/docproc/NotAcceptingNewProcessingsTestCase.java b/docproc/src/test/java/com/yahoo/docproc/NotAcceptingNewProcessingsTestCase.java index 7b1cee20d17..b1dd3ce1c9f 100644 --- a/docproc/src/test/java/com/yahoo/docproc/NotAcceptingNewProcessingsTestCase.java +++ b/docproc/src/test/java/com/yahoo/docproc/NotAcceptingNewProcessingsTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.docproc; +import com.yahoo.docproc.impl.DocprocService; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -9,7 +10,6 @@ import static org.junit.Assert.fail; /** * @author Einar M R Rosenvinge */ -@SuppressWarnings("removal") // TODO Vespa 8: remove public class NotAcceptingNewProcessingsTestCase { @Test diff --git a/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java b/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java index 9afce98dda1..e37b1d78259 100644 --- a/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java +++ b/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.docproc; +import com.yahoo.docproc.impl.DocprocService; import com.yahoo.document.DataType; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; @@ -35,7 +36,6 @@ public class ProcessingUpdateTestCase { private DocumentTypeManager dtm; @Test - @SuppressWarnings("removal") // TODO Vespa 8: remove public void testProcessingUpdates() { DocumentType articleType = new DocumentType("article"); Field bodyField = new Field("body", DataType.STRING); diff --git a/docproc/src/test/java/com/yahoo/docproc/SimpleDocumentProcessingTestCase.java b/docproc/src/test/java/com/yahoo/docproc/SimpleDocumentProcessingTestCase.java index 84d0cad7a4e..2c97f79bdd5 100644 --- a/docproc/src/test/java/com/yahoo/docproc/SimpleDocumentProcessingTestCase.java +++ b/docproc/src/test/java/com/yahoo/docproc/SimpleDocumentProcessingTestCase.java @@ -3,6 +3,7 @@ package com.yahoo.docproc; import com.yahoo.component.chain.dependencies.After; import com.yahoo.docproc.Accesses.Field.Tree; +import com.yahoo.docproc.impl.DocprocService; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -20,7 +21,6 @@ public class SimpleDocumentProcessingTestCase extends DocumentProcessingAbstract * on some documents */ @Test - @SuppressWarnings("removal") // TODO Vespa 8: remove public void testSimpleProcessing() { // Set up service programmatically DocprocService service = new DocprocService("simple"); diff --git a/docproc/src/test/java/com/yahoo/docproc/SimpleDocumentProcessorTestCase.java b/docproc/src/test/java/com/yahoo/docproc/SimpleDocumentProcessorTestCase.java index e23c1d2e7fd..ce39e265bc8 100644 --- a/docproc/src/test/java/com/yahoo/docproc/SimpleDocumentProcessorTestCase.java +++ b/docproc/src/test/java/com/yahoo/docproc/SimpleDocumentProcessorTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.docproc; +import com.yahoo.docproc.impl.DocprocService; import com.yahoo.jdisc.test.MockMetric; import com.yahoo.document.DataType; import com.yahoo.document.DocumentId; @@ -23,7 +24,6 @@ import static org.junit.Assert.assertTrue; /** * @author Einar M R Rosenvinge */ -@SuppressWarnings("removal") // TODO Vespa 8: remove public class SimpleDocumentProcessorTestCase { private static DocprocService setupDocprocService(SimpleDocumentProcessor processor, Metric metric) { diff --git a/docproc/src/test/java/com/yahoo/docproc/TransientFailureTestCase.java b/docproc/src/test/java/com/yahoo/docproc/TransientFailureTestCase.java index 44c0347aa1f..12943b8086d 100644 --- a/docproc/src/test/java/com/yahoo/docproc/TransientFailureTestCase.java +++ b/docproc/src/test/java/com/yahoo/docproc/TransientFailureTestCase.java @@ -1,6 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.docproc; +import com.yahoo.docproc.impl.DocprocService; +import com.yahoo.docproc.impl.ProcessingEndpoint; +import com.yahoo.docproc.impl.TransientFailureException; import com.yahoo.document.DataType; import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentOperation; @@ -14,7 +17,6 @@ import static org.junit.Assert.assertEquals; /** * @author Einar M R Rosenvinge */ -@SuppressWarnings("removal") // TODO Vespa 8: remove public class TransientFailureTestCase { DocumentType type; diff --git a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerTestBase.java b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerTestBase.java index de5cbfa505b..b7c02101e3d 100644 --- a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerTestBase.java +++ b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerTestBase.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.docproc.jdisc; -import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.collections.Pair; import com.yahoo.component.ComponentId; import com.yahoo.component.provider.ComponentRegistry; @@ -11,16 +10,13 @@ import com.yahoo.container.jdisc.messagebus.MbusServerProvider; import com.yahoo.container.jdisc.messagebus.NetworkMultiplexerProvider; import com.yahoo.container.jdisc.messagebus.SessionCache; import com.yahoo.docproc.CallStack; -import com.yahoo.docproc.DocprocService; +import com.yahoo.docproc.impl.DocprocService; import com.yahoo.docproc.jdisc.messagebus.MbusRequestContext; import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.documentapi.messagebus.loadtypes.LoadType; import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.jdisc.AbstractResource; import com.yahoo.jdisc.ReferencedResource; import com.yahoo.jdisc.application.ContainerBuilder; @@ -34,8 +30,6 @@ import com.yahoo.messagebus.network.NetworkMultiplexer; import com.yahoo.messagebus.network.rpc.RPCNetwork; import com.yahoo.messagebus.routing.Route; import com.yahoo.messagebus.shared.SharedSourceSession; -import com.yahoo.vespa.config.content.DistributionConfig; -import com.yahoo.vespa.config.content.LoadTypeConfig; import org.junit.After; import org.junit.Before; @@ -55,7 +49,6 @@ public abstract class DocumentProcessingHandlerTestBase { private final List<MbusServerProvider> serviceProviders = new ArrayList<>(); @Before - @SuppressWarnings("removal") // TODO Vespa 8: remove public void createHandler() { documentTypeManager.register(getType()); @@ -72,7 +65,6 @@ public abstract class DocumentProcessingHandlerTestBase { protocol); ContainerBuilder builder = driver.parent().newContainerBuilder(); - @SuppressWarnings("removal") // TODO Vespa 8: remove ComponentRegistry<DocprocService> registry = new ComponentRegistry<>(); handler = new DocumentProcessingHandler(registry, @@ -135,11 +127,8 @@ public abstract class DocumentProcessingHandlerTestBase { protected abstract DocumentType getType(); - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public boolean sendMessage(String destinationChainName, DocumentMessage msg) { msg.setRoute(Route.parse("test/chain." + destinationChainName + " " + remoteServer.connectionSpec())); - msg.setPriority(DocumentProtocol.Priority.HIGH_1); // TODO: Remove on Vespa 8 - msg.setLoadType(LoadType.DEFAULT); msg.getTrace().setLevel(9); msg.setTimeRemaining(60 * 1000); return driver.client().sendMessage(msg).isAccepted(); diff --git a/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java b/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java index cfb051e131e..a986c4deff3 100644 --- a/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java +++ b/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java @@ -305,7 +305,6 @@ public class SchemaMappingAndAccessesTest { mapped.setId(new DocumentId("id:map:album::2")); assertEquals(mapped.getId().toString(), "id:map:album::2"); assertEquals(doc.getId().toString(), "id:map:album::2"); - assertEquals(doc.getHeader(), mapped.getHeader()); assertEquals(doc.getSerializedSize(), mapped.getSerializedSize()); ByteArrayOutputStream bos = new ByteArrayOutputStream(); ByteArrayOutputStream bos2 = new ByteArrayOutputStream(); diff --git a/docproc/src/test/java/com/yahoo/docproc/util/SplitterJoinerTestCase.java b/docproc/src/test/java/com/yahoo/docproc/util/SplitterJoinerTestCase.java deleted file mode 100644 index 10e10011e55..00000000000 --- a/docproc/src/test/java/com/yahoo/docproc/util/SplitterJoinerTestCase.java +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.docproc.util; - -import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.config.docproc.SplitterJoinerDocumentProcessorConfig; -import com.yahoo.document.DocumentPut; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.docproc.Processing; -import com.yahoo.document.Document; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.datatypes.Array; -import com.yahoo.document.datatypes.StringFieldValue; -import org.junit.Test; - -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * @author Einar M R Rosenvinge - */ -@SuppressWarnings({"unchecked", "removal"}) -public class SplitterJoinerTestCase { - - @Test - public void testSplitJoin() { - ConfigGetter<SplitterJoinerDocumentProcessorConfig> getter = new ConfigGetter<>(SplitterJoinerDocumentProcessorConfig.class); - ConfigGetter<DocumentmanagerConfig> docManGetter = new ConfigGetter<>(DocumentmanagerConfig.class); - - SplitterJoinerDocumentProcessorConfig cfg = - getter.getConfig("file:src/test/java/com/yahoo/docproc/util/splitter-joiner-document-processor.cfg"); - DocumentmanagerConfig docManCfg = - docManGetter.getConfig("file:src/test/java/com/yahoo/docproc/util/documentmanager.docindoc.cfg"); - - SplitterDocumentProcessor splitter = new SplitterDocumentProcessor(cfg, docManCfg); - - DocumentTypeManager manager = splitter.manager; - - - // Create documents - - Document inner1 = new Document(manager.getDocumentType("docindoc"), "id:inner:docindoc::one"); - inner1.setFieldValue("name", new StringFieldValue("Donald Duck")); - inner1.setFieldValue("content", new StringFieldValue("Lives in Duckburg")); - Document inner2 = new Document(manager.getDocumentType("docindoc"), "id:inner:docindoc::number:two"); - inner2.setFieldValue("name", new StringFieldValue("Uncle Scrooge")); - inner2.setFieldValue("content", new StringFieldValue("Lives in Duckburg, too.")); - - Array<Document> innerArray = (Array<Document>) manager.getDocumentType("outerdoc").getField("innerdocuments").getDataType().createFieldValue(); - innerArray.add(inner1); - innerArray.add(inner2); - - Document outer = new Document(manager.getDocumentType("outerdoc"), "id:outer:outerdoc::the:only:one"); - outer.setFieldValue("innerdocuments", innerArray); - - // End create documents - - - Processing p = Processing.of(new DocumentPut(outer)); - splitter.process(p); - - assertEquals(2, p.getDocumentOperations().size()); - assertSame(inner1, ((DocumentPut)(p.getDocumentOperations().get(0))).getDocument()); - assertSame(inner2, ((DocumentPut)(p.getDocumentOperations().get(1))).getDocument()); - assertSame(outer, ((DocumentPut)(p.getVariable(cfg.contextFieldName()))).getDocument()); - assertSame(innerArray, outer.getFieldValue("innerdocuments")); - assertTrue(innerArray.isEmpty()); - - - JoinerDocumentProcessor joiner = new JoinerDocumentProcessor(cfg, docManCfg); - - joiner.process(p); - - assertEquals(1, p.getDocumentOperations().size()); - assertSame(outer, ((DocumentPut)p.getDocumentOperations().get(0)).getDocument()); - assertNull(p.getVariable(cfg.contextFieldName())); - assertSame(innerArray, outer.getFieldValue("innerdocuments")); - assertEquals(2, innerArray.size()); - assertSame(inner1, innerArray.get(0)); - assertSame(inner2, innerArray.get(1)); - } - -} diff --git a/docproc/src/test/java/com/yahoo/docproc/util/docindoc.sd b/docproc/src/test/java/com/yahoo/docproc/util/docindoc.sd deleted file mode 100644 index ca774778984..00000000000 --- a/docproc/src/test/java/com/yahoo/docproc/util/docindoc.sd +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search docindoc { - document docindoc { - field name type string { } - field content type string { } - } -} diff --git a/docproc/src/test/java/com/yahoo/docproc/util/documentmanager.docindoc.cfg b/docproc/src/test/java/com/yahoo/docproc/util/documentmanager.docindoc.cfg deleted file mode 100644 index 0b6c59ca381..00000000000 --- a/docproc/src/test/java/com/yahoo/docproc/util/documentmanager.docindoc.cfg +++ /dev/null @@ -1,73 +0,0 @@ -doctype[3] -doctype[0].name "document" -doctype[0].idx 10000 -doctype[0].contentstruct 10001 -doctype[0].primitivetype[0].idx 10002 -doctype[0].primitivetype[0].internalid 0 -doctype[0].primitivetype[0].name "int" -doctype[0].primitivetype[1].idx 10003 -doctype[0].primitivetype[1].internalid 5 -doctype[0].primitivetype[1].name "double" -doctype[0].primitivetype[2].idx 10004 -doctype[0].primitivetype[2].internalid 2 -doctype[0].primitivetype[2].name "string" -doctype[0].annotationtype[0].idx 10005 -doctype[0].annotationtype[0].name "proximity_break" -doctype[0].annotationtype[0].internalid 8 -doctype[0].annotationtype[0].datatype 10003 -doctype[0].annotationtype[1].idx 10006 -doctype[0].annotationtype[1].name "normalized" -doctype[0].annotationtype[1].internalid 4 -doctype[0].annotationtype[1].datatype 10004 -doctype[0].annotationtype[2].idx 10007 -doctype[0].annotationtype[2].name "reading" -doctype[0].annotationtype[2].internalid 5 -doctype[0].annotationtype[2].datatype 10004 -doctype[0].annotationtype[3].idx 10008 -doctype[0].annotationtype[3].name "term" -doctype[0].annotationtype[3].internalid 1 -doctype[0].annotationtype[3].datatype 10004 -doctype[0].annotationtype[4].idx 10009 -doctype[0].annotationtype[4].name "transformed" -doctype[0].annotationtype[4].internalid 7 -doctype[0].annotationtype[4].datatype 10004 -doctype[0].annotationtype[5].idx 10010 -doctype[0].annotationtype[5].name "canonical" -doctype[0].annotationtype[5].internalid 3 -doctype[0].annotationtype[5].datatype 10004 -doctype[0].annotationtype[6].idx 10011 -doctype[0].annotationtype[6].name "token_type" -doctype[0].annotationtype[6].internalid 2 -doctype[0].annotationtype[6].datatype 10002 -doctype[0].annotationtype[7].idx 10012 -doctype[0].annotationtype[7].name "special_token" -doctype[0].annotationtype[7].internalid 9 -doctype[0].annotationtype[8].idx 10013 -doctype[0].annotationtype[8].name "stem" -doctype[0].annotationtype[8].internalid 6 -doctype[0].annotationtype[8].datatype 10004 -doctype[0].structtype[0].idx 10001 -doctype[0].structtype[0].name document.header -doctype[1].name "docindoc" -doctype[1].idx 10014 -doctype[1].inherits[0].idx 10000 -doctype[1].contentstruct 10015 -doctype[1].structtype[0].idx 10015 -doctype[1].structtype[0].name docindoc.header -doctype[1].structtype[0].field[0].name "name" -doctype[1].structtype[0].field[0].internalid 1160796772 -doctype[1].structtype[0].field[0].type 10004 -doctype[1].structtype[0].field[1].name "content" -doctype[1].structtype[0].field[1].internalid 482196823 -doctype[1].structtype[0].field[1].type 10004 -doctype[2].name "outerdoc" -doctype[2].idx 10016 -doctype[2].inherits[0].idx 10000 -doctype[2].contentstruct 10017 -doctype[2].arraytype[0].idx 10018 -doctype[2].arraytype[0].elementtype 10014 -doctype[2].structtype[0].idx 10017 -doctype[2].structtype[0].name outerdoc.header -doctype[2].structtype[0].field[0].name "innerdocuments" -doctype[2].structtype[0].field[0].internalid 1655182511 -doctype[2].structtype[0].field[0].type 10018 diff --git a/docproc/src/test/java/com/yahoo/docproc/util/outerdoc.sd b/docproc/src/test/java/com/yahoo/docproc/util/outerdoc.sd deleted file mode 100644 index 50981d307b9..00000000000 --- a/docproc/src/test/java/com/yahoo/docproc/util/outerdoc.sd +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search outerdoc { - document outerdoc { - field innerdocuments type array<docindoc> { } - } -} diff --git a/docproc/src/test/java/com/yahoo/docproc/util/splitter-joiner-document-processor.cfg b/docproc/src/test/java/com/yahoo/docproc/util/splitter-joiner-document-processor.cfg deleted file mode 100644 index 16c26f7a6d8..00000000000 --- a/docproc/src/test/java/com/yahoo/docproc/util/splitter-joiner-document-processor.cfg +++ /dev/null @@ -1,2 +0,0 @@ -documentTypeName "outerdoc" -arrayFieldName "innerdocuments" diff --git a/docprocs/pom.xml b/docprocs/pom.xml index 24df58ad268..42ae75e36f1 100644 --- a/docprocs/pom.xml +++ b/docprocs/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>docprocs</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -89,6 +89,10 @@ <version>${project.version}</version> <exclusions> <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + <exclusion> <groupId>com.yahoo.vespa</groupId> <artifactId>document</artifactId> </exclusion> @@ -100,10 +104,6 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>annotation</artifactId> </exclusion> - <exclusion> - <groupId>org.json</groupId> - <artifactId>json</artifactId> - </exclusion> </exclusions> </dependency> </dependencies> diff --git a/document/abi-spec.json b/document/abi-spec.json index d7cc513619f..e7960298b90 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -42,9 +42,6 @@ "public boolean hasField(java.lang.String)", "public java.util.Collection getFields()", "public int getFieldCount()", - "public com.yahoo.compress.Compressor getCompressor()", - "public com.yahoo.document.CompressionConfig getCompressionConfig()", - "public void setCompressionConfig(com.yahoo.document.CompressionConfig)", "public bridge synthetic com.yahoo.document.DataType clone()", "public bridge synthetic com.yahoo.vespa.objects.Identifiable clone()", "public bridge synthetic java.lang.Object clone()" @@ -132,7 +129,6 @@ "public com.yahoo.document.CollectionDataType clone()", "public com.yahoo.document.DataType getNestedType()", "protected com.yahoo.document.datatypes.FieldValue createByReflection(java.lang.Object)", - "public void setNestedType(com.yahoo.document.DataType)", "public com.yahoo.document.PrimitiveDataType getPrimitiveType()", "public boolean isValueCompatible(com.yahoo.document.datatypes.FieldValue)", "protected void register(com.yahoo.document.DocumentTypeManager, java.util.List)", @@ -266,8 +262,6 @@ "public void <init>(com.yahoo.document.serialization.DocumentReader)", "public com.yahoo.document.DocumentId getId()", "public void setId(com.yahoo.document.DocumentId)", - "public com.yahoo.document.datatypes.Struct getHeader()", - "public com.yahoo.document.datatypes.Struct getBody()", "public void assign(java.lang.Object)", "public com.yahoo.document.Document clone()", "public void setDataType(com.yahoo.document.DataType)", @@ -428,9 +422,7 @@ "methods": [ "public void <init>(java.lang.String)", "public void <init>(java.lang.String, com.yahoo.document.StructDataType)", - "public void <init>(java.lang.String, com.yahoo.document.StructDataType, com.yahoo.document.StructDataType)", "public void <init>(java.lang.String, com.yahoo.document.StructDataType, java.util.Set)", - "public void <init>(java.lang.String, com.yahoo.document.StructDataType, com.yahoo.document.StructDataType, java.util.Set)", "public void <init>(java.lang.String, java.util.Set)", "public com.yahoo.document.DocumentType clone()", "public com.yahoo.document.Document createFieldValue()", @@ -439,13 +431,11 @@ "public com.yahoo.document.StructDataType contentStruct()", "public com.yahoo.document.StructDataType getStructType(java.lang.String)", "public com.yahoo.document.StructDataType getDeclaredStructType(java.lang.String)", - "public com.yahoo.document.StructDataType getHeaderType()", "protected void register(com.yahoo.document.DocumentTypeManager, java.util.List)", "public boolean isA(java.lang.String)", "public void addField(com.yahoo.document.Field)", "public void addFieldSets(java.util.Map)", "public com.yahoo.document.Field addField(java.lang.String, com.yahoo.document.DataType)", - "public com.yahoo.document.Field addHeaderField(java.lang.String, com.yahoo.document.DataType)", "public void inherit(com.yahoo.document.DocumentType)", "public java.util.Collection getInheritedTypes()", "public java.util.ListIterator inheritedIterator()", @@ -501,26 +491,18 @@ "methods": [ "public void <init>()", "public void <init>(com.yahoo.document.config.DocumentmanagerConfig)", - "public void assign(com.yahoo.document.DocumentTypeManager)", - "public com.yahoo.document.DocumentTypeManager configure(java.lang.String)", "public static com.yahoo.document.DocumentTypeManager fromFile(java.lang.String)", - "public boolean hasDataType(java.lang.String)", - "public boolean hasDataType(int)", - "public com.yahoo.document.DataType getDataType(java.lang.String)", - "public com.yahoo.document.DataType getDataType(int)", - "public com.yahoo.document.DataType getDataType(int, java.lang.String)", "public void register(com.yahoo.document.DataType)", "public com.yahoo.document.DocumentType registerDocumentType(com.yahoo.document.DocumentType)", "public com.yahoo.document.DocumentType getDocumentType(com.yahoo.document.DataTypeName)", "public com.yahoo.document.DocumentType getDocumentType(java.lang.String)", + "public boolean hasDocumentType(java.lang.String)", "public final com.yahoo.document.Document createDocument(com.yahoo.io.GrowableByteBuffer)", "public com.yahoo.document.Document createDocument(com.yahoo.document.serialization.DocumentDeserializer)", "public java.util.Collection getDataTypes()", "public java.util.Map getDocumentTypes()", "public java.util.Iterator documentTypeIterator()", - "public void clear()", - "public com.yahoo.document.annotation.AnnotationTypeRegistry getAnnotationTypeRegistry()", - "public void shutdown()" + "public com.yahoo.document.annotation.AnnotationTypeRegistry getAnnotationTypeRegistry()" ], "fields": [] }, @@ -534,7 +516,6 @@ ], "methods": [ "public void <init>(com.yahoo.document.DocumentTypeManager)", - "public static com.yahoo.compress.CompressionType toCompressorType(com.yahoo.document.config.DocumentmanagerConfig$Datatype$Structtype$Compresstype$Enum)", "public static com.yahoo.config.subscription.ConfigSubscriber configure(com.yahoo.document.DocumentTypeManager, java.lang.String)", "public com.yahoo.config.subscription.ConfigSubscriber configure(java.lang.String)", "public static com.yahoo.document.DocumentTypeManager configureNewManager(com.yahoo.document.config.DocumentmanagerConfig)", @@ -669,12 +650,9 @@ "public" ], "methods": [ - "public void <init>(java.lang.String, int, com.yahoo.document.DataType, boolean)", "public void <init>(java.lang.String, int, com.yahoo.document.DataType)", "public void <init>(java.lang.String)", - "public void <init>(java.lang.String, com.yahoo.document.DataType, boolean, com.yahoo.document.DocumentType)", "public void <init>(java.lang.String, com.yahoo.document.DataType, com.yahoo.document.DocumentType)", - "public void <init>(java.lang.String, com.yahoo.document.DataType, boolean)", "public void <init>(java.lang.String, com.yahoo.document.DataType)", "public void <init>(java.lang.String, com.yahoo.document.Field)", "public int compareTo(java.lang.Object)", @@ -682,11 +660,8 @@ "public void setId(int, com.yahoo.document.DocumentType)", "public final com.yahoo.document.DataType getDataType()", "public void setDataType(com.yahoo.document.DataType)", - "public final int getId(int)", "public final int getId()", "public final boolean hasForcedId()", - "public boolean isHeader()", - "public void setHeader(boolean)", "public boolean equals(java.lang.Object)", "public int hashCode()", "public java.lang.String toString()", @@ -854,8 +829,6 @@ "public boolean isValueCompatible(com.yahoo.document.datatypes.FieldValue)", "public com.yahoo.document.DataType getKeyType()", "public com.yahoo.document.DataType getValueType()", - "public void setKeyType(com.yahoo.document.DataType)", - "public void setValueType(com.yahoo.document.DataType)", "public com.yahoo.document.datatypes.MapFieldValue createFieldValue()", "public java.lang.Class getValueClass()", "protected void register(com.yahoo.document.DocumentTypeManager, java.util.List)", @@ -902,9 +875,7 @@ "public static com.yahoo.document.datatypes.Struct fromString(java.lang.String)", "public static com.yahoo.document.datatypes.IntegerFieldValue getXValue(com.yahoo.document.datatypes.FieldValue)", "public static com.yahoo.document.datatypes.IntegerFieldValue getYValue(com.yahoo.document.datatypes.FieldValue)", - "public static java.lang.String getZCurveFieldName(java.lang.String)", - "public static java.lang.String getPositionSummaryFieldName(java.lang.String)", - "public static java.lang.String getDistanceSummaryFieldName(java.lang.String)" + "public static java.lang.String getZCurveFieldName(java.lang.String)" ], "fields": [ "public static final com.yahoo.document.StructDataType INSTANCE", @@ -958,11 +929,8 @@ ], "methods": [ "public void <init>(com.yahoo.document.DocumentType, int)", - "public void <init>(com.yahoo.document.TemporaryStructuredDataType, int)", "public static com.yahoo.document.ReferenceDataType createWithInferredId(com.yahoo.document.DocumentType)", - "public static com.yahoo.document.ReferenceDataType createWithInferredId(com.yahoo.document.TemporaryStructuredDataType)", "public com.yahoo.document.StructuredDataType getTargetType()", - "public void setTargetType(com.yahoo.document.StructuredDataType)", "public com.yahoo.document.datatypes.ReferenceFieldValue createFieldValue()", "public java.lang.Class getValueClass()", "public boolean isValueCompatible(com.yahoo.document.datatypes.FieldValue)", @@ -1059,19 +1027,6 @@ "public static final int classId" ] }, - "com.yahoo.document.TemporaryStructuredDataType": { - "superClass": "com.yahoo.document.StructDataType", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public static com.yahoo.document.TemporaryStructuredDataType create(java.lang.String)", - "public static com.yahoo.document.TemporaryStructuredDataType createById(int)", - "protected void setName(java.lang.String)" - ], - "fields": [] - }, "com.yahoo.document.TensorDataType": { "superClass": "com.yahoo.document.DataType", "interfaces": [], @@ -1124,7 +1079,6 @@ "methods": [ "public void <init>(com.yahoo.document.DataType, boolean, boolean)", "public void <init>(com.yahoo.document.DataType, boolean, boolean, int)", - "public void <init>(java.lang.String, int, com.yahoo.document.DataType, boolean, boolean)", "public com.yahoo.document.WeightedSetDataType clone()", "public void setTag(boolean)", "public boolean isTag()", @@ -2245,9 +2199,6 @@ "public com.yahoo.document.StructDataType getDataType()", "public void setVersion(int)", "public int getVersion()", - "public com.yahoo.compress.CompressionType getCompressionType()", - "public int getCompressionLevel()", - "public float getCompressionThreshold()", "public com.yahoo.document.datatypes.Struct clone()", "public void clear()", "public java.util.Iterator iterator()", @@ -3398,7 +3349,6 @@ "public" ], "methods": [ - "public void <init>(com.yahoo.document.serialization.DocumentUpdateReader, com.yahoo.document.DocumentType, int)", "public void <init>(com.yahoo.document.serialization.DocumentUpdateReader, com.yahoo.document.DocumentType)", "public com.yahoo.document.DocumentType getDocumentType()", "public com.yahoo.document.Field getField()", diff --git a/document/doc/document-format.html b/document/doc/document-format.html index 26e6cceca5c..ce985b8a10d 100644 --- a/document/doc/document-format.html +++ b/document/doc/document-format.html @@ -19,7 +19,6 @@ look-ups.</p> <li><b>Robustness</b>. The format shall detect errors gracefully.</li> <li><b>Speed</b>. Deserialization shall be fast, especially for basic fields like <b>DocumentId</b>.</li> <li><b>Size</b>. The serialized format shall be compact and allow for efficient storage and transfer. -That is partly achieved by allowing different kinds of compression. As of now <b>lz4</b> are supported.</li> </ul> </p> @@ -137,26 +136,9 @@ Inventory bits describing the FieldMap element with data:<br> <td>2, 4 or 8</td> <td>Length of data block (see below). NOTE THAT THIS LENGTH INCLUDE ITSELF.</td> </tr> -<tr><td>Compression</td> -<td>Byte</td> -<td>1</td> -<td>Compression method -<br> - 0: No compression<br> - 5: Uncompressable<br> - 6: lz4 <br> -<p>Note that the uncompressable flag is not a configurable option. Rather it -will be used in document instances who are configured for compression, but -where compression yields negative results, to avoid later serializations to -retry compression.</p> -</td> <tr><td>Number of fields<td>Integer_1_4</td> <td>1 or 4</td> <td>Number of fields in data array</td> -<tr><td colspan = "4"><b>Below item is present if compression method is not uncompressed or uncompressable</b></td></tr> -<tr><td>Uncompressed data length<td>Integer_2_4_8</td> -<td>2, 4 or 8</td> -<td>Length of data block after decompression</td> <tr><td colspan = "4"><b>Below block is repeated "Number of fields" times</b></td></tr> <tr><td>Field ID<td>Integer_1_4</td> <td>1 or 4</td> diff --git a/document/pom.xml b/document/pom.xml index 06488d799eb..5db432d2447 100644 --- a/document/pom.xml +++ b/document/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>document</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/document/src/main/java/com/yahoo/document/BaseStructDataType.java b/document/src/main/java/com/yahoo/document/BaseStructDataType.java index 1bce5d716a4..adecae9d7eb 100755 --- a/document/src/main/java/com/yahoo/document/BaseStructDataType.java +++ b/document/src/main/java/com/yahoo/document/BaseStructDataType.java @@ -1,9 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document; -import com.yahoo.compress.CompressionType; -import com.yahoo.compress.Compressor; - import java.util.Collection; import java.util.LinkedHashMap; import java.util.Map; @@ -99,25 +96,4 @@ public abstract class BaseStructDataType extends StructuredDataType { return fields.size(); } - /** Returns the compressor to use to compress data of this type - * @deprecated Will go away on Vespa 8 - */ - @Deprecated - public Compressor getCompressor() { return new Compressor(CompressionType.NONE); } - - /** Returns a view of the configuration of the compressor used to compress this type - * @deprecated Will go away on Vespa 8 - */ - @Deprecated - public CompressionConfig getCompressionConfig() { - return new CompressionConfig(); - } - - /** - * Set the config to the compressor used to compress data of this type - * @deprecated Ignored and will go away on Vespa 8 - */ - @Deprecated - public void setCompressionConfig(CompressionConfig config) { } - } diff --git a/document/src/main/java/com/yahoo/document/CollectionDataType.java b/document/src/main/java/com/yahoo/document/CollectionDataType.java index de2aa0cff19..1aeccb7e2fa 100644 --- a/document/src/main/java/com/yahoo/document/CollectionDataType.java +++ b/document/src/main/java/com/yahoo/document/CollectionDataType.java @@ -40,16 +40,6 @@ public abstract class CollectionDataType extends DataType { @Override protected FieldValue createByReflection(Object arg) { return null; } - /** - * WARNING! Do not use! Only to be used by config system! - * Sets the nested type of this CollectionDataType. - * @deprecated // TODO Vespa 8 remove - */ - @Deprecated(forRemoval = true, since = "7") - public void setNestedType(DataType nestedType) { - this.nestedType = nestedType; - } - @Override public PrimitiveDataType getPrimitiveType() { return nestedType.getPrimitiveType(); diff --git a/document/src/main/java/com/yahoo/document/Document.java b/document/src/main/java/com/yahoo/document/Document.java index 1a29bf29191..5937ba00292 100644 --- a/document/src/main/java/com/yahoo/document/Document.java +++ b/document/src/main/java/com/yahoo/document/Document.java @@ -96,14 +96,6 @@ public class Document extends StructuredFieldValue { docId = id; } - /** @deprecated do not use: Use getField(), getFieldValue() or iterator() instead */ - @Deprecated // TODO: Remove on Vespa 8 - public Struct getHeader() { return header; } - - /** @deprecated do not use: Use getField(), getFieldValue() or iterator() instead */ - @Deprecated // TODO: Remove on Vespa 8 - public Struct getBody() { return null; } - @Override public void assign(Object o) { throw new IllegalArgumentException("Assign not implemented for " + getClass() + " objects"); diff --git a/document/src/main/java/com/yahoo/document/DocumentType.java b/document/src/main/java/com/yahoo/document/DocumentType.java index 4fb50f48688..0f771e4d55a 100644 --- a/document/src/main/java/com/yahoo/document/DocumentType.java +++ b/document/src/main/java/com/yahoo/document/DocumentType.java @@ -32,12 +32,11 @@ import java.util.Set; * @author Thomas Gundersen * @author bratseth */ -// TODO: Remove header/body concept on Vespa 8 public class DocumentType extends StructuredDataType { public static final String DOCUMENT = "[document]"; public static final int classId = registerClass(Ids.document + 58, DocumentType.class); - private StructDataType headerType; + private StructDataType contentStructType; private List<DocumentType> inherits = new ArrayList<>(1); private Map<String, Set<Field>> fieldSets = new HashMap<>(); private final Set<String> importedFieldNames; @@ -52,7 +51,7 @@ public class DocumentType extends StructuredDataType { * @param name The name of the new document type */ public DocumentType(String name) { - this(name, createHeaderStructType(name)); + this(name, createContentStructType(name)); } /** @@ -61,47 +60,30 @@ public class DocumentType extends StructuredDataType { * The document type id will be generated as a hash from the document type name. * * @param name The name of the new document type - * @param headerType The type of the header struct + * @param contentStructType The type of the content struct */ - public DocumentType(String name, StructDataType headerType) { - this(name, headerType, Collections.emptySet()); + public DocumentType(String name, StructDataType contentStructType) { + this(name, contentStructType, Collections.emptySet()); } - /** - * @deprecated //TODO Will be removed on Vespa 8 - */ - @Deprecated - public DocumentType(String name, StructDataType headerType, StructDataType bodyType) { - this(name, headerType, Collections.emptySet()); - } - - public DocumentType(String name, StructDataType headerType, Set<String> importedFieldNames) { + public DocumentType(String name, StructDataType contentStructType, Set<String> importedFieldNames) { super(name); - this.headerType = headerType; + this.contentStructType = contentStructType; this.importedFieldNames = Collections.unmodifiableSet(importedFieldNames); } - /** - * @deprecated //TODO Will be removed on Vespa 8 - */ - @Deprecated - public DocumentType(String name, StructDataType headerType, - StructDataType bodyType, Set<String> importedFieldNames) { - this(name, headerType, importedFieldNames); - } - public DocumentType(String name, Set<String> importedFieldNames) { - this(name, createHeaderStructType(name), importedFieldNames); + this(name, createContentStructType(name), importedFieldNames); } - private static StructDataType createHeaderStructType(String name) { + private static StructDataType createContentStructType(String name) { return new StructDataType(name + ".header"); } @Override public DocumentType clone() { DocumentType type = (DocumentType) super.clone(); - type.headerType = headerType.clone(); + type.contentStructType = contentStructType.clone(); type.inherits = new ArrayList<>(inherits.size()); type.inherits.addAll(inherits); return type; @@ -136,7 +118,7 @@ public class DocumentType extends StructuredDataType { * @return a struct describing the document fields. */ public StructDataType contentStruct() { - return headerType; + return contentStructType; } /** @@ -185,12 +167,6 @@ public class DocumentType extends StructuredDataType { } } - /** @deprecated use contentStruct instead */ - @Deprecated // TODO: Remove on Vespa 8 - public StructDataType getHeaderType() { - return contentStruct(); - } - @Override protected void register(DocumentTypeManager manager, List<DataType> seenTypes) { seenTypes.add(this); @@ -200,17 +176,17 @@ public class DocumentType extends StructuredDataType { } } // Get parent fields into fields specified in this type - StructDataType header = headerType.clone(); + StructDataType header = contentStructType.clone(); header.clearFields(); for (Field field : getAllUniqueFields()) { header.addField(field); } - headerType.assign(header); + contentStructType.assign(header); - if (!seenTypes.contains(headerType)) { - headerType.register(manager, seenTypes); + if (!seenTypes.contains(contentStructType)) { + contentStructType.register(manager, seenTypes); } manager.registerSingleType(this); } @@ -240,7 +216,7 @@ public class DocumentType extends StructuredDataType { if (isRegistered()) { throw new IllegalStateException("You cannot add fields to a document type that is already registered."); } - headerType.addField(field); + contentStructType.addField(field); } // Do not use, public only for testing @@ -280,24 +256,11 @@ public class DocumentType extends StructuredDataType { throw new IllegalStateException("You cannot add fields to a document type that is already registered."); } Field field = new Field(name, type); - headerType.addField(field); + contentStructType.addField(field); return field; } /** - * Adds a new header field to this document type and returns the new field object - * - * @param name The name of the field to add - * @param type The datatype of the field to add - * @return The field created - * TODO Fix searchdefinition so that exception can be thrown if filed is already registerd - */ - @Deprecated - public Field addHeaderField(String name, DataType type) { - return addField(name, type); - } - - /** * Adds a document to the inherited document types of this. * If this type is already directly inherited, nothing is done * @@ -324,8 +287,8 @@ public class DocumentType extends StructuredDataType { inherits.add(type); for (var field : type.getAllUniqueFields()) { - if (! headerType.hasField(field)) { - headerType.addField(field); + if (! contentStructType.hasField(field)) { + contentStructType.addField(field); } } } @@ -391,7 +354,7 @@ public class DocumentType extends StructuredDataType { * @return returns the matching field, or null if not found */ public Field getField(String name) { - Field field = headerType.getField(name); + Field field = contentStructType.getField(name); if (field == null && !isRegistered()) { for (DocumentType inheritedType : inherits) { field = inheritedType.getField(name); @@ -403,7 +366,7 @@ public class DocumentType extends StructuredDataType { @Override public Field getField(int id) { - Field field = headerType.getField(id); + Field field = contentStructType.getField(id); if (field == null && !isRegistered()) { for (DocumentType inheritedType : inherits) { field = inheritedType.getField(id); @@ -424,7 +387,7 @@ public class DocumentType extends StructuredDataType { } public int getFieldCount() { - return headerType.getFieldCount(); + return contentStructType.getFieldCount(); } public Set<String> getImportedFieldNames() { @@ -445,7 +408,7 @@ public class DocumentType extends StructuredDataType { if (isRegistered()) { throw new IllegalStateException("You cannot remove fields from a document type that is already registered."); } - Field field = headerType.removeField(name); + Field field = contentStructType.removeField(name); if (field == null) { for (DocumentType inheritedType : inherits) { field = inheritedType.removeField(name); @@ -469,7 +432,7 @@ public class DocumentType extends StructuredDataType { collection.addAll(type.getFields()); } - collection.addAll(headerType.getFields()); + collection.addAll(contentStructType.getFields()); return ImmutableList.copyOf(collection); } @@ -519,14 +482,14 @@ public class DocumentType extends StructuredDataType { * @return An iterator for iterating the fields in this documenttype. */ public Iterator<Field> fieldIteratorThisTypeOnly() { - return headerType.getFields().iterator(); + return contentStructType.getFields().iterator(); } public boolean equals(Object o) { if (!(o instanceof DocumentType)) return false; DocumentType other = (DocumentType) o; // Ignore whether one of them have added inheritance to super Document.0 type - if (super.equals(o) && headerType.equals(other.headerType)) { + if (super.equals(o) && contentStructType.equals(other.contentStructType)) { if ((inherits.size() > 1 || other.inherits.size() > 1) || (inherits.size() == 1 && other.inherits.size() == 1)) { return inherits.equals(other.inherits); @@ -538,7 +501,7 @@ public class DocumentType extends StructuredDataType { } public int hashCode() { - return super.hashCode() + headerType.hashCode() + inherits.hashCode(); + return super.hashCode() + contentStructType.hashCode() + inherits.hashCode(); } @Override @@ -553,7 +516,7 @@ public class DocumentType extends StructuredDataType { @Override public void visitMembers(ObjectVisitor visitor) { super.visitMembers(visitor); - visitor.visit("headertype", headerType); + visitor.visit("headertype", contentStructType); visitor.visit("inherits", inherits); } } diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManager.java b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java index af7c267c8c4..ee9366285a9 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManager.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java @@ -40,8 +40,6 @@ public class DocumentTypeManager { private final static Logger log = Logger.getLogger(DocumentTypeManager.class.getName()); - @SuppressWarnings("removal") // TODO Vespa 8: remove - private ConfigSubscriber subscriber; // *Configured data types* (not built-in/primitive) indexed by their id // @@ -67,22 +65,12 @@ public class DocumentTypeManager { DocumentTypeManagerConfigurer.configureNewManager(config, this); } - public void assign(DocumentTypeManager other) { + void internalAssign(DocumentTypeManager other) { dataTypes = other.dataTypes; documentTypes = other.documentTypes; annotationTypeRegistry = other.annotationTypeRegistry; } - /** - * For testing, use fromFile factory method instead - * @deprecated //TODO Will be package-private or removed on Vespa 8 - */ - @Deprecated - public DocumentTypeManager configure(String configId) { - subscriber = DocumentTypeManagerConfigurer.configure(this, configId); - return this; - } - /** Only for unit tests */ public static DocumentTypeManager fromFile(String fileName) { var manager = new DocumentTypeManager(); @@ -115,7 +103,7 @@ public class DocumentTypeManager { } } - public boolean hasDataType(String name) { + boolean hasDataTypeInternal(String name) { if (name.startsWith("tensor(")) return true; // built-in dynamic: Always present for (DataType type : dataTypes.values()) { if (type.getName().equalsIgnoreCase(name)) { @@ -126,30 +114,6 @@ public class DocumentTypeManager { } /** - * @deprecated //TODO Will be package-private or removed on Vespa 8 - */ - @Deprecated - public boolean hasDataType(int code) { - if (code == DataType.tensorDataTypeCode) return true; // built-in dynamic: Always present - return dataTypes.containsKey(code); - } - - /** - * @deprecated //TODO Will be package-private or removed on Vespa 8 - * Use constants and factories in DataType instead. - * For structs, use getStructType() in DocumentType. - * For annotation payloads, use getDataType() in AnnotationType. - **/ - @Deprecated - public DataType getDataType(String name) { - var type = getDataTypeInternal(name); - if (type == null) { - throw new IllegalArgumentException("No datatype named " + name); - } - return type; - } - - /** * For internal use only, avoid whenever possible. * Use constants and factories in DataType instead. * For structs, use getStructType() in DocumentType. @@ -187,10 +151,11 @@ public class DocumentTypeManager { } /** - * @deprecated //TODO Will be package-private or removed on Vespa 8 + * Return a data type instance + * + * @param code the code of the data type to return, which must be either built in or present in this manager */ - @Deprecated - public DataType getDataType(int code) { return getDataType(code, ""); } + DataType getDataTypeByCode(int code) { return getDataTypeByCode(code, ""); } /** * Return a data type instance @@ -198,11 +163,8 @@ public class DocumentTypeManager { * @param code the code of the data type to return, which must be either built in or present in this manager * @param detailedType detailed type information, or the empty string if none * @return the appropriate DataType instance - * - * @deprecated //TODO Will be package-private or removed on Vespa 8 */ - @Deprecated - public DataType getDataType(int code, String detailedType) { + DataType getDataTypeByCode(int code, String detailedType) { if (code == DataType.tensorDataTypeCode) // built-in dynamic return new TensorDataType(TensorType.fromSpec(detailedType)); @@ -219,18 +181,6 @@ public class DocumentTypeManager { } /** - * @deprecated //TODO Will be package-private or removed on Vespa 8 - */ - @SuppressWarnings("deprecation") - @Deprecated - DataType getDataTypeAndReturnTemporary(int code, String detailedType) { - if (hasDataType(code)) { - return getDataType(code, detailedType); - } - return new TemporaryDataType(code, detailedType); - } - - /** * Register a data type of any sort, including document types. * @param type The datatype to register * TODO Give unique ids to document types @@ -244,15 +194,8 @@ public class DocumentTypeManager { * * @param type The datatype to register */ - @SuppressWarnings("deprecation") void registerSingleType(DataType type) { if (type instanceof TensorDataType) return; // built-in dynamic: Created on the fly - if (type instanceof TemporaryDataType) { - throw new IllegalArgumentException("TemporaryDataType no longer supported: " + type); - } - if (type instanceof TemporaryStructuredDataType) { - throw new IllegalArgumentException("TemporaryStructuredDataType no longer supported: " + type); - } if (dataTypes.containsKey(type.getId())) { DataType existingType = dataTypes.get(type.getId()); if ((existingType == type) || existingType.equals(type)) { @@ -307,6 +250,15 @@ public class DocumentTypeManager { return documentTypes.get(new DataTypeName(name)); } + /** + * Convenience method + * @param name the name of a document type + * @return returns true if a document type having this name is registered in this manager + */ + public boolean hasDocumentType(String name) { + return (getDocumentType(name) != null); + } + final public Document createDocument(GrowableByteBuffer buf) { DocumentDeserializer data = DocumentDeserializerFactory.create6(this, buf); return new Document(data); @@ -340,11 +292,8 @@ public class DocumentTypeManager { /** * Clears the DocumentTypeManager. After this operation, * only the default document type and data types are available. - * - * @deprecated //TODO Will be package-private or removed on Vespa 8 */ - @Deprecated - public void clear() { + void internalClear() { documentTypes.clear(); dataTypes.clear(); registerDefaultDataTypes(); @@ -354,11 +303,4 @@ public class DocumentTypeManager { return annotationTypeRegistry; } - /** - * @deprecated //TODO Will be package-private or removed on Vespa 8 - */ - @Deprecated - public void shutdown() { - if (subscriber!=null) subscriber.close(); - } } diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index 76dac2bdc20..d6833a482f1 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -1,30 +1,28 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document; -import com.yahoo.compress.CompressionType; import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.document.internal.GeoPosType; -import java.util.logging.Level; +import com.yahoo.tensor.TensorType; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; -import java.util.Map; import java.util.HashSet; +import java.util.Map; import java.util.Set; +import java.util.function.Supplier; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; -import java.util.function.Supplier; -import com.yahoo.tensor.TensorType; /** * Configures the Vespa document manager from a config id. * * @author Einar M R Rosenvinge */ -@SuppressWarnings("removal") // TODO Vespa 8: remove public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSubscriber<DocumentmanagerConfig> { private final static Logger log = Logger.getLogger(DocumentTypeManagerConfigurer.class.getName()); @@ -35,16 +33,6 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub this.managerToConfigure = manager; } - /** Deprecated and will go away on Vespa 8 */ - @Deprecated - public static CompressionType toCompressorType(DocumentmanagerConfig.Datatype.Structtype.Compresstype.Enum value) { - switch (value) { - case NONE: return CompressionType.NONE; - case LZ4: return CompressionType.LZ4; - case UNCOMPRESSABLE: return CompressionType.INCOMPRESSIBLE; - } - throw new IllegalArgumentException("Compression type " + value + " is not supported"); - } /** * <p>Makes the DocumentTypeManager subscribe on its config.</p> * @@ -174,14 +162,13 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub return type; } - @SuppressWarnings("deprecation") private DataType getOrCreateType(int id) { if (typesById.containsKey(id)) { return typesById.get(id); } var config = configMap.remove(id); if (config == null) { - return manager.getDataType(id); + return manager.getDataTypeByCode(id); } assert(id == config.id()); for (var o : config.arraytype()) { @@ -207,7 +194,6 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub } } - @SuppressWarnings("deprecation") private void fillStructs(DocumentmanagerConfig config) { for (var thisDataType : config.datatype()) { for (var struct : thisDataType.structtype()) { @@ -226,7 +212,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub } DataType fieldType = typesById.get(field.datatype()); if (fieldType == null) { - fieldType = manager.getDataType(field.datatype(), field.detailedtype()); + fieldType = manager.getDataTypeByCode(field.datatype(), field.detailedtype()); } if (field.id().size() == 1) { type.addField(new Field(field.name(), field.id().get(0).id(), fieldType)); @@ -274,7 +260,6 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub } } - @SuppressWarnings("deprecation") private void apply(DocumentmanagerConfig config) { splitConfig(config); setupAnnotationTypesWithoutPayloads(config); @@ -312,11 +297,10 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub } } - @SuppressWarnings("deprecation") private void addAnnotationTypePayloads(DocumentmanagerConfig config) { for (DocumentmanagerConfig.Annotationtype annType : config.annotationtype()) { AnnotationType annotationType = manager.getAnnotationTypeRegistry().getType(annType.id()); - DataType payload = manager.getDataType(annType.datatype(), ""); + DataType payload = manager.getDataTypeByCode(annType.datatype(), ""); if (! payload.equals(DataType.NONE)) { annotationType.setDataType(payload); } @@ -449,7 +433,6 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub } void createEmptyStructs() { - String docName = docTypeConfig.name(); for (var typeconf : docTypeConfig.structtype()) { if (isPositionStruct(typeconf)) { int geoVersion = usev8geopositions ? 8 : 7; @@ -632,7 +615,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub if (this.managerToConfigure.getDataTypes().size() != defaultTypeCount) { log.log(Level.FINE, "Live document config overwritten with new config."); } - managerToConfigure.assign(manager); + managerToConfigure.internalAssign(manager); } } diff --git a/document/src/main/java/com/yahoo/document/Field.java b/document/src/main/java/com/yahoo/document/Field.java index a45afea6747..e2204ada7e5 100644 --- a/document/src/main/java/com/yahoo/document/Field.java +++ b/document/src/main/java/com/yahoo/document/Field.java @@ -16,7 +16,6 @@ import java.io.Serializable; * @author Thomas Gundersen * @author bratseth */ -// TODO: Remove header/body concept on Vespa 8 public class Field extends FieldBase implements FieldSet, Comparable, Serializable { protected DataType dataType; @@ -27,14 +26,9 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab * Creates a new field. * * @param name The name of the field + * @param id Serialized ID * @param dataType The datatype of the field - * @param isHeader Whether this is a "header" field or a "content" field - * (true = "header"). */ - @Deprecated - public Field(String name, int id, DataType dataType, boolean isHeader) { - this(name, id, dataType); - } public Field(String name, int id, DataType dataType) { super(name); this.fieldId = id; @@ -53,15 +47,8 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab * * @param name The name of the field * @param dataType The datatype of the field - * @param isHeader Whether this is a "header" field or a "content" field - * (true = "header"). * @param owner the owning document (used to check for id collisions) */ - @Deprecated - public Field(String name, DataType dataType, boolean isHeader, DocumentType owner) { - this(name, dataType, owner); - } - public Field(String name, DataType dataType, DocumentType owner) { this(name, 0, dataType); this.fieldId = calculateIdV7(owner); @@ -69,20 +56,7 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab } /** - * Creates a new field. - * - * @param name The name of the field - * @param dataType The datatype of the field - * @param isHeader Whether this is a "header" field or a "content" field - * (true = "header"). - */ - @Deprecated - public Field(String name, DataType dataType, boolean isHeader) { - this(name, dataType); - } - - /** - * Constructor for <b>header</b> fields + * Constructor for normal fields * * @param name The name of the field * @param dataType The datatype of the field @@ -183,12 +157,6 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab forcedId = false; } - /** Returns the numeric ID used to represent this field when serialized */ - @Deprecated - public final int getId(int version) { - return getId(); - } - public final int getId() { return fieldId; } @@ -201,16 +169,6 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab return forcedId; } - /** @deprecated this has no longer any semantic meaning as this is no longer an aspect with a field */ - @Deprecated // TODO: Remove on Vespa 8 - public boolean isHeader() { - return true; - } - - /** @deprecated this has no longer any semantic meaning as this is no longer an aspect with a field */ - @Deprecated // TODO: Remove on Vespa 8 - public void setHeader(boolean header) { } - /** Two fields are equal if they have the same name and the same data type */ @Override public boolean equals(Object o) { diff --git a/document/src/main/java/com/yahoo/document/MapDataType.java b/document/src/main/java/com/yahoo/document/MapDataType.java index 97a533def8d..457ae084107 100644 --- a/document/src/main/java/com/yahoo/document/MapDataType.java +++ b/document/src/main/java/com/yahoo/document/MapDataType.java @@ -52,26 +52,6 @@ public class MapDataType extends DataType { return valueType; } - /** - * WARNING! Do not use! Only to be used by config system! - * Sets the key type of this MapDataType. - * @deprecated // TODO Vespa 8 remove - */ - @Deprecated(forRemoval = true, since = "7") - public void setKeyType(DataType keyType) { - this.keyType = keyType; - } - - /** - * WARNING! Do not use! Only to be used by config system! - * Sets the value type of this MapDataType. - * @deprecated // TODO Vespa 8 remove - */ - @Deprecated(forRemoval = true, since = "7") - public void setValueType(DataType valueType) { - this.valueType = valueType; - } - @Override public MapFieldValue createFieldValue() { return new MapFieldValue(this); diff --git a/document/src/main/java/com/yahoo/document/PositionDataType.java b/document/src/main/java/com/yahoo/document/PositionDataType.java index f81dffa3837..d4160fc58c7 100644 --- a/document/src/main/java/com/yahoo/document/PositionDataType.java +++ b/document/src/main/java/com/yahoo/document/PositionDataType.java @@ -99,16 +99,6 @@ public final class PositionDataType { return fieldName + "_zcurve"; } - public static String getPositionSummaryFieldName(String fieldName) { - // TODO for Vespa 8, consider renaming to _position to use a field name that is actually legal - return fieldName + ".position"; - } - - public static String getDistanceSummaryFieldName(String fieldName) { - // TODO for Vespa 8, consider renaming to _distance to use a field name that is actually legal - return fieldName + ".distance"; - } - private static StructDataType newInstance() { StructDataType ret = new StructDataType(STRUCT_NAME); ret.addField(new Field(FIELD_X, DataType.INT)); diff --git a/document/src/main/java/com/yahoo/document/ReferenceDataType.java b/document/src/main/java/com/yahoo/document/ReferenceDataType.java index c3b5f6590b6..8cf6b665acf 100644 --- a/document/src/main/java/com/yahoo/document/ReferenceDataType.java +++ b/document/src/main/java/com/yahoo/document/ReferenceDataType.java @@ -22,16 +22,6 @@ public class ReferenceDataType extends DataType { this((StructuredDataType)targetType, id); } - /** - * Constructor used when building a multi document type model where the concrete instance - * of the target document type might not yet be known. The temporary data type should be - * replaced later using setTargetType(). - */ - @SuppressWarnings("deprecation") - public ReferenceDataType(TemporaryStructuredDataType temporaryTargetType, int id) { - this((StructuredDataType) temporaryTargetType, id); - } - private ReferenceDataType(StructuredDataType targetType, int id) { super(buildTypeName(targetType), id); this.targetType = targetType; @@ -52,34 +42,8 @@ public class ReferenceDataType extends DataType { return new ReferenceDataType(targetType); } - /** - * Creates a new type where the numeric ID is based on the hash of targetType - */ - @SuppressWarnings("deprecation") - public static ReferenceDataType createWithInferredId(TemporaryStructuredDataType targetType) { - return new ReferenceDataType(targetType); - } - public StructuredDataType getTargetType() { return targetType; } - /** - * Overrides the stored temporary data type with a concrete StructuredDataType instance. Should only - * be invoked from configuration or model code when resolving temporary types. - * - * @throws IllegalStateException if the previously stored target type is already a concrete - * instance (not TemporaryStructuredDataType). - */ - @SuppressWarnings("deprecation") - public void setTargetType(StructuredDataType targetType) { - if (! (this.targetType instanceof TemporaryStructuredDataType)) { - throw new IllegalStateException(String.format( - "Unexpected attempt to replace already concrete target " + - "type in ReferenceDataType instance (type is '%s')", this.targetType.getName())); - } - this.targetType = targetType; - setName(buildTypeName(targetType)); - } - @Override public ReferenceFieldValue createFieldValue() { return new ReferenceFieldValue(this); diff --git a/document/src/main/java/com/yahoo/document/TemporaryDataType.java b/document/src/main/java/com/yahoo/document/TemporaryDataType.java deleted file mode 100644 index 71f36ecde90..00000000000 --- a/document/src/main/java/com/yahoo/document/TemporaryDataType.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document; - -import com.yahoo.document.datatypes.FieldValue; - -/** - * @author Einar M R Rosenvinge - * @deprecated will be removed soon - */ -@Deprecated -class TemporaryDataType extends DataType { - - private final String detailedType; - - TemporaryDataType(int dataTypeId, String detailedType) { - super("temporary_" + dataTypeId, dataTypeId); - this.detailedType = detailedType; - } - - @Override - public FieldValue createFieldValue() { - return null; - } - - @Override - public Class getValueClass() { - return null; - } - - @Override - public boolean isValueCompatible(FieldValue value) { - return false; - } - - String getDetailedType() { return detailedType; } - -} diff --git a/document/src/main/java/com/yahoo/document/TemporaryStructuredDataType.java b/document/src/main/java/com/yahoo/document/TemporaryStructuredDataType.java deleted file mode 100644 index 865310e7009..00000000000 --- a/document/src/main/java/com/yahoo/document/TemporaryStructuredDataType.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document; - -/** - * Internal class, DO NOT USE!! - * Only public because it must be used from com.yahoo.searchdefinition.parser. - * - * @deprecated will be removed soon - * @author Einar M R Rosenvinge - */ -@Deprecated // TODO: Remove on Vespa 8 -public class TemporaryStructuredDataType extends StructDataType { - - TemporaryStructuredDataType(String name) { - super(name); - } - - private TemporaryStructuredDataType(int id) { - super(id, "temporary_struct_" + id); - } - - public static TemporaryStructuredDataType create(String name) { - return new TemporaryStructuredDataType(name); - } - - public static TemporaryStructuredDataType createById(int id) { - return new TemporaryStructuredDataType(id); - } - - @Override - protected void setName(String name) { - super.setName(name); - setId(createId(getName())); - } - -} diff --git a/document/src/main/java/com/yahoo/document/WeightedSetDataType.java b/document/src/main/java/com/yahoo/document/WeightedSetDataType.java index b21f059bd7d..04ec779bc16 100644 --- a/document/src/main/java/com/yahoo/document/WeightedSetDataType.java +++ b/document/src/main/java/com/yahoo/document/WeightedSetDataType.java @@ -43,16 +43,6 @@ public class WeightedSetDataType extends CollectionDataType { } } - /* - * @deprecated // TODO remove on Vespa 8 - * Do not use - use one of the constructors above. - * Note: ignores typeName argument. - */ - @Deprecated - public WeightedSetDataType(String typeName, int code, DataType nestedType, boolean createIfNonExistent, boolean removeIfZero) { - this(nestedType, createIfNonExistent, removeIfZero, code); - } - @Override public WeightedSetDataType clone() { return (WeightedSetDataType) super.clone(); diff --git a/document/src/main/java/com/yahoo/document/datatypes/Array.java b/document/src/main/java/com/yahoo/document/datatypes/Array.java index 672690bafad..790cc5d4cde 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/Array.java +++ b/document/src/main/java/com/yahoo/document/datatypes/Array.java @@ -220,12 +220,10 @@ public final class Array<T extends FieldValue> extends CollectionFieldValue<T> i return values.addAll(index, c); } - @SuppressWarnings("deprecation") public boolean retainAll(Collection<?> c) { return values.retainAll(c); } - @SuppressWarnings("deprecation") public boolean removeAll(Collection<?> c) { return values.removeAll(c); } @@ -234,12 +232,10 @@ public final class Array<T extends FieldValue> extends CollectionFieldValue<T> i return values.get(index); } - @SuppressWarnings("deprecation") public int indexOf(Object o) { return values.indexOf(o); } - @SuppressWarnings("deprecation") public int lastIndexOf(Object o) { return values.lastIndexOf(o); } @@ -256,7 +252,6 @@ public final class Array<T extends FieldValue> extends CollectionFieldValue<T> i return values.remove(index); } - @SuppressWarnings("deprecation") public T set(int index, T o) { verifyElementCompatibility(o); T fval = values.set(index, o); @@ -512,7 +507,6 @@ public final class Array<T extends FieldValue> extends CollectionFieldValue<T> i }; } - @SuppressWarnings("deprecation") public List<E> subList(int fromIndex, int toIndex) { return new ListWrapper<E>(myvalues.subList(fromIndex, toIndex)); } diff --git a/document/src/main/java/com/yahoo/document/datatypes/Struct.java b/document/src/main/java/com/yahoo/document/datatypes/Struct.java index b47dbe69199..d522e8cda44 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/Struct.java +++ b/document/src/main/java/com/yahoo/document/datatypes/Struct.java @@ -65,22 +65,6 @@ public class Struct extends StructuredFieldValue { return this.version; } - /** @deprecated Will go away on Vespa 8 */ - @Deprecated - public com.yahoo.compress.CompressionType getCompressionType() { - return com.yahoo.compress.CompressionType.NONE; - } - - /** @deprecated Will go away on Vespa 8 */ - @Deprecated - public int getCompressionLevel() { return 9; } - - /** @deprecated Will go away on Vespa 8 */ - @Deprecated - public float getCompressionThreshold() { - return .95f; - } - @Override public Struct clone() { Struct struct = (Struct) super.clone(); diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java index a9d34fc78d8..d33cc8078dd 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java @@ -183,11 +183,6 @@ public class JsonSerializationHelper { public static void serializeStructField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, Struct value) { DataType dt = value.getDataType(); - // TODO remove in Vespa 8: - if (dt == PositionDataType.INSTANCE) { - serializeString(generator, field, PositionDataType.renderAsString(value)); - return; - } if (dt instanceof GeoPosType) { var gpt = (GeoPosType)dt; if (gpt.renderJsonAsVespa8()) { diff --git a/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java b/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java index d6166f8fcbe..55b94941994 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java @@ -90,7 +90,6 @@ public class ComparisonNode implements ExpressionNode { return null; } - @SuppressWarnings("deprecation") private BucketSet compare(BucketIdFactory factory, IdNode id, LiteralNode literal, String operator) { String field = id.getField(); Object value = literal.getValue(); diff --git a/document/src/main/java/com/yahoo/document/select/rule/DocumentTypeNode.java b/document/src/main/java/com/yahoo/document/select/rule/DocumentTypeNode.java index 57bb809de40..b161f2190f0 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/DocumentTypeNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/DocumentTypeNode.java @@ -46,27 +46,7 @@ public class DocumentTypeNode implements ExpressionNode { } private Object evaluate(DocumentOperation op) { - // TODO Vespa 8: Uncomment the following line and remove the legacy one - // return op.getId().getDocType().equals(type) ? op : false; - return legacyEvaluate(op); - } - - private Object legacyEvaluate(DocumentOperation op) { - DocumentType doct; - if (op instanceof DocumentPut) { - doct = ((DocumentPut) op).getDocument().getDataType(); - } else if (op instanceof DocumentUpdate) { - doct = ((DocumentUpdate) op).getDocumentType(); - } else if (op instanceof DocumentRemove) { - DocumentRemove removeOp = (DocumentRemove) op; - return (removeOp.getId().getDocType().equals(type) ? op : Boolean.FALSE); - } else if (op instanceof DocumentGet) { - DocumentGet getOp = (DocumentGet) op; - return (getOp.getId().getDocType().equals(type) ? op : Boolean.FALSE); - } else { - throw new IllegalStateException("Document class '" + op.getClass().getName() + "' is not supported."); - } - return doct.isA(this.type) ? op : Boolean.FALSE; + return op.getId().getDocType().equals(type) ? op : false; } public void accept(Visitor visitor) { diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java index 4afab9233c6..2796609543f 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java @@ -2,8 +2,6 @@ package com.yahoo.document.serialization; import com.yahoo.collections.Tuple2; -import com.yahoo.compress.CompressionType; -import com.yahoo.compress.Compressor; import com.yahoo.document.annotation.AlternateSpanList; import com.yahoo.document.annotation.Annotation; import com.yahoo.document.annotation.AnnotationReference; @@ -69,7 +67,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import static com.yahoo.text.Utf8.calculateStringPositions; @@ -80,8 +77,7 @@ import static com.yahoo.text.Utf8.calculateStringPositions; */ public class VespaDocumentDeserializer6 extends BufferSerializer implements DocumentDeserializer { - private final Compressor compressor = new Compressor(); - private DocumentTypeManager manager; + private final DocumentTypeManager manager; private short version; private List<SpanNode> spanNodes; private List<Annotation> annotations; @@ -99,7 +95,6 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu read(null, document); } - @SuppressWarnings("deprecation") public void read(FieldBase field, Document doc) { // Verify that we have correct version version = getShort(null); @@ -264,21 +259,7 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu } int dataSize = getInt(null); - byte comprCode = getByte(null); - CompressionType compression = CompressionType.valueOf(comprCode); - - int uncompressedSize = 0; - if (compression != CompressionType.NONE && - compression != CompressionType.INCOMPRESSIBLE) - { - // uncompressedsize (full size of FIELDS only, after decompression) - long pSize = getInt2_4_8Bytes(null); - //TODO: Look into how to support data segments larger than INT_MAX bytes - if (pSize > Integer.MAX_VALUE) { - throw new DeserializationException("Uncompressed size of data block is too large."); - } - uncompressedSize = (int) pSize; - } + byte ignoredComprCode = getByte(null); int numberOfFields = getInt1_4Bytes(null); @@ -290,14 +271,12 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu // save a reference to the big buffer we're reading from: GrowableByteBuffer bigBuf = buf; - - byte[] destination = compressor.decompress(compression, getBuf().array(), position(), uncompressedSize, Optional.of(dataSize)); - + GrowableByteBuffer thisStructOnly = GrowableByteBuffer.wrap(getBuf().array(), position(), dataSize); // set position in original buffer to after data position(position() + dataSize); // for a while: deserialize from this buffer instead: - buf = GrowableByteBuffer.wrap(destination); + buf = thisStructOnly; s.clear(); StructDataType type = s.getDataType(); @@ -326,21 +305,7 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu } int dataSize = getInt(null); - byte comprCode = getByte(null); - CompressionType compression = CompressionType.valueOf(comprCode); - - int uncompressedSize = 0; - if (compression != CompressionType.NONE && - compression != CompressionType.INCOMPRESSIBLE) - { - // uncompressedsize (full size of FIELDS only, after decompression) - long pSize = getInt2_4_8Bytes(null); - //TODO: Look into how to support data segments larger than INT_MAX bytes - if (pSize > Integer.MAX_VALUE) { - throw new DeserializationException("Uncompressed size of data block is too large."); - } - uncompressedSize = (int) pSize; - } + byte unusedComprCode = getByte(null); int numberOfFields = getInt1_4Bytes(null); @@ -352,14 +317,13 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu // save a reference to the big buffer we're reading from: GrowableByteBuffer bigBuf = buf; - - byte[] destination = compressor.decompress(compression, getBuf().array(), position(), uncompressedSize, Optional.of(dataSize)); + GrowableByteBuffer thisStructOnly = GrowableByteBuffer.wrap(getBuf().array(), position(), dataSize); // set position in original buffer to after data position(position() + dataSize); // for a while: deserialize from this buffer instead: - buf = GrowableByteBuffer.wrap(destination); + buf = thisStructOnly; StructDataType priType = target.getDataType().contentStruct(); @@ -614,7 +578,7 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu DocumentType docType = manager.getDocumentType(new DataTypeName(docTypeName)); if (docType == null) { throw new DeserializationException("No known document type with name " + - new Utf8String(docTypeName).toString()); + new Utf8String(docTypeName)); } return docType; } diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java index 8f5a0f6acd5..2de345b6e35 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java @@ -84,7 +84,6 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume write(new Field(doc.getDataType().getName(), 0, doc.getDataType()), doc); } - @SuppressWarnings("deprecation") public void write(FieldBase field, Document doc) { buf.putShort(Document.SERIALIZED_VERSION); @@ -484,7 +483,7 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume write(tree.getRoot()); { //add all annotations to temporary list and sort it, to get predictable serialization - List<Annotation> tmpAnnotationList = new ArrayList<Annotation>(tree.numAnnotations()); + List<Annotation> tmpAnnotationList = new ArrayList<>(tree.numAnnotations()); for (Annotation annotation : tree) { tmpAnnotationList.add(annotation); } diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java index 082a88bb000..aeb4bbe79b1 100644 --- a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java +++ b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java @@ -41,12 +41,11 @@ import java.util.Map; * * @author Steinar Knutsen */ -@SuppressWarnings("deprecation") public final class XmlDocumentWriter implements DocumentWriter { private final String indent; private XmlStream buffer; - private Deque<FieldBase> optionalWrapperMarker = new ArrayDeque<FieldBase>(); + private final Deque<FieldBase> optionalWrapperMarker = new ArrayDeque<>(); public static XmlDocumentWriter createWriter(String indent) { return new XmlDocumentWriter(indent); diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java b/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java index 1bb91b91762..899ef1f9195 100644 --- a/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java @@ -3,7 +3,21 @@ package com.yahoo.document.serialization; import com.yahoo.document.Document; import com.yahoo.document.Field; -import com.yahoo.document.datatypes.*; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.BoolFieldValue; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.Float16FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.datatypes.WeightedSet; import com.yahoo.text.Utf8; import java.util.Base64; @@ -46,7 +60,6 @@ public class XmlSerializationHelper { xml.addContent(b.toString()); } - @SuppressWarnings("deprecation") public static void printDocumentXml(Document doc, XmlStream xml) { xml.addAttribute("documenttype", doc.getDataType().getName()); xml.addAttribute("documentid", doc.getId()); @@ -121,13 +134,6 @@ public class XmlSerializationHelper { } } - private static boolean containsNonPrintableCharactersByte(final byte[] buffer) { - for (byte b : buffer) { - if (b < 32 && (b != 9 && b != 10 && b != 13)) return true; - } - return false; - } - private static boolean containsNonPrintableCharactersString(final CharSequence buffer) { for (int i = 0; i < buffer.length(); i++) { char b = buffer.charAt(i); diff --git a/document/src/main/java/com/yahoo/document/update/FieldUpdate.java b/document/src/main/java/com/yahoo/document/update/FieldUpdate.java index c2a3dd824a5..b2992bf4988 100644 --- a/document/src/main/java/com/yahoo/document/update/FieldUpdate.java +++ b/document/src/main/java/com/yahoo/document/update/FieldUpdate.java @@ -75,11 +75,6 @@ public class FieldUpdate { addValueUpdates(valueUpdates); } - @Deprecated - public FieldUpdate(DocumentUpdateReader reader, DocumentType type, int serializationVersion) { - this(reader, type); - } - public FieldUpdate(DocumentUpdateReader reader, DocumentType type) { documentType = type; reader.read(this); diff --git a/document/src/test/document/serializecpp-lz4-level9.dat b/document/src/test/document/serializecpp-lz4-level9.dat Binary files differdeleted file mode 100644 index 1dffaa2d7a7..00000000000 --- a/document/src/test/document/serializecpp-lz4-level9.dat +++ /dev/null diff --git a/document/src/test/java/com/yahoo/document/DataTypeNameTestCase.java b/document/src/test/java/com/yahoo/document/DataTypeNameTestCase.java index 0d6c50e3e4f..efdb37b90d4 100644 --- a/document/src/test/java/com/yahoo/document/DataTypeNameTestCase.java +++ b/document/src/test/java/com/yahoo/document/DataTypeNameTestCase.java @@ -12,7 +12,6 @@ import static org.junit.Assert.assertFalse; public class DataTypeNameTestCase { @Test - @SuppressWarnings("deprecation") public void requireThatAccessorsWork() { DataTypeName name = new DataTypeName("foo"); assertEquals("foo", name.getName()); diff --git a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java index db877bb9712..1a5016fb155 100644 --- a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document; -import com.yahoo.compress.CompressionType; import com.yahoo.document.annotation.AbstractTypesTest; import com.yahoo.document.datatypes.Array; import com.yahoo.document.datatypes.BoolFieldValue; @@ -13,7 +12,10 @@ import com.yahoo.document.datatypes.LongFieldValue; import com.yahoo.document.datatypes.Raw; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.WeightedSet; -import com.yahoo.document.serialization.*; +import com.yahoo.document.serialization.DocumentDeserializer; +import com.yahoo.document.serialization.DocumentDeserializerFactory; +import com.yahoo.document.serialization.DocumentSerializer; +import com.yahoo.document.serialization.DocumentSerializerFactory; import com.yahoo.io.GrowableByteBuffer; import org.junit.Test; @@ -27,9 +29,10 @@ import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Tests serialization of all versions. @@ -58,7 +61,6 @@ import static org.junit.Assert.assertTrue; public class DocumentSerializationTestCase extends AbstractTypesTest { @Test - @SuppressWarnings("deprecation") public void testSerializationAllVersions() throws IOException { DocumentType docInDocType = new DocumentType("docindoc"); @@ -93,7 +95,7 @@ public class DocumentSerializationTestCase extends AbstractTypesTest { doc.setFieldValue("intfield", 5); doc.setFieldValue("floatfield", -9.23); doc.setFieldValue("stringfield", "This is a string."); - doc.setFieldValue("longfield", new LongFieldValue(398420092938472983l)); + doc.setFieldValue("longfield", new LongFieldValue(398420092938472983L)); doc.setFieldValue("doublefield", new DoubleFieldValue(98374532.398820)); doc.setFieldValue("bytefield", new ByteFieldValue(254)); doc.setFieldValue("boolfield", new BoolFieldValue(true)); @@ -121,8 +123,8 @@ public class DocumentSerializationTestCase extends AbstractTypesTest { class TestDoc { - String testFile; - int version; + final String testFile; + final int version; TestDoc(String testFile, int version) { this.testFile = testFile; @@ -134,10 +136,8 @@ public class DocumentSerializationTestCase extends AbstractTypesTest { List<TestDoc> tests = new ArrayList<>(); tests.add(new TestDoc(path + "document-java-currentversion-uncompressed.dat", Document.SERIALIZED_VERSION)); - tests.add(new TestDoc(path + "document-java-currentversion-lz4-9.dat", Document.SERIALIZED_VERSION)); tests.add(new TestDoc(path + "document-java-v8-uncompressed.dat", 8)); tests.add(new TestDoc(cpppath + "document-cpp-currentversion-uncompressed.dat", 7)); - tests.add(new TestDoc(cpppath + "document-cpp-currentversion-lz4-9.dat", 7)); tests.add(new TestDoc(cpppath + "document-cpp-v8-uncompressed.dat", 7)); for (TestDoc test : tests) { File f = new File(test.testFile); @@ -147,7 +147,7 @@ public class DocumentSerializationTestCase extends AbstractTypesTest { int remaining = buffer.length; while (remaining > 0) { int read = fin.read(buffer, pos, remaining); - assertFalse(read == -1); + assertNotEquals(-1, read); pos += read; remaining -= read; } @@ -160,7 +160,7 @@ public class DocumentSerializationTestCase extends AbstractTypesTest { assertEquals(new IntegerFieldValue(5), doc.getFieldValue("intfield")); assertEquals(-9.23, ((FloatFieldValue)doc.getFieldValue("floatfield")).getFloat(), 1E-6); assertEquals(new StringFieldValue("This is a string."), doc.getFieldValue("stringfield")); - assertEquals(new LongFieldValue(398420092938472983l), doc.getFieldValue("longfield")); + assertEquals(new LongFieldValue(398420092938472983L), doc.getFieldValue("longfield")); assertEquals(98374532.398820, ((DoubleFieldValue)doc.getFieldValue("doublefield")).getDouble(), 1E-6); assertEquals(new ByteFieldValue((byte)254), doc.getFieldValue("bytefield")); // Todo add cpp serialization @@ -168,20 +168,20 @@ public class DocumentSerializationTestCase extends AbstractTypesTest { ByteBuffer bbuffer = ((Raw)doc.getFieldValue("rawfield")).getByteBuffer(); if (!Arrays.equals("RAW DATA".getBytes(), bbuffer.array())) { System.err.println("Expected 'RAW DATA' but got '" + new String(bbuffer.array()) + "'."); - assertTrue(false); + fail(); } if (test.version > 6) { Document docInDoc = (Document)doc.getFieldValue("docfield"); - assertTrue(docInDoc != null); + assertNotNull(docInDoc); assertEquals(new StringFieldValue("Elvis is dead"), docInDoc.getFieldValue("stringindocfield")); } Array array = (Array)doc.getFieldValue("arrayoffloatfield"); - assertTrue(array != null); + assertNotNull(array); assertEquals(1.0f, ((FloatFieldValue)array.get(0)).getFloat(), 1E-6); assertEquals(2.0f, ((FloatFieldValue)array.get(1)).getFloat(), 1E-6); WeightedSet wset = (WeightedSet)doc.getFieldValue("wsfield"); - assertTrue(wset != null); + assertNotNull(wset); assertEquals(Integer.valueOf(50), wset.get(new StringFieldValue("Weighted 0"))); assertEquals(Integer.valueOf(199), wset.get(new StringFieldValue("Weighted 1"))); } diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTestCase.java index 47605264d44..f1c74ad6efc 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.document; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.compress.CompressionType; import com.yahoo.document.datatypes.Array; import com.yahoo.document.datatypes.BoolFieldValue; import com.yahoo.document.datatypes.ByteFieldValue; @@ -29,8 +28,8 @@ import org.junit.Test; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Map; @@ -100,24 +99,23 @@ public class DocumentTestCase extends DocumentTestCaseBase { return dcMan; } - @SuppressWarnings("deprecation") public void setUpSertestDocType() { docMan = new DocumentTypeManager(); DocumentType docInDocType = new DocumentType("docindoc"); - docInDocType.addField(new Field("tull", 2, docMan.getDataType(2))); + docInDocType.addField(new Field("tull", 2, DataType.STRING)); docMan.registerDocumentType(docInDocType); DocumentType sertestDocType = new DocumentType("sertest"); - sertestDocType.addField(new Field("mailid", 2, docMan.getDataType(2))); - sertestDocType.addField(new Field("date", 3, docMan.getDataType(0))); - sertestDocType.addField(new Field("from", 4, docMan.getDataType(2))); - sertestDocType.addField(new Field("to", 6, docMan.getDataType(2))); - sertestDocType.addField(new Field("subject", 9, docMan.getDataType(2))); - sertestDocType.addField(new Field("body", 10, docMan.getDataType(2))); - sertestDocType.addField(new Field("attachmentcount", 11, docMan.getDataType(0))); - sertestDocType.addField(new Field("attachments", 1081629685, DataType.getArray(docMan.getDataType(2)))); + sertestDocType.addField(new Field("mailid", 2, DataType.STRING)); + sertestDocType.addField(new Field("date", 3, DataType.INT)); + sertestDocType.addField(new Field("from", 4, DataType.STRING)); + sertestDocType.addField(new Field("to", 6, DataType.STRING)); + sertestDocType.addField(new Field("subject", 9, DataType.STRING)); + sertestDocType.addField(new Field("body", 10, DataType.STRING)); + sertestDocType.addField(new Field("attachmentcount", 11, DataType.INT)); + sertestDocType.addField(new Field("attachments", 1081629685, DataType.getArray(DataType.STRING))); sertestDocType.addField(new Field("rawfield", 879, DataType.RAW)); sertestDocType.addField(new Field("weightedfield", 880, DataType.getWeightedSet(DataType.STRING))); sertestDocType.addField(new Field("weightedfieldCreate", 881, DataType.getWeightedSet(DataType.STRING, true, false))); @@ -218,7 +216,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { doc.setFieldValue("long", longVal); } - class VariableIteratorHandler extends FieldPathIteratorHandler { + static class VariableIteratorHandler extends FieldPathIteratorHandler { public String retVal = ""; @@ -463,7 +461,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { } } - class RemoveIteratorHandler extends FieldPathIteratorHandler { + static class RemoveIteratorHandler extends FieldPathIteratorHandler { public ModificationStatus doModify(FieldValue fv) { return ModificationStatus.REMOVED; @@ -640,17 +638,6 @@ public class DocumentTestCase extends DocumentTestCaseBase { } @Test - public void testCppDocCompressed() throws IOException { - docMan = setUpCppDocType(); - byte[] data = readFile("src/test/document/serializecpp-lz4-level9.dat"); - ByteBuffer buf = ByteBuffer.wrap(data); - - Document doc = docMan.createDocument(new GrowableByteBuffer(buf)); - - validateCppDoc(doc); - } - - @Test public void testCppDoc() throws IOException { docMan = setUpCppDocType(); byte[] data = readFile("src/test/document/serializecpp.dat"); @@ -748,7 +735,6 @@ public class DocumentTestCase extends DocumentTestCaseBase { } @Test - @SuppressWarnings("deprecation") public void testSerializeDeserialize() { setUpSertestDocType(); Document doc = getSertestDocument(); @@ -796,7 +782,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { BufferSerializer buf = new BufferSerializer(); try { new Document(DocumentDeserializerFactory.create6(docMan, buf.getBuf())); - assertTrue(false); + fail(); } catch (Exception e) { assertTrue(true); } @@ -804,7 +790,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { buf = BufferSerializer.wrap("Hello world".getBytes()); try { new Document(DocumentDeserializerFactory.create6(docMan, buf.getBuf())); - assertTrue(false); + fail(); } catch (Exception e) { assertTrue(true); } @@ -878,40 +864,6 @@ public class DocumentTestCase extends DocumentTestCaseBase { doc.setFieldValue("something", testlist); } - @SuppressWarnings("deprecation") - @Test - public void testCompressionConfiguredIsIgnored() { - - int size_uncompressed; - { - DocumentTypeManager docMan = new DocumentTypeManager(); - docMan.configure("file:src/tests/data/cppdocument.cfg"); - - Document doc = new Document(docMan.getDocumentType("serializetest"), new DocumentId("id:ns:serializetest::test")); - - doc.setFieldValue("stringfield", - "compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me "); - - GrowableByteBuffer data = new GrowableByteBuffer(); - doc.serialize(data); - size_uncompressed = data.position(); - } - - DocumentTypeManager docMan = new DocumentTypeManager(); - docMan.configure("file:src/tests/data/compressed.cfg"); - - Document doc = new Document(docMan.getDocumentType("serializetest"), new DocumentId("id:ns:serializetest::test")); - - doc.setFieldValue("stringfield", - "compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me compress me "); - - GrowableByteBuffer data = new GrowableByteBuffer(); - doc.serialize(data); - int size_compressed = data.position(); - - assertEquals(size_compressed, size_uncompressed); - } - @Test public void testDocumentDataType() { //use documenttypemanagerconfigurer to read config @@ -1019,7 +971,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { setUpSertestDocType(); Document doc = getSertestDocument(); String json = doc.toJson(); - Map<String, Object> parsed = new ObjectMapper().readValue(json, new TypeReference<Map<String, Object>>() { + Map<String, Object> parsed = new ObjectMapper().readValue(json, new TypeReference<>() { }); assertEquals(parsed.get("id"), "id:ns:sertest::foobar"); assertTrue(parsed.get("fields") instanceof Map); @@ -1096,7 +1048,6 @@ public class DocumentTestCase extends DocumentTestCaseBase { assertEquals(doc, doc2); } - @SuppressWarnings("deprecation") @Test public void testUnknownFieldsDeserialization() { DocumentTypeManager docTypeManasjer = new DocumentTypeManager(); @@ -1118,7 +1069,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { docWithDinner.serialize(buf); buf.flip(); - docTypeManasjer.clear(); + docTypeManasjer.internalClear(); } { @@ -1255,11 +1206,11 @@ public class DocumentTestCase extends DocumentTestCaseBase { } @Test - public void testDocumentIdWithNonTextCharacterCanBeDeserialized() throws UnsupportedEncodingException { + public void testDocumentIdWithNonTextCharacterCanBeDeserialized() { DocumentIdFixture f = new DocumentIdFixture(); // Document id = "id:a:b::0x7c" - String docId = new String(new byte[]{105, 100, 58, 97, 58, 98, 58, 58, 7, 99}, "UTF-8"); + String docId = new String(new byte[]{105, 100, 58, 97, 58, 98, 58, 58, 7, 99}, StandardCharsets.UTF_8); f.serialize(docId); Document result = f.deserialize(); diff --git a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java index a455fa4818e..d85905d5b66 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java @@ -25,7 +25,6 @@ public class DocumentTypeManagerTestCase { // Verify that we can register and retrieve fields. @Test - @SuppressWarnings("deprecation") public void testRegisterAndGet() { DocumentTypeManager manager = new DocumentTypeManager(); @@ -59,7 +58,6 @@ public class DocumentTypeManagerTestCase { assertSame(DataType.DOUBLE, doubleType); } - @SuppressWarnings("deprecation") @Test public void testRecursiveRegister() { StructDataType struct = new StructDataType("mystruct"); @@ -77,18 +75,17 @@ public class DocumentTypeManagerTestCase { DocumentTypeManager manager = new DocumentTypeManager(); manager.register(docType2); - assertEquals(struct, manager.getDataType(struct.getId())); + assertEquals(struct, manager.getDataTypeByCode(struct.getId())); assertEquals(struct, manager.getDataTypeInternal("mystruct")); - assertEquals(wset1, manager.getDataType(wset1.getId())); - assertEquals(wset2, manager.getDataType(wset2.getId())); - assertEquals(array, manager.getDataType(array.getId())); + assertEquals(wset1, manager.getDataTypeByCode(wset1.getId())); + assertEquals(wset2, manager.getDataTypeByCode(wset2.getId())); + assertEquals(array, manager.getDataTypeByCode(array.getId())); assertEquals(docType, manager.getDataTypeInternal("mydoc")); assertEquals(docType2, manager.getDataTypeInternal("myotherdoc")); assertEquals(docType, manager.getDocumentType(new DataTypeName("mydoc"))); assertEquals(docType2, manager.getDocumentType(new DataTypeName("myotherdoc"))); } - @SuppressWarnings("deprecation") @Test public void testMultipleDocuments() { DocumentType docType1 = new DocumentType("foo0"); @@ -104,7 +101,7 @@ public class DocumentTypeManagerTestCase { docType4.addField("bar", DataType.RAW); DocumentTypeManager manager = new DocumentTypeManager(); - manager.clear(); + manager.internalClear(); manager.register(docType1); manager.register(docType2); manager.register(docType3); @@ -122,16 +119,14 @@ public class DocumentTypeManagerTestCase { assertEquals(manager.getDocumentTypes().get(new DataTypeName("foo1")), docType2); } - @SuppressWarnings("deprecation") @Test public void testReverseMapOrder() { DocumentTypeManager manager = createConfiguredManager("file:src/test/document/documentmanager.map.cfg"); - assertNotNull(manager.getDataType(1000)); - assertNotNull(manager.getDataType(1001)); + assertNotNull(manager.getDataTypeByCode(1000)); + assertNotNull(manager.getDataTypeByCode(1001)); } - @SuppressWarnings("deprecation") @Test public void testConfigure() { DocumentTypeManager manager = createConfiguredManager("file:src/test/document/documentmanager.cfg"); @@ -154,7 +149,6 @@ public class DocumentTypeManagerTestCase { assertTrue(foobarfield0.getDataType().getCode() == 2); Field foobarfield1 = type.getField("foobarfield1"); - assertTrue(foobarfield1.isHeader()); assertTrue(foobarfield1.getDataType().getCode() == 4); @@ -169,7 +163,6 @@ public class DocumentTypeManagerTestCase { assertTrue(type.hasField("bananafield0")); Field bananafield0 = type.getField("bananafield0"); - assertTrue(bananafield0.isHeader()); assertTrue(bananafield0.getDataType().getCode() == 16); //inheritance: @@ -504,10 +497,9 @@ search annotationsimplicitstruct { assertReferenceTypePresentInManager(manager, 12345678, "referenced_type"); } - @SuppressWarnings("deprecation") private static void assertReferenceTypePresentInManager(DocumentTypeManager manager, int refTypeId, String refTargetTypeName) { - DataType type = manager.getDataType(refTypeId); + DataType type = manager.getDataTypeByCode(refTypeId); assertTrue(type instanceof ReferenceDataType); ReferenceDataType refType = (ReferenceDataType) type; @@ -530,14 +522,12 @@ search annotationsimplicitstruct { assertReferenceTypePresentInManager(manager, 87654321, "referenced_type2"); } - @SuppressWarnings("deprecation") @Test public void no_temporary_targets_in_references_or_names() { DocumentTypeManager manager = createConfiguredManager("file:src/test/document/documentmanager.replaced_temporary.cfg"); DocumentType docType = manager.getDocumentType("ad"); Field f = docType.getField("campaign_ref"); assertTrue(f.getDataType() instanceof ReferenceDataType); - assertFalse(((ReferenceDataType)f.getDataType()).getTargetType() instanceof TemporaryStructuredDataType); assertEquals("Reference<mystiqueCampaign>", f.getDataType().getName()); } diff --git a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java index 9bbc894758b..9733cd41a88 100644 --- a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java @@ -502,7 +502,6 @@ public class DocumentUpdateTestCase { } @Test - @SuppressWarnings("deprecation") public void testFieldUpdatesInDocUp() { DocumentType t1 = new DocumentType("doo"); Field f1 = new Field("field1", DataType.STRING); diff --git a/document/src/test/java/com/yahoo/document/PositionTypeTestCase.java b/document/src/test/java/com/yahoo/document/PositionTypeTestCase.java index 159b016d6ef..af01075adc9 100644 --- a/document/src/test/java/com/yahoo/document/PositionTypeTestCase.java +++ b/document/src/test/java/com/yahoo/document/PositionTypeTestCase.java @@ -38,8 +38,6 @@ public class PositionTypeTestCase { assertEquals("x", PositionDataType.FIELD_X); assertEquals("y", PositionDataType.FIELD_Y); assertEquals("foo_zcurve", PositionDataType.getZCurveFieldName("foo")); - assertEquals("foo.position", PositionDataType.getPositionSummaryFieldName("foo")); - assertEquals("foo.distance", PositionDataType.getDistanceSummaryFieldName("foo")); } @Test diff --git a/document/src/test/java/com/yahoo/document/ReferenceDataTypeTestCase.java b/document/src/test/java/com/yahoo/document/ReferenceDataTypeTestCase.java index 53c8a0ecc94..a14b56cc539 100644 --- a/document/src/test/java/com/yahoo/document/ReferenceDataTypeTestCase.java +++ b/document/src/test/java/com/yahoo/document/ReferenceDataTypeTestCase.java @@ -72,32 +72,4 @@ public class ReferenceDataTypeTestCase { assertTrue(fixture.refType.isValueCompatible(fixture.refTypeClone.createFieldValue())); } - @SuppressWarnings("deprecation") - @Test - public void reference_type_can_be_constructed_with_temporary_structured_data_type() { - TemporaryStructuredDataType tempType = new TemporaryStructuredDataType("cooldoc"); - ReferenceDataType refType = new ReferenceDataType(tempType, 321); - assertEquals("Reference<cooldoc>", refType.getName()); - assertEquals(321, refType.getId()); - assertEquals(tempType, refType.getTargetType()); - } - - @SuppressWarnings("deprecation") - @Test - public void can_replace_temporary_target_data_type() { - TemporaryStructuredDataType tempType = new TemporaryStructuredDataType("cooldoc"); - ReferenceDataType refType = new ReferenceDataType(tempType, 321); - DocumentType concreteType = new DocumentType("cooldoc"); - refType.setTargetType(concreteType); - assertEquals("Reference<cooldoc>", refType.getName()); - assertEquals(321, refType.getId()); - assertEquals(concreteType, refType.getTargetType()); - } - - @Test(expected = IllegalStateException.class) - public void replacing_already_concrete_type_throws_illegal_state_exception() { - ReferenceDataType refType = new ReferenceDataType(new DocumentType("foo"), 123); - refType.setTargetType(new DocumentType("foo")); - } - } diff --git a/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java b/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java deleted file mode 100644 index 80154891d83..00000000000 --- a/document/src/test/java/com/yahoo/document/TemporaryDataTypeTestCase.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document; - -import com.yahoo.document.datatypes.StringFieldValue; -import org.junit.Test; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; - -/** - * @author Einar M R Rosenvinge - */ -@SuppressWarnings("deprecation") -public class TemporaryDataTypeTestCase { - - @Test - public void requireNulls() { - TemporaryDataType type = new TemporaryDataType(0, ""); - assertNull(type.createFieldValue(new Object())); - assertNull(type.createFieldValue()); - assertNull(type.getValueClass()); - assertFalse(type.isValueCompatible(new StringFieldValue(""))); - } - -} diff --git a/document/src/test/java/com/yahoo/document/TemporaryStructuredDataTypeTestCase.java b/document/src/test/java/com/yahoo/document/TemporaryStructuredDataTypeTestCase.java deleted file mode 100644 index 3b5cd29b90d..00000000000 --- a/document/src/test/java/com/yahoo/document/TemporaryStructuredDataTypeTestCase.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.10 - */ -@SuppressWarnings("deprecation") -public class TemporaryStructuredDataTypeTestCase { - @Test - public void basic() { - TemporaryStructuredDataType type = TemporaryStructuredDataType.create("banana"); - assertEquals("banana", type.getName()); - int originalId = type.getId(); - type.setName("apple"); - assertEquals("apple", type.getName()); - assertNotEquals(originalId, type.getId()); - } -} diff --git a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java index 039880185ce..e396fe8912b 100644 --- a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java +++ b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java @@ -529,26 +529,6 @@ public class DocumentUpdateJsonSerializerTest { } @Test - public void testSimultaneousFieldsAndFieldPathsUpdate() { - roundtripSerializeJsonAndMatch(inputJson( - "{", - " 'update': 'DOCUMENT_ID',", - " 'fields': {", - " 'singlepos_field': {", - " 'assign': 'N60.222333;E10.12'", - " },", - " 'deep_map{my_field}': {", - " 'assign': {", - " 'my_key': 'my value',", - " 'new_key': 'new value'", - " }", - " }", - " }", - "}" - )); - } - - @Test public void testAssignFieldPathUpdate() { roundtripSerializeJsonAndMatch(inputJson( "{", @@ -637,34 +617,6 @@ public class DocumentUpdateJsonSerializerTest { } @Test - public void testAssignSinglePos() { - roundtripSerializeJsonAndMatch(inputJson( - "{", - " 'update': 'DOCUMENT_ID',", - " 'fields': {", - " 'singlepos_field': {", - " 'assign': 'N60.222333;E10.12'", - " }", - " }", - "}" - )); - } - - @Test - public void testAssignMultiPos() { - roundtripSerializeJsonAndMatch(inputJson( - "{", - " 'update': 'DOCUMENT_ID',", - " 'fields': {", - " 'multipos_field': {", - " 'assign': [ 'N0.0;E0.0', 'S1.1;W1.1', 'N10.2;W122.2' ]", - " }", - " }", - "}" - )); - } - - @Test public void testClearField() { roundtripSerializeJsonAndMatch(inputJson( "{", diff --git a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java index b7f24368ad7..92f28d8f219 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java @@ -215,13 +215,11 @@ public class JsonWriterTestCase { @Test public void singlePosTest() throws IOException { - roundTripEquality("id:unittest:testsinglepos::bamf", "{ \"singlepos\": \"N60.222333;E10.12\" }"); roundTripEquality("id:unittest:testsinglepos::bamf", "{ \"geopos\": { \"lat\": 60.222333, \"lng\": 10.12 } }"); } @Test public void multiPosTest() throws IOException { - roundTripEquality("id:unittest:testmultipos::bamf", "{ \"multipos\": [ \"N0.0;E0.0\", \"S1.1;W1.1\", \"N10.2;W122.2\" ] }"); roundTripEquality("id:unittest:testmultipos::bamf", "{ \"geopos\": [ { \"lat\": -1.5, \"lng\": -1.5 }, { \"lat\": 63.4, \"lng\": 10.4 }, { \"lat\": 0.0, \"lng\": 0.0 } ] }"); } diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java index 38fdadb18e4..b7a3589a4d0 100644 --- a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java +++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java @@ -778,9 +778,7 @@ public class DocumentSelectorTestCase { var s=new DocumentSelector("parent.parentField = \"parentValue\""); List<DocumentPut> documents = createDocs(); assertEquals(Result.TRUE, evaluate("test", documents.get(0))); - // TODO Vespa 8: Change the following assert (only) to expect Result.FALSE - assertEquals("Matching on type is [on Vespa 7, not] exact", - Result.TRUE, evaluate("parent", documents.get(0))); + assertEquals("Matching on type is exact", Result.FALSE, evaluate("parent", documents.get(0))); assertEquals(Result.TRUE, evaluate("test.parentField = \"parentValue\"", documents.get(0))); assertEquals("Fields may be accessed by parent type", Result.TRUE, evaluate("parent.parentField = \"parentValue\"", documents.get(0))); diff --git a/document/src/test/java/com/yahoo/document/serialization/SerializeAnnotationsTestCase.java b/document/src/test/java/com/yahoo/document/serialization/SerializeAnnotationsTestCase.java index 9e58e2540a0..321df9e2660 100644 --- a/document/src/test/java/com/yahoo/document/serialization/SerializeAnnotationsTestCase.java +++ b/document/src/test/java/com/yahoo/document/serialization/SerializeAnnotationsTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.serialization; +import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentTypeManagerConfigurer; @@ -36,7 +37,6 @@ import static org.junit.Assert.assertEquals; * test to fail, so you will need to update the * AnnotationDeserialization component to handle the format changes. */ -@SuppressWarnings("deprecation") public class SerializeAnnotationsTestCase { private static final String PATH = "src/tests/serialization/"; @@ -89,10 +89,6 @@ public class SerializeAnnotationsTestCase { SpanList root = new SpanList(); SpanTree tree = new SpanTree("html", root); - DataType positionType = docMan.getDataType("myposition"); - StructDataType cityDataType = - (StructDataType) docMan.getDataType("annotation.city"); - AnnotationTypeRegistry registry = docMan.getAnnotationTypeRegistry(); AnnotationType textType = registry.getType("text"); AnnotationType beginTag = registry.getType("begintag"); @@ -101,9 +97,10 @@ public class SerializeAnnotationsTestCase { AnnotationType paragraphType = registry.getType("paragraph"); AnnotationType cityType = registry.getType("city"); - AnnotationReferenceDataType annRefType = - (AnnotationReferenceDataType) - docMan.getDataType("annotationreference<text>"); + var cityDataType = (StructDataType) cityType.getDataType(); + var positionType = (StructDataType) cityDataType.getField("position").getDataType(); + var refArrT = (ArrayDataType) cityDataType.getField("references").getDataType(); + var annRefType = (AnnotationReferenceDataType) refArrT.getNestedType(); Struct position = new Struct(positionType); position.setFieldValue("latitude", new DoubleFieldValue(37.774929)); diff --git a/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java b/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java index 9814e765a89..57f9a14d60b 100644 --- a/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java +++ b/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java @@ -18,7 +18,6 @@ import static org.junit.Assert.assertEquals; * @author Simon Thoresen Hult * @author vekterli */ -@SuppressWarnings("deprecation") public class VespaDocumentSerializerTestCase { @Test diff --git a/document/src/test/serializeddocuments/document-java-currentversion-lz4-9.dat b/document/src/test/serializeddocuments/document-java-currentversion-lz4-9.dat Binary files differdeleted file mode 100644 index 033844ac09b..00000000000 --- a/document/src/test/serializeddocuments/document-java-currentversion-lz4-9.dat +++ /dev/null diff --git a/document/src/tests/.gitignore b/document/src/tests/.gitignore index 4d5d93dd093..f310f65eb7e 100644 --- a/document/src/tests/.gitignore +++ b/document/src/tests/.gitignore @@ -15,5 +15,4 @@ testrunner *_test document_gtest_runner_app document_testrunner_app -/serializecpp-lz4-level9.dat /serializecppsplit_body.dat diff --git a/document/src/tests/data/document-cpp-currentversion-lz4-9.dat b/document/src/tests/data/document-cpp-currentversion-lz4-9.dat Binary files differdeleted file mode 100644 index 3383d97f253..00000000000 --- a/document/src/tests/data/document-cpp-currentversion-lz4-9.dat +++ /dev/null diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp index 1606cbc634d..464d3c66a7a 100644 --- a/document/src/tests/documentselectparsertest.cpp +++ b/document/src/tests/documentselectparsertest.cpp @@ -673,8 +673,8 @@ TEST_F(DocumentSelectParserTest, operators_1) // Inherited doctypes PARSE("testdoctype2", *_doc[4], True); PARSE("testdoctype2", *_doc[3], False); - PARSE("testdoctype1", *_doc[4], True); - PARSE("testdoctype1.headerval = 10", *_doc[4], True); + PARSE("testdoctype1", *_doc[4], False); // testdoctype2 inherits testdoctype1, but we use exact matching for types + PARSE("testdoctype1.headerval = 10", *_doc[4], Invalid); } TEST_F(DocumentSelectParserTest, operators_2) @@ -1182,7 +1182,7 @@ void DocumentSelectParserTest::testDocumentUpdates1() // Inherited doctypes PARSE("testdoctype2", *_update[4], True); PARSE("testdoctype2", *_update[3], False); - PARSE("testdoctype1", *_update[4], True); + PARSE("testdoctype1", *_update[4], False); // testdoctype2 inherits testdoctype1, but we use exact matching for types PARSE("testdoctype1.headerval = 10", *_update[4], Invalid); } diff --git a/document/src/vespa/document/repo/documenttyperepo.cpp b/document/src/vespa/document/repo/documenttyperepo.cpp index 488dfea2fd7..c9473c90934 100644 --- a/document/src/vespa/document/repo/documenttyperepo.cpp +++ b/document/src/vespa/document/repo/documenttyperepo.cpp @@ -819,13 +819,6 @@ private: for (const auto & importD : docT.importedfield) { doc_type->add_imported_field_name(importD.name); } - for (const auto & entry : docT.fieldsets) { - DocumentType::FieldSet::Fields fields; - for (const auto& f : entry.second.fields) { - fields.insert(f); - } - doc_type->addFieldSet(entry.first, fields); - } for (const auto & inheritD : docT.inherits) { const DataType *dt = _made_types[inheritD.idx]; const DocumentType * parent = dynamic_cast<const DocumentType *>(dt); @@ -837,6 +830,13 @@ private: doc_type->inherit(*parent); } } + for (const auto & entry : docT.fieldsets) { + DocumentType::FieldSet::Fields fields; + for (const auto& f : entry.second.fields) { + fields.insert(f); + } + doc_type->addFieldSet(entry.first, fields); + } } void fillAnnotationTypes(DocTypeInProgress & dtInP) { diff --git a/document/src/vespa/document/select/doctype.cpp b/document/src/vespa/document/select/doctype.cpp index 800e8aa34b4..c86fd20e4f1 100644 --- a/document/src/vespa/document/select/doctype.cpp +++ b/document/src/vespa/document/select/doctype.cpp @@ -14,15 +14,7 @@ namespace { bool documentTypeEqualsName(const DocumentType& type, vespalib::stringref name) { - if (type.getName() == name) return true; - // TODO Vespa 8: Remove this for loop on Vespa 8 - for (std::vector<const DocumentType *>::const_iterator it - = type.getInheritedTypes().begin(); - it != type.getInheritedTypes().end(); ++it) - { - if (documentTypeEqualsName(**it, name)) return true; - } - return false; + return (type.getName() == name); } } diff --git a/document/src/vespa/document/select/valuenodes.cpp b/document/src/vespa/document/select/valuenodes.cpp index 452779ca5ba..8102a944ff0 100644 --- a/document/src/vespa/document/select/valuenodes.cpp +++ b/document/src/vespa/document/select/valuenodes.cpp @@ -22,14 +22,7 @@ namespace document::select { namespace { bool documentTypeEqualsName(const DocumentType& type, vespalib::stringref name) { - if (type.getName() == name) return true; - for (std::vector<const DocumentType *>::const_iterator it - = type.getInheritedTypes().begin(); - it != type.getInheritedTypes().end(); ++it) - { - if (documentTypeEqualsName(**it, name)) return true; - } - return false; + return (type.getName() == name); } } diff --git a/documentapi-dependencies/pom.xml b/documentapi-dependencies/pom.xml index 7a977a36842..25552700528 100644 --- a/documentapi-dependencies/pom.xml +++ b/documentapi-dependencies/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>documentapi-dependencies</artifactId> <packaging>pom</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> diff --git a/documentapi/abi-spec.json b/documentapi/abi-spec.json index 49bdf32bfb1..f6ed4805e7f 100644 --- a/documentapi/abi-spec.json +++ b/documentapi/abi-spec.json @@ -37,20 +37,14 @@ ], "methods": [ "public abstract com.yahoo.documentapi.Result put(com.yahoo.document.Document)", - "public com.yahoo.documentapi.Result put(com.yahoo.document.Document, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public com.yahoo.documentapi.Result put(com.yahoo.document.DocumentPut)", - "public com.yahoo.documentapi.Result put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public com.yahoo.documentapi.Result put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.DocumentOperationParameters)", "public abstract com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId)", - "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, boolean, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", - "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, com.yahoo.documentapi.DocumentOperationParameters)", "public abstract com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId)", - "public com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId, com.yahoo.documentapi.DocumentOperationParameters)", "public com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentRemove, com.yahoo.documentapi.DocumentOperationParameters)", "public abstract com.yahoo.documentapi.Result update(com.yahoo.document.DocumentUpdate)", - "public com.yahoo.documentapi.Result update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public com.yahoo.documentapi.Result update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.DocumentOperationParameters)", "public abstract double getCurrentWindowSize()" ], @@ -77,7 +71,6 @@ "abstract" ], "methods": [ - "public static com.yahoo.documentapi.DocumentAccess createDefault()", "public static com.yahoo.documentapi.DocumentAccess createForNonContainer()", "protected void <init>(com.yahoo.documentapi.DocumentAccessParams)", "public abstract com.yahoo.documentapi.SyncSession createSyncSession(com.yahoo.documentapi.SyncParameters)", @@ -132,8 +125,6 @@ "methods": [ "public void <init>(long)", "public void <init>(long, com.yahoo.document.DocumentId)", - "public void <init>(long, java.lang.String, boolean)", - "public void <init>(long, com.yahoo.document.DocumentId, java.lang.String, boolean)", "public void <init>(long, com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.Response$Outcome)", "public void <init>(long, com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.Response$Outcome, com.yahoo.messagebus.Trace)", "public com.yahoo.document.DocumentId getDocumentId()", @@ -163,14 +154,12 @@ ], "methods": [ "public static com.yahoo.documentapi.DocumentOperationParameters parameters()", - "public com.yahoo.documentapi.DocumentOperationParameters withPriority(com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public com.yahoo.documentapi.DocumentOperationParameters withFieldSet(com.yahoo.document.fieldset.FieldSet)", "public com.yahoo.documentapi.DocumentOperationParameters withFieldSet(java.lang.String)", "public com.yahoo.documentapi.DocumentOperationParameters withRoute(java.lang.String)", "public com.yahoo.documentapi.DocumentOperationParameters withTraceLevel(int)", "public com.yahoo.documentapi.DocumentOperationParameters withDeadline(java.time.Instant)", "public com.yahoo.documentapi.DocumentOperationParameters withResponseHandler(com.yahoo.documentapi.ResponseHandler)", - "public java.util.Optional priority()", "public java.util.Optional fieldSet()", "public java.util.Optional route()", "public java.util.OptionalInt traceLevel()", @@ -192,8 +181,6 @@ "public void <init>(long)", "public void <init>(long, com.yahoo.document.Document)", "public void <init>(long, com.yahoo.document.Document, com.yahoo.messagebus.Trace)", - "public void <init>(long, java.lang.String, boolean)", - "public void <init>(long, com.yahoo.document.Document, java.lang.String, boolean)", "public void <init>(long, com.yahoo.document.Document, java.lang.String, com.yahoo.documentapi.Response$Outcome)", "public void <init>(long, com.yahoo.document.Document, java.lang.String, com.yahoo.documentapi.Response$Outcome, com.yahoo.messagebus.Trace)", "public com.yahoo.document.Document getDocument()", @@ -213,9 +200,7 @@ "methods": [ "public void <init>(long)", "public void <init>(long, com.yahoo.document.DocumentUpdate)", - "public void <init>(long, java.lang.String, boolean)", "public void <init>(long, java.lang.String, com.yahoo.documentapi.Response$Outcome)", - "public void <init>(long, com.yahoo.document.DocumentUpdate, java.lang.String, boolean)", "public void <init>(long, com.yahoo.document.DocumentUpdate, java.lang.String, com.yahoo.documentapi.Response$Outcome)", "public void <init>(long, com.yahoo.document.DocumentUpdate, java.lang.String, com.yahoo.documentapi.Response$Outcome, com.yahoo.messagebus.Trace)", "public com.yahoo.document.DocumentUpdate getDocumentUpdate()", @@ -431,7 +416,6 @@ "methods": [ "public void <init>(long)", "public void <init>(long, java.lang.String)", - "public void <init>(long, java.lang.String, boolean)", "public void <init>(long, java.lang.String, com.yahoo.documentapi.Response$Outcome)", "public void <init>(long, java.lang.String, com.yahoo.documentapi.Response$Outcome, com.yahoo.messagebus.Trace)", "public java.lang.String getTextMessage()", @@ -473,8 +457,7 @@ "fields": [ "public static final enum com.yahoo.documentapi.Result$ResultType SUCCESS", "public static final enum com.yahoo.documentapi.Result$ResultType TRANSIENT_ERROR", - "public static final enum com.yahoo.documentapi.Result$ResultType FATAL_ERROR", - "public static final enum com.yahoo.documentapi.Result$ResultType CONDITION_NOT_MET_ERROR" + "public static final enum com.yahoo.documentapi.Result$ResultType FATAL_ERROR" ] }, "com.yahoo.documentapi.Result": { @@ -486,10 +469,8 @@ "methods": [ "public void <init>()", "public void <init>(long)", - "public void <init>(com.yahoo.documentapi.Result$ResultType, java.lang.Error)", "public void <init>(com.yahoo.documentapi.Result$ResultType, com.yahoo.messagebus.Error)", "public boolean isSuccess()", - "public java.lang.Error getError()", "public com.yahoo.messagebus.Error error()", "public long getRequestId()", "public com.yahoo.documentapi.Result$ResultType type()", @@ -589,18 +570,13 @@ ], "methods": [ "public abstract void put(com.yahoo.document.DocumentPut)", - "public void put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public void put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.DocumentOperationParameters)", "public com.yahoo.document.Document get(com.yahoo.document.DocumentId)", - "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public abstract com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.time.Duration)", - "public abstract com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority, java.time.Duration)", "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, com.yahoo.documentapi.DocumentOperationParameters, java.time.Duration)", "public abstract boolean remove(com.yahoo.document.DocumentRemove)", - "public abstract boolean remove(com.yahoo.document.DocumentRemove, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public boolean remove(com.yahoo.document.DocumentRemove, com.yahoo.documentapi.DocumentOperationParameters)", "public abstract boolean update(com.yahoo.document.DocumentUpdate)", - "public abstract boolean update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public boolean update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.DocumentOperationParameters)" ], "fields": [] @@ -881,7 +857,6 @@ "public long getToTimestamp()", "public boolean visitRemoves()", "public boolean getVisitRemoves()", - "public boolean getVisitHeadersOnly()", "public java.lang.String fieldSet()", "public java.lang.String getFieldSet()", "public boolean visitInconsistentBuckets()", @@ -892,8 +867,6 @@ "public java.lang.String getRemoteDataHandler()", "public com.yahoo.documentapi.VisitorDataHandler getLocalDataHandler()", "public com.yahoo.documentapi.VisitorControlHandler getControlHandler()", - "public boolean getDynamicallyIncreaseMaxBucketsPerVisitor()", - "public float getDynamicMaxBucketsIncreaseFactor()", "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority getPriority()", "public void setDocumentSelection(java.lang.String)", "public void setBucketSpace(java.lang.String)", @@ -905,7 +878,6 @@ "public void setToTimestamp(long)", "public void visitRemoves(boolean)", "public void setVisitRemoves(boolean)", - "public void setVisitHeadersOnly(boolean)", "public void fieldSet(java.lang.String)", "public void setFieldSet(java.lang.String)", "public void visitInconsistentBuckets(boolean)", @@ -921,8 +893,6 @@ "public void setRoute(java.lang.String)", "public void setRoute(com.yahoo.messagebus.routing.Route)", "public com.yahoo.messagebus.routing.Route getRoute()", - "public void setMaxFirstPassHits(long)", - "public long getMaxFirstPassHits()", "public void setMaxTotalHits(long)", "public long getMaxTotalHits()", "public java.util.Set getBucketsToVisit()", @@ -932,15 +902,11 @@ "public void setTraceLevel(int)", "public int getTraceLevel()", "public void setPriority(com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", - "public void setLoadType(com.yahoo.documentapi.messagebus.loadtypes.LoadType)", - "public com.yahoo.documentapi.messagebus.loadtypes.LoadType getLoadType()", "public boolean skipBucketsOnFatalErrors()", "public void skipBucketsOnFatalErrors(boolean)", "public void slice(int, int)", "public int getSlices()", "public int getSliceId()", - "public void setDynamicallyIncreaseMaxBucketsPerVisitor(boolean)", - "public void setDynamicMaxBucketsIncreaseFactor(float)", "public java.lang.String toString()" ], "fields": [] @@ -1033,13 +999,9 @@ "methods": [ "public void <init>(com.yahoo.documentapi.local.LocalDocumentAccess)", "public void put(com.yahoo.document.DocumentPut)", - "public void put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.time.Duration)", - "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority, java.time.Duration)", "public boolean remove(com.yahoo.document.DocumentRemove)", - "public boolean remove(com.yahoo.document.DocumentRemove, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public boolean update(com.yahoo.document.DocumentUpdate)", - "public boolean update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public com.yahoo.documentapi.Response getNext()", "public com.yahoo.documentapi.Response getNext(int)", "public void destroy()" @@ -1081,7 +1043,6 @@ "public com.yahoo.documentapi.Result put(com.yahoo.document.Document)", "public com.yahoo.documentapi.Result put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.DocumentOperationParameters)", "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId)", - "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, boolean, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, com.yahoo.documentapi.DocumentOperationParameters)", "public com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId)", "public com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentRemove, com.yahoo.documentapi.DocumentOperationParameters)", @@ -1133,8 +1094,6 @@ ], "methods": [ "public void <init>()", - "public void <init>(com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet)", - "public com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet getLoadTypes()", "public java.lang.String getRoutingConfigId()", "public com.yahoo.documentapi.messagebus.MessageBusParams setRoutingConfigId(java.lang.String)", "public java.lang.String getProtocolConfigId()", @@ -1190,16 +1149,12 @@ "public void destroy()", "public com.yahoo.messagebus.Reply syncSend(com.yahoo.messagebus.Message)", "public void put(com.yahoo.document.DocumentPut)", - "public void put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public void put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.DocumentOperationParameters)", "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.time.Duration)", - "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority, java.time.Duration)", "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, com.yahoo.documentapi.DocumentOperationParameters, java.time.Duration)", "public boolean remove(com.yahoo.document.DocumentRemove)", - "public boolean remove(com.yahoo.document.DocumentRemove, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public boolean remove(com.yahoo.document.DocumentRemove, com.yahoo.documentapi.DocumentOperationParameters)", "public boolean update(com.yahoo.document.DocumentUpdate)", - "public boolean update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public boolean update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.DocumentOperationParameters)", "public java.lang.String getRoute()", "public void setRoute(java.lang.String)", @@ -1796,8 +1751,6 @@ "public void swapState(com.yahoo.messagebus.Routable)", "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority getPriority()", "public void setPriority(com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", - "public com.yahoo.documentapi.messagebus.loadtypes.LoadType getLoadType()", - "public void setLoadType(com.yahoo.documentapi.messagebus.loadtypes.LoadType)", "public int getApproxSize()", "public com.yahoo.text.Utf8String getProtocol()" ], @@ -1847,10 +1800,8 @@ "public static com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority getPriority(int)", "public static com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority getPriorityByName(java.lang.String)", "public void <init>(com.yahoo.document.DocumentTypeManager)", - "public void <init>(com.yahoo.document.DocumentTypeManager, java.lang.String)", "public void <init>(com.yahoo.document.DocumentTypeManager, com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig, com.yahoo.vespa.config.content.DistributionConfig)", - "public void <init>(com.yahoo.document.DocumentTypeManager, com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet, com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig, com.yahoo.vespa.config.content.DistributionConfig)", - "public void <init>(com.yahoo.document.DocumentTypeManager, java.lang.String, com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet)", + "public void <init>(com.yahoo.document.DocumentTypeManager, java.lang.String)", "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocol putRoutingPolicyFactory(java.lang.String, com.yahoo.documentapi.messagebus.protocol.RoutingPolicyFactory)", "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocol putRoutableFactory(int, com.yahoo.documentapi.messagebus.protocol.RoutableFactory, com.yahoo.component.VersionSpecification)", "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocol putRoutableFactory(int, com.yahoo.documentapi.messagebus.protocol.RoutableFactory, java.util.List)", @@ -2055,7 +2006,6 @@ "public static java.lang.String getDefMd5()", "public static java.lang.String getDefName()", "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", "public void <init>(com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Builder)", "public java.util.Map cluster()", "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Cluster cluster(java.lang.String)" @@ -2064,7 +2014,6 @@ "public static final java.lang.String CONFIG_DEF_MD5", "public static final java.lang.String CONFIG_DEF_NAME", "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", "public static final java.lang.String[] CONFIG_DEF_SCHEMA" ] }, @@ -2243,7 +2192,6 @@ "public static java.lang.String getDefMd5()", "public static java.lang.String getDefName()", "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", "public void <init>(com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig$Builder)", "public java.util.List route()", "public com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig$Route route(int)" @@ -2252,7 +2200,6 @@ "public static final java.lang.String CONFIG_DEF_MD5", "public static final java.lang.String CONFIG_DEF_NAME", "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", "public static final java.lang.String[] CONFIG_DEF_SCHEMA" ] }, @@ -2702,7 +2649,7 @@ "protected abstract boolean doEncode(com.yahoo.documentapi.messagebus.protocol.DocumentMessage, com.yahoo.document.serialization.DocumentSerializer)", "protected abstract com.yahoo.documentapi.messagebus.protocol.DocumentMessage doDecode(com.yahoo.document.serialization.DocumentDeserializer)", "public boolean encode(com.yahoo.messagebus.Routable, com.yahoo.document.serialization.DocumentSerializer)", - "public com.yahoo.messagebus.Routable decode(com.yahoo.document.serialization.DocumentDeserializer, com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet)" + "public com.yahoo.messagebus.Routable decode(com.yahoo.document.serialization.DocumentDeserializer)" ], "fields": [] }, @@ -2718,7 +2665,7 @@ "protected abstract boolean doEncode(com.yahoo.documentapi.messagebus.protocol.DocumentReply, com.yahoo.document.serialization.DocumentSerializer)", "protected abstract com.yahoo.documentapi.messagebus.protocol.DocumentReply doDecode(com.yahoo.document.serialization.DocumentDeserializer)", "public boolean encode(com.yahoo.messagebus.Routable, com.yahoo.document.serialization.DocumentSerializer)", - "public com.yahoo.messagebus.Routable decode(com.yahoo.document.serialization.DocumentDeserializer, com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet)" + "public com.yahoo.messagebus.Routable decode(com.yahoo.document.serialization.DocumentDeserializer)" ], "fields": [] }, @@ -3128,8 +3075,7 @@ ], "methods": [ "public abstract boolean encode(com.yahoo.messagebus.Routable, com.yahoo.document.serialization.DocumentSerializer)", - "public abstract com.yahoo.messagebus.Routable decode(com.yahoo.document.serialization.DocumentDeserializer, com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet)", - "public com.yahoo.messagebus.Routable decode(com.yahoo.document.serialization.DocumentDeserializer)" + "public abstract com.yahoo.messagebus.Routable decode(com.yahoo.document.serialization.DocumentDeserializer)" ], "fields": [] }, diff --git a/documentapi/pom.xml b/documentapi/pom.xml index 7d91e246cfe..9b690f345ae 100644 --- a/documentapi/pom.xml +++ b/documentapi/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>documentapi</artifactId> <packaging>jar</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <!-- WARNING: dependencies (apart from test scoped) must be added to documentapi-dependencies, not here! --> diff --git a/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java index 5a63a403de3..c297b61fc91 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java @@ -37,25 +37,6 @@ public interface AsyncSession extends Session { Result put(Document document); /** - * <p>Puts a document. This method returns immediately.</p> - * - * <p>If this result is a success, this - * call will cause one or more {@link DocumentResponse} objects to appear within the timeout time of this session. - * The response returned later will either be a success, or contain the document submitted here. - * If it was not a success, this method has no further effects.</p> - * - * @param document the Document to put - * @param priority the priority with which to send the operation - * @return the synchronous result of this operation - * @deprecated specifying explicit operation priority is deprecated - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - default Result put(Document document, DocumentProtocol.Priority priority) { - return put(new DocumentPut(document), parameters().withPriority(priority)); - } - - /** * <p>Puts a document, with optional conditions on the operation. This method returns immediately.</p> * * <p>If this result is a success, this @@ -79,25 +60,6 @@ public interface AsyncSession extends Session { * If it was not a success, this method has no further effects.</p> * * @param documentPut the DocumentPut to perform - * @param priority the priority with which to send the operation - * @return the synchronous result of this operation - * @deprecated specifying explicit operation priority is deprecated - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - default Result put(DocumentPut documentPut, DocumentProtocol.Priority priority) { - return put(documentPut, parameters().withPriority(priority)); - } - - /** - * <p>Puts a document, with optional conditions on the operation. This method returns immediately.</p> - * - * <p>If this result is a success, this - * call will cause one or more {@link DocumentResponse} objects to appear within the timeout time of this session. - * The response returned later will either be a success, or contain the document submitted here. - * If it was not a success, this method has no further effects.</p> - * - * @param documentPut the DocumentPut to perform * @param parameters parameters for the operation * @return the synchronous result of this operation */ @@ -128,45 +90,6 @@ public interface AsyncSession extends Session { * If it was not a success, this method has no further effects.</p> * * @param id the id of the document to get - * @param priority The priority with which to perform this operation. - * @return the synchronous result of this operation - * @throws UnsupportedOperationException if this access implementation does not support retrieving - * @deprecated The 'headersonly' flag has no effect. Specifying explicit operation priority is deprecated - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - default Result get(DocumentId id, boolean headersOnly, DocumentProtocol.Priority priority) { - return get(id); - } - - /** - * <p>Gets a document. This method returns immediately.</p> - * - * <p>If this result is a success, this - * call will cause one or more {@link DocumentResponse} objects to appear within the timeout time of this session. - * The response returned later will contain the requested document if it is a success. - * If it was not a success, this method has no further effects.</p> - * - * @param id the id of the document to get - * @param priority The priority with which to perform this operation. - * @return the synchronous result of this operation - * @throws UnsupportedOperationException if this access implementation does not support retrieving - * @deprecated specifying explicit operation priority is deprecated - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - default Result get(DocumentId id, DocumentProtocol.Priority priority) { - return get(id, parameters().withPriority(priority)); - } - - /** - * <p>Gets a document. This method returns immediately.</p> - * - * <p>If this result is a success, this - * call will cause one or more {@link DocumentResponse} objects to appear within the timeout time of this session. - * The response returned later will contain the requested document if it is a success. - * If it was not a success, this method has no further effects.</p> - * - * @param id the id of the document to get * @param parameters parameters for the operation * @return the synchronous result of this operation * @throws UnsupportedOperationException if this access implementation does not support retrieving @@ -194,26 +117,6 @@ public interface AsyncSession extends Session { * <p>Removes a document if it is present. This method returns immediately.</p> * * <p>If this result is a success, this - * call will cause one or more {@link DocumentIdResponse} objects to apprear within the timeout time of this session. - * The response returned later will either be a success, or contain the document id submitted here. - * If it was not a success, this method has no further effects.</p> - * - * @param id the id of the document to remove - * @param priority The priority with which to perform this operation. - * @return the synchronous result of this operation - * @throws UnsupportedOperationException if this access implementation does not support removal - * @deprecated specifying explicit operation priority is deprecated. Use methods without priority parameter. - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - default Result remove(DocumentId id, DocumentProtocol.Priority priority) { - return remove(id, parameters().withPriority(priority)); - } - - /** - * <p>Removes a document if it is present. This method returns immediately.</p> - * - * <p>If this result is a success, this * call will cause one or more {@link DocumentIdResponse} objects to appear within the timeout time of this session. * The response returned later will either be a success, or contain the document id submitted here. * If it was not a success, this method has no further effects.</p> @@ -267,26 +170,6 @@ public interface AsyncSession extends Session { * If it was not a success, this method has no further effects.</p> * * @param update the updates to perform - * @param priority The priority with which to perform this operation. - * @return the synchronous result of this operation - * @throws UnsupportedOperationException if this access implementation does not support update - * @deprecated specifying explicit operation priority is deprecated - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - default Result update(DocumentUpdate update, DocumentProtocol.Priority priority) { - return update(update, parameters().withPriority(priority)); - } - - /** - * <p>Updates a document. This method returns immediately.</p> - * - * <p>If this result is a success, this - * call will cause one or more {@link DocumentUpdateResponse} within the timeout time of this session. - * The returned response returned later will either be a success or contain the update submitted here. - * If it was not a success, this method has no further effects.</p> - * - * @param update the updates to perform * @param parameters parameters for the operation * @return the synchronous result of this operation * @throws UnsupportedOperationException if this access implementation does not support update diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java index 29fc470fe2a..9a6e634f071 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java @@ -47,27 +47,9 @@ import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; public abstract class DocumentAccess { private final DocumentTypeManager documentTypeManager; - @SuppressWarnings("removal") // TODO Vespa 8: remove private final ConfigSubscriber documentTypeConfigSubscriber; /** - * This is a convenience method to return a document access object with - * all default parameter values. The client that calls this method is also - * responsible for shutting the object down when done. If an error occurred - * while attempting to create such an object, this method will throw an - * exception. - * - * @deprecated DocumentAccess may be injected in containers — otherwise use {@link #createForNonContainer()}. - * - * @return a running document access object with all default configuration - */ - @Deprecated(since = "7") - public static DocumentAccess createDefault() { - return new MessageBusDocumentAccess(); - } - - - /** * This is a convenience method to return a document access object when running * outside of a Vespa application container, with all default parameter values. * The client that calls this method is also responsible for shutting the object diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java index 9625a7c92ad..68e43edf082 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentIdResponse.java @@ -31,30 +31,6 @@ public class DocumentIdResponse extends Response { } /** - * Creates a response containing a textual message - * - * @param textMessage the message to encapsulate in the Response - * @param success true if the response represents a successful call - */ - @Deprecated(since = "7") // TODO: Remove on Vespa 8 - public DocumentIdResponse(long requestId, String textMessage, boolean success) { - this(requestId, null, textMessage, success ? Outcome.SUCCESS : Outcome.ERROR); - } - - /** - * Creates a response containing a textual message and/or a document id - * - * @param documentId the DocumentId to encapsulate in the Response - * @param textMessage the message to encapsulate in the Response - * @param success true if the response represents a successful call - */ - @Deprecated(since = "7") // TODO: Remove on Vespa 8 - public DocumentIdResponse(long requestId, DocumentId documentId, String textMessage, boolean success) { - this(requestId, documentId, textMessage, success ? Outcome.SUCCESS : Outcome.ERROR); - } - - - /** * Creates a response containing a textual message and/or a document id * * @param documentId the DocumentId to encapsulate in the Response diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentOperationParameters.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentOperationParameters.java index 74cdcc02b94..9677aaf1905 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentOperationParameters.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentOperationParameters.java @@ -42,14 +42,6 @@ public class DocumentOperationParameters { return empty; } - /** Sets the priority with which to perform an operation. - * @deprecated specifying explicit operation priority is deprecated - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - public DocumentOperationParameters withPriority(DocumentProtocol.Priority priority) { - return new DocumentOperationParameters(requireNonNull(priority), fieldSet, route, traceLevel, deadline, responseHandler); - } - /** Sets the field set used for retrieval. */ public DocumentOperationParameters withFieldSet(FieldSet fieldSet) { return new DocumentOperationParameters(priority, new FieldSetRepo().serialize(fieldSet), route, traceLevel, deadline, responseHandler); @@ -83,11 +75,6 @@ public class DocumentOperationParameters { return new DocumentOperationParameters(priority, fieldSet, route, traceLevel, deadline, requireNonNull(responseHandler)); } - /** - * @deprecated explicit operation priority is deprecated - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - public Optional<DocumentProtocol.Priority> priority() { return Optional.ofNullable(priority); } public Optional<String> fieldSet() { return Optional.ofNullable(fieldSet); } public Optional<String> route() { return Optional.ofNullable(route); } public OptionalInt traceLevel() { return traceLevel >= 0 ? OptionalInt.of(traceLevel) : OptionalInt.empty(); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java index 4b17fef7f59..4bc5fb400a7 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentResponse.java @@ -39,31 +39,6 @@ public class DocumentResponse extends Response { } /** - * Creates a response containing a textual message - * - * @param textMessage the message to encapsulate in the Response - * @param success true if the response represents a successful call - */ - @Deprecated(since = "7") // TODO: Remove on Vespa 8 - public DocumentResponse(long requestId, String textMessage, boolean success) { - super(requestId, textMessage, success ? Outcome.NOT_FOUND : Outcome.ERROR); - document = null; - } - - /** - * Creates a response containing a textual message and/or a document - * - * @param document the Document to encapsulate in the Response - * @param textMessage the message to encapsulate in the Response - * @param success true if the response represents a successful call - */ - @Deprecated(since = "7") // TODO: Remove on Vespa 8 - public DocumentResponse(long requestId, Document document, String textMessage, boolean success) { - this(requestId, document, textMessage, success ? Outcome.SUCCESS : Outcome.ERROR); - } - - - /** * Creates a response containing a textual message and/or a document * * @param document the Document to encapsulate in the Response diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentUpdateResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentUpdateResponse.java index d9c286446ac..109ec2523fe 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentUpdateResponse.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentUpdateResponse.java @@ -34,17 +34,6 @@ public class DocumentUpdateResponse extends Response { * Creates a response containing a textual message * * @param textMessage the message to encapsulate in the Response - * @param success true if the response represents a successful call - */ - @Deprecated(since = "7") // TODO: Remove on Vespa 8 - public DocumentUpdateResponse(long requestId, String textMessage, boolean success) { - this(requestId, null, textMessage, success ? Outcome.SUCCESS : Outcome.ERROR); - } - - /** - * Creates a response containing a textual message - * - * @param textMessage the message to encapsulate in the Response * @param outcome the outcome of this operation */ public DocumentUpdateResponse(long requestId, String textMessage, Outcome outcome) { @@ -56,19 +45,6 @@ public class DocumentUpdateResponse extends Response { * * @param documentUpdate the DocumentUpdate to encapsulate in the Response * @param textMessage the message to encapsulate in the Response - * @param success true if the response represents a successful call - */ - @Deprecated(since = "7") // TODO: Remove on Vespa 8 - public DocumentUpdateResponse(long requestId, DocumentUpdate documentUpdate, String textMessage, boolean success) { - this(requestId, documentUpdate, textMessage, success ? Outcome.SUCCESS : Outcome.ERROR); - } - - - /** - * Creates a response containing a textual message and/or a document update - * - * @param documentUpdate the DocumentUpdate to encapsulate in the Response - * @param textMessage the message to encapsulate in the Response * @param outcome the outcome of this operation */ public DocumentUpdateResponse(long requestId, DocumentUpdate documentUpdate, String textMessage, Outcome outcome) { diff --git a/documentapi/src/main/java/com/yahoo/documentapi/Response.java b/documentapi/src/main/java/com/yahoo/documentapi/Response.java index aee9dc010ab..3778189e272 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/Response.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/Response.java @@ -41,17 +41,6 @@ public class Response { * Creates a response containing a textual message * * @param textMessage the message to encapsulate in the Response - * @param success true if the response represents a successful call - */ - @Deprecated(since = "7") // TODO: Remove on Vespa 8 - public Response(long requestId, String textMessage, boolean success) { - this(requestId, textMessage, success ? SUCCESS : ERROR); - } - - /** - * Creates a response containing a textual message - * - * @param textMessage the message to encapsulate in the Response * @param outcome the outcome of the operation */ public Response(long requestId, String textMessage, Outcome outcome) { diff --git a/documentapi/src/main/java/com/yahoo/documentapi/Result.java b/documentapi/src/main/java/com/yahoo/documentapi/Result.java index 6c83797cd89..76891b90352 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/Result.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/Result.java @@ -40,17 +40,6 @@ public class Result { /** * Creates a unsuccessful result * - * @deprecated Will be removed on Vespa 8 due to incorrect java.lang.Error - */ - @Deprecated(forRemoval = true, since="7") - public Result(ResultType type, java.lang.Error error) { - this.type = type; - this.error = new Error(0, error.getMessage()); - this.requestId = 0; - } - /** - * Creates a unsuccessful result - * * @param type the type of failure * @param error the error to encapsulate in this Result * @see com.yahoo.documentapi.Result.ResultType @@ -71,20 +60,6 @@ public class Result { */ public boolean isSuccess() { return type == ResultType.SUCCESS; } - /** - * Returns the error causes by this. If this was not a success, this method always returns an error - * If this was a success, this method returns null. - * - * @return the Error, or null - * @deprecated Will be removed on Vespa 8 - */ - @Deprecated(forRemoval = true, since="7") - public java.lang.Error getError() { - return error != null - ? new java.lang.Error(error.getMessage()) - : null; - } - public Error error() { return error; } /** @@ -110,17 +85,13 @@ public class Result { /** The request failed, but may be successful if retried at a later time. */ TRANSIENT_ERROR, /** The request failed, and retrying is pointless. */ - FATAL_ERROR, - /** Condition specified in operation not met error */ - @Deprecated(since = "7", forRemoval = true) // TODO: Remove on Vespa 8 — this is a Response outcome, not a Result outcome. - CONDITION_NOT_MET_ERROR + FATAL_ERROR } + public static Error toError(ResultType result) { switch (result) { case TRANSIENT_ERROR: return new Error(ErrorCode.TRANSIENT_ERROR, ResultType.TRANSIENT_ERROR.name()); - case CONDITION_NOT_MET_ERROR: - return new Error(ErrorCode.FATAL_ERROR, ResultType.CONDITION_NOT_MET_ERROR.name()); case FATAL_ERROR: return new Error(ErrorCode.FATAL_ERROR, ResultType.FATAL_ERROR.name()); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java index 02a1634f8ec..01d67c8e1d9 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java @@ -34,19 +34,6 @@ public interface SyncSession extends Session { * Puts a document. When this method returns, the document is safely received. * * @param documentPut the DocumentPut operation - * @param priority the priority with which to perform this operation - * @deprecated specifying explicit operation priority is deprecated - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - default void put(DocumentPut documentPut, DocumentProtocol.Priority priority) { - put(documentPut, parameters().withPriority(priority)); - } - - /** - * Puts a document. When this method returns, the document is safely received. - * - * @param documentPut the DocumentPut operation * @param parameters parameters for the operation */ default void put(DocumentPut documentPut, DocumentOperationParameters parameters) { @@ -63,22 +50,6 @@ public interface SyncSession extends Session { default Document get(DocumentId id) { return get(id, null); } /** - * Gets a document with an unspecified timeout - * - * @param id the id of the document to get - * @param fieldSet a comma-separated list of fields to retrieve - * @param priority the priority with which to perform this operation - * @return the document with this id, or null if there is none - * @throws UnsupportedOperationException thrown if this does not support retrieving - * @deprecated specifying explicit operation priority is deprecated. Set fieldSet via - * {@link #get(DocumentId, DocumentOperationParameters, Duration)} instead. - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - default Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority priority) { - return get(id, fieldSet, priority, null); - } - - /** * Gets a document with timeout. * * @param id the id of the document to get @@ -93,21 +64,6 @@ public interface SyncSession extends Session { * Gets a document with timeout. * * @param id the id of the document to get - * @param fieldSet a comma-separated list of fields to retrieve - * @param priority the priority with which to perform this operation - * @param timeout timeout. If timeout is null, an unspecified default will be used - * @return the known document having this id, or null if there is no document having this id - * @throws UnsupportedOperationException thrown if this access does not support retrieving - * @throws DocumentAccessException on any messagebus error, including timeout ({@link com.yahoo.messagebus.ErrorCode#TIMEOUT}) - * @deprecated specifying explicit operation priority is deprecated - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority priority, Duration timeout); - - /** - * Gets a document with timeout. - * - * @param id the id of the document to get * @param parameters parameters for the operation * @param timeout timeout. If timeout is null, an unspecified default will be used * @return the known document having this id, or null if there is no document having this id @@ -130,18 +86,6 @@ public interface SyncSession extends Session { * Removes a document if it is present. * * @param documentRemove document remove operation - * @param priority the priority with which to perform this operation - * @return true if the document with this id was removed, false otherwise. - * @throws UnsupportedOperationException thrown if this access does not support removal - * @deprecated specifying explicit operation priority is deprecated - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - boolean remove(DocumentRemove documentRemove, DocumentProtocol.Priority priority); - - /** - * Removes a document if it is present. - * - * @param documentRemove document remove operation * @param parameters parameters for the operation * @return true if the document with this id was removed, false otherwise. * @throws UnsupportedOperationException thrown if this access does not support removal @@ -166,21 +110,6 @@ public interface SyncSession extends Session { * Updates a document. * * @param update the updates to perform. - * @param priority the priority with which to perform this operation - * @return false if the updates could not be applied as the document does not exist and - * {@link DocumentUpdate#setCreateIfNonExistent(boolean) create-if-non-existent} is not set. - * @throws DocumentAccessException on update error, including but not limited to: 1. timeouts, - * 2. the document exists but the {@link DocumentUpdate#setCondition(TestAndSetCondition) condition} - * is not met. - * @deprecated specifying explicit operation priority is deprecated - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - boolean update(DocumentUpdate update, DocumentProtocol.Priority priority); - - /** - * Updates a document. - * - * @param update the updates to perform. * @param parameters parameters for the operation * @return false if the updates could not be applied as the document does not exist and * {@link DocumentUpdate#setCreateIfNonExistent(boolean) create-if-non-existent} is not set. diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java index c76a25f5c22..d22d7ecb550 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java @@ -3,10 +3,8 @@ package com.yahoo.documentapi; import com.yahoo.document.BucketId; import com.yahoo.document.FixedBucketSpaces; -import com.yahoo.document.fieldset.AllFields; -import com.yahoo.documentapi.messagebus.loadtypes.LoadType; +import com.yahoo.document.fieldset.DocumentOnly; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.messagebus.ThrottlePolicy; import com.yahoo.messagebus.routing.Route; import com.yahoo.text.Utf8; @@ -19,7 +17,6 @@ import java.util.TreeMap; * * @author Håkon Humberset */ -@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public class VisitorParameters extends Parameters { private String documentSelection; @@ -31,8 +28,7 @@ public class VisitorParameters extends Parameters { private long fromTimestamp = 0; private long toTimestamp = 0; boolean visitRemoves = false; - // TODO Vespa 8: change to DocumentOnly.NAME; - private String fieldSet = AllFields.NAME; + private String fieldSet = DocumentOnly.NAME; boolean visitInconsistentBuckets = false; private ProgressToken resumeToken = null; private String resumeFileName = ""; @@ -42,12 +38,8 @@ public class VisitorParameters extends Parameters { private Map<String, byte []> libraryParameters = new TreeMap<>(); private Route visitRoute = null; private final float weight = 1; - private long maxFirstPassHits = -1; private long maxTotalHits = -1; private int maxBucketsPerVisitor = 1; - private boolean dynamicallyIncreaseMaxBucketsPerVisitor = false; - private float dynamicMaxBucketsIncreaseFactor = 2; - private LoadType loadType = LoadType.DEFAULT; // TODO: Remove on Vespa 8 private DocumentProtocol.Priority priority = null; private int traceLevel = 0; private boolean skipBucketsOnFatalErrors = false; @@ -93,15 +85,9 @@ public class VisitorParameters extends Parameters { setLocalDataHandler(params.getLocalDataHandler()); } setControlHandler(params.getControlHandler()); - setMaxFirstPassHits(params.getMaxFirstPassHits()); setMaxTotalHits(params.getMaxTotalHits()); setMaxBucketsPerVisitor(params.getMaxBucketsPerVisitor()); - setLoadType(params.getLoadType()); setPriority(params.getPriority()); - setDynamicallyIncreaseMaxBucketsPerVisitor( - params.getDynamicallyIncreaseMaxBucketsPerVisitor()); - setDynamicMaxBucketsIncreaseFactor( - params.getDynamicMaxBucketsIncreaseFactor()); setTraceLevel(params.getTraceLevel()); skipBucketsOnFatalErrors(params.skipBucketsOnFatalErrors()); slice(params.getSlices(), getSliceId()); @@ -109,89 +95,69 @@ public class VisitorParameters extends Parameters { // Get functions - // TODO: s/@return/Returns/ - this javadoc will not contain text in the method overview - - /** @return The selection string used for visiting. */ + /** Returns the selection string used for visiting. */ public String getDocumentSelection() { return documentSelection; } - /** @return The bucket space to visit */ + /** Returns the bucket space to visit */ public String getBucketSpace() { return bucketSpace; } - /** @return What visitor library to use for the visiting. The library in question must be installed on each storage node in the target cluster. */ + /** Returns what visitor library to use for the visiting. The library in question must be installed on each storage node in the target cluster. */ public String getVisitorLibrary() { return visitorLibrary; } - /** @return The maximum number of messages each storage visitor will have pending before waiting for acks from client. */ + /** Returns the maximum number of messages each storage visitor will have pending before waiting for acks from client. */ public int getMaxPending() { return maxPending; } - /** @return The timeout for each sent visitor operation in milliseconds. */ + /** Returns the timeout for each sent visitor operation in milliseconds. */ public long getTimeoutMs() { return timeoutMs; } /** - * @return Session timeout in milliseconds, or -1 if not timeout has been set. -1 implies - * that session will run to completion without automatically timing out. + * Returns session timeout in milliseconds, or -1 if not timeout has been set. -1 implies + * that session will run to completion without automatically timing out. */ public long getSessionTimeoutMs() { return sessionTimeoutMs; } - /** @return The minimum timestamp (in microsecs) of documents the visitor will visit. */ + /** Returns the minimum timestamp (in microsecs) of documents the visitor will visit. */ public long getFromTimestamp() { return fromTimestamp; } - /** @return The maximum timestamp (in microsecs) of documents the visitor will visit. */ + /** Returns the maximum timestamp (in microsecs) of documents the visitor will visit. */ public long getToTimestamp() { return toTimestamp; } - /** @return If this method returns true, the visitor will visit remove entries as well as documents (you can see what documents have been deleted). */ + /** Returns if this method returns true, the visitor will visit remove entries as well as documents (you can see what documents have been deleted). */ public boolean visitRemoves() { return visitRemoves; } public boolean getVisitRemoves() { return visitRemoves; } - @Deprecated // TODO: Remove on Vespa 8 - public boolean getVisitHeadersOnly() { return false; } - - /** @return The field set to use. */ + /** Returns the field set to use. */ public String fieldSet() { return fieldSet; } public String getFieldSet() { return fieldSet; } - /** @return If this method returns true, the visitor will visit inconsistent buckets. */ + /** Returns if this method returns true, the visitor will visit inconsistent buckets. */ public boolean visitInconsistentBuckets() { return visitInconsistentBuckets; } public boolean getVisitInconsistentBuckets() { return visitInconsistentBuckets; } - /** @return Returns a map of string → string of arguments that are passed to the visitor library. */ + /** Returns a map of string → string of arguments that are passed to the visitor library. */ public Map<String, byte []> getLibraryParameters() { return libraryParameters; } - /** @return The progress token, which can be used to resume visitor. */ + /** Returns the progress token, which can be used to resume visitor. */ public ProgressToken getResumeToken() { return resumeToken; } - /** @return The filename for reading/storing progress token. */ + /** Returns the filename for reading/storing progress token. */ public String getResumeFileName() { return resumeFileName; } - /** @return Address to the remote data handler. */ + /** Returns address to the remote data handler. */ public String getRemoteDataHandler() { return remoteDataHandler; } - /** @return The local data handler. */ + /** Returns the local data handler. */ public VisitorDataHandler getLocalDataHandler() { return localDataHandler; } - /** @return The control handler. */ + /** Returns the control handler. */ public VisitorControlHandler getControlHandler() { return controlHandler; } - /** @return Whether or not max buckets per visitor value should be dynamically - * increased when using orderdoc and visitors do not return at least half - * the desired amount of documents - */ - public boolean getDynamicallyIncreaseMaxBucketsPerVisitor() { - return dynamicallyIncreaseMaxBucketsPerVisitor; - } - - /** @return Factor with which max buckets are dynamically increased each time */ - public float getDynamicMaxBucketsIncreaseFactor() { - return dynamicMaxBucketsIncreaseFactor; - } - public DocumentProtocol.Priority getPriority() { if (priority != null) { return priority; - } else if (loadType != null) { - return loadType.getPriority(); } else { return DocumentProtocol.Priority.NORMAL_3; } @@ -199,23 +165,23 @@ public class VisitorParameters extends Parameters { // Set functions - /** Set the document selection expression */ + /** Sets the document selection expression */ public void setDocumentSelection(String selection) { documentSelection = selection; } - /** Set which (single) bucket space this visiting will be against. */ + /** Sets which (single) bucket space this visiting will be against. */ public void setBucketSpace(String bucketSpace) { this.bucketSpace = bucketSpace; } - /** Set which visitor library is used for visiting in storage. DumpVisitor is most common implementation. */ + /** Sets which visitor library is used for visiting in storage. DumpVisitor is most common implementation. */ public void setVisitorLibrary(String library) { visitorLibrary = library; } - /** Set maximum pending messages one storage visitor will have pending to this client before stalling, waiting for acks. */ + /** Sets maximum pending messages one storage visitor will have pending to this client before stalling, waiting for acks. */ public void setMaxPending(int maxPending) { this.maxPending = maxPending; } - /** Set the timeout for each visitor command in milliseconds. */ + /** Sets the timeout for each visitor command in milliseconds. */ public void setTimeoutMs(long timeoutMs) { this.timeoutMs = timeoutMs; } /** - * Sets timeout for the entire visiting session, in milliseconds. -1 implies infinity. + * Setss timeout for the entire visiting session, in milliseconds. -1 implies infinity. * * If the session takes more time than this to complete, it will automatically * be failed with CompletionCode.TIMEOUT. @@ -224,87 +190,69 @@ public class VisitorParameters extends Parameters { */ public void setSessionTimeoutMs(long timeoutMs) { this.sessionTimeoutMs = timeoutMs; } - /** Set from timestamp in microseconds. Documents put/updated before this timestamp will not be visited. */ + /** Sets from timestamp in microseconds. Documents put/updated before this timestamp will not be visited. */ public void setFromTimestamp(long timestamp) { fromTimestamp = timestamp; } - /** Set to timestamp in microseconds. Documents put/updated after this timestamp will not be visited. */ + /** Sets to timestamp in microseconds. Documents put/updated after this timestamp will not be visited. */ public void setToTimestamp(long timestamp) { toTimestamp = timestamp; } - /** Set whether to visit remove entries. That is, entries saying that some document has been removed. */ + /** Sets whether to visit remove entries. That is, entries saying that some document has been removed. */ public void visitRemoves(boolean visitRemoves) { this.visitRemoves = visitRemoves; } public void setVisitRemoves(boolean visitRemoves) { this.visitRemoves = visitRemoves; } - /** @deprecated this option is ignored */ - @Deprecated // TODO: Remove on Vespa 8 - public void setVisitHeadersOnly(boolean headersOnly) { } - - /** Set field set to use. */ + /** Sets field set to use. */ public void fieldSet(String fieldSet) { this.fieldSet = fieldSet; } public void setFieldSet(String fieldSet) { this.fieldSet = fieldSet; } - /** Set whether to visit inconsistent buckets. */ + /** Sets whether to visit inconsistent buckets. */ public void visitInconsistentBuckets(boolean visitInconsistentBuckets) { this.visitInconsistentBuckets = visitInconsistentBuckets; } public void setVisitInconsistentBuckets(boolean visitInconsistentBuckets) { this.visitInconsistentBuckets = visitInconsistentBuckets; } - /** Set a visitor library specific parameter. */ + /** Sets a visitor library specific parameter. */ public void setLibraryParameter(String param, String value) { libraryParameters.put(param, Utf8.toBytes(value)); } - /** Set a visitor library specific parameter. */ + /** Sets a visitor library specific parameter. */ public void setLibraryParameter(String param, byte [] value) { libraryParameters.put(param, value); } - /** Set all visitor library specific parameters. */ + /** Sets all visitor library specific parameters. */ public void setLibraryParameters(Map<String, byte []> params) { libraryParameters = params; } - /** Set progress token, which can be used to resume visitor. */ + /** Sets progress token, which can be used to resume visitor. */ public void setResumeToken(ProgressToken token) { resumeToken = token; } /** - * Set filename for reading/storing progress token. If the file exists and + * Sets filename for reading/storing progress token. If the file exists and * contains progress data, visitor should resume visiting from this point. */ public void setResumeFileName(String fileName) { resumeFileName = fileName; } - /** Set address for the remote data handler. */ + /** Sets address for the remote data handler. */ public void setRemoteDataHandler(String remoteDataHandler) { this.remoteDataHandler = remoteDataHandler; localDataHandler = null; } - /** Set local data handler. */ + /** Sets local data handler. */ public void setLocalDataHandler(VisitorDataHandler localDataHandler) { this.localDataHandler = localDataHandler; remoteDataHandler = null; } - /** Set control handler. */ + /** Sets control handler. */ public void setControlHandler(VisitorControlHandler controlHandler) { this.controlHandler = controlHandler; } - /** Set the name of the storage cluster route to visit. Default is "storage/cluster.storage". */ + /** Sets the name of the storage cluster route to visit. Default is "storage/cluster.storage". */ public void setRoute(String route) { setRoute(Route.parse(route)); } - /** Set the route to visit. */ + /** Sets the route to visit. */ public void setRoute(Route route) { visitRoute = route; } - /** @return Returns the name of the storage cluster to visit. */ + /** Returns the name of the storage cluster to visit. */ // TODO: Document: Where is the default - does this ever return null, or does it return "storage" if input is null? public Route getRoute() { return visitRoute; } - /** Set the maximum number of documents to visit (max documents returned by the visitor) - * - * @deprecated use setMaxTotalHits instead - */ - @Deprecated(since = "7", forRemoval = true) // TODO: Vespa 8: remove - public void setMaxFirstPassHits(long max) { maxFirstPassHits = max; } - - /** @return Returns the maximum number of documents to visit (max documents returned by the visitor) - * - * @deprecated Use getMaxTotalHits() instead - */ - @Deprecated(since = "7", forRemoval = true) // TODO: Vespa 8: remove - public long getMaxFirstPassHits() { return maxFirstPassHits; } - - /** Set the maximum number of documents to visit (max documents returned by the visitor) */ + /** Sets the maximum number of documents to visit (max documents returned by the visitor) */ public void setMaxTotalHits(long max) { maxTotalHits = max; } - /** @return Returns the maximum number of documents to visit (max documents returned by the visitor) */ + /** Returns the maximum number of documents to visit (max documents returned by the visitor) */ public long getMaxTotalHits() { return maxTotalHits; } public Set<BucketId> getBucketsToVisit() { return bucketsToVisit; } @@ -323,22 +271,6 @@ public class VisitorParameters extends Parameters { this.priority = priority; } - /** - * @deprecated load types are deprecated - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - public void setLoadType(LoadType loadType) { - this.loadType = loadType; - } - - /** - * @deprecated load types are deprecated - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - public LoadType getLoadType() { - return loadType; - } - public boolean skipBucketsOnFatalErrors() { return skipBucketsOnFatalErrors; } public void skipBucketsOnFatalErrors(boolean skipBucketsOnFatalErrors) { this.skipBucketsOnFatalErrors = skipBucketsOnFatalErrors; } @@ -352,28 +284,7 @@ public class VisitorParameters extends Parameters { public int getSliceId() { return sliceId; } - /** - * Set whether or not max buckets per visitor value should be dynamically - * increased when using orderdoc and visitors do not return at least half - * the desired amount of documents - * - * @param dynamicallyIncreaseMaxBucketsPerVisitor whether or not to increase - * - * @deprecated deprecated feature, will be removed on Vespa 8 - */ - @Deprecated(since = "7", forRemoval = true) // TODO: Vespa 8: remove - public void setDynamicallyIncreaseMaxBucketsPerVisitor(boolean dynamicallyIncreaseMaxBucketsPerVisitor) { - this.dynamicallyIncreaseMaxBucketsPerVisitor = dynamicallyIncreaseMaxBucketsPerVisitor; - } - - /** - * Set factor with which max buckets are dynamically increased each time - * @param dynamicMaxBucketsIncreaseFactor increase factor (must be 1 or more) - */ - public void setDynamicMaxBucketsIncreaseFactor(float dynamicMaxBucketsIncreaseFactor) { - this.dynamicMaxBucketsIncreaseFactor = dynamicMaxBucketsIncreaseFactor; - } - + @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("VisitorParameters(\n") @@ -399,18 +310,12 @@ public class VisitorParameters extends Parameters { sb.append(" Field set: ").append(fieldSet).append('\n'); sb.append(" Route: ").append(visitRoute).append('\n'); sb.append(" Weight: ").append(weight).append('\n'); - sb.append(" Max firstpass hits: ").append(maxFirstPassHits).append('\n'); sb.append(" Max total hits: ").append(maxTotalHits).append('\n'); sb.append(" Max buckets: ").append(maxBucketsPerVisitor).append('\n'); sb.append(" Priority: ").append(getPriority().toString()).append('\n'); - if (dynamicallyIncreaseMaxBucketsPerVisitor) { - sb.append(" Dynamically increasing max buckets per visitor\n"); - sb.append(" Increase factor: ") - .append(dynamicMaxBucketsIncreaseFactor) - .append('\n'); - } sb.append(')'); return sb.toString(); } + } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java index e4f615b7d63..62ad3a34a74 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java @@ -33,25 +33,11 @@ public class LocalSyncSession implements SyncSession { } @Override - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - public void put(DocumentPut documentPut, DocumentProtocol.Priority priority) { - access.documents.put(documentPut.getId(), documentPut.getDocument()); - } - - @Override public Document get(DocumentId id, Duration timeout) { return access.documents.get(id); } @Override - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - public Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority priority, Duration timeout) { - return access.documents.get(id); - } - - @Override public boolean remove(DocumentRemove documentRemove) { if (documentRemove.getCondition().isPresent()) { throw new UnsupportedOperationException("test-and-set is not supported."); @@ -61,13 +47,6 @@ public class LocalSyncSession implements SyncSession { } @Override - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - public boolean remove(DocumentRemove documentRemove, DocumentProtocol.Priority priority) { - return remove(documentRemove); - } - - @Override public boolean update(DocumentUpdate update) { Document document = access.documents.get(update.getId()); if (document == null) { @@ -78,18 +57,6 @@ public class LocalSyncSession implements SyncSession { } @Override - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - public boolean update(DocumentUpdate update, DocumentProtocol.Priority pri) { - Document document = access.documents.get(update.getId()); - if (document == null) { - return false; - } - update.applyTo(document); - return true; - } - - @Override public Response getNext() { throw new UnsupportedOperationException("Queue not supported."); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java index 8809e05caf3..f8cb71d7ba8 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java @@ -6,7 +6,7 @@ import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentUpdate; -import com.yahoo.document.fieldset.AllFields; +import com.yahoo.document.fieldset.DocumentOnly; import com.yahoo.documentapi.AsyncParameters; import com.yahoo.documentapi.AsyncSession; import com.yahoo.documentapi.DocumentIdResponse; @@ -110,10 +110,8 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { } @Override - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public Result put(DocumentPut documentPut, DocumentOperationParameters parameters) { PutDocumentMessage msg = new PutDocumentMessage(documentPut); - msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_3)); return send(msg, parameters); } @@ -123,18 +121,8 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { } @Override - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - public Result get(DocumentId id, boolean headersOnly, DocumentProtocol.Priority pri) { - return get(id, pri); - } - - @Override - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public Result get(DocumentId id, DocumentOperationParameters parameters) { - // TODO Vespa 8: change to DocumentOnly.NAME - GetDocumentMessage msg = new GetDocumentMessage(id, parameters.fieldSet().orElse(AllFields.NAME)); - msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_1)); + GetDocumentMessage msg = new GetDocumentMessage(id, parameters.fieldSet().orElse(DocumentOnly.NAME)); return send(msg, parameters); } @@ -144,10 +132,8 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { } @Override - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public Result remove(DocumentRemove remove, DocumentOperationParameters parameters) { RemoveDocumentMessage msg = new RemoveDocumentMessage(remove); - msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_2)); return send(msg, parameters); } @@ -157,10 +143,8 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { } @Override - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public Result update(DocumentUpdate update, DocumentOperationParameters parameters) { UpdateDocumentMessage msg = new UpdateDocumentMessage(update); - msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_2)); return send(msg, parameters); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java index c2e6dde7f60..3b16def1f1c 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java @@ -53,13 +53,12 @@ public class MessageBusDocumentAccess extends DocumentAccess { * * @param params All parameters for construction. */ - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public MessageBusDocumentAccess(MessageBusParams params) { super(params); this.params = params; try { com.yahoo.messagebus.MessageBusParams mbusParams = new com.yahoo.messagebus.MessageBusParams(params.getMessageBusParams()); - mbusParams.addProtocol(new DocumentProtocol(getDocumentTypeManager(), params.getProtocolConfigId(), params.getLoadTypes())); + mbusParams.addProtocol(new DocumentProtocol(getDocumentTypeManager(), params.getProtocolConfigId())); if (System.getProperty("vespa.local", "false").equals("true")) { // set by Application when running locally LocalNetwork network = new LocalNetwork(); bus = new NetworkMessageBus(network, new MessageBus(network, mbusParams)); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java index bb8c3a3b1b1..c0e5b80dbff 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java @@ -2,7 +2,6 @@ package com.yahoo.documentapi.messagebus; import com.yahoo.documentapi.DocumentAccessParams; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.messagebus.SourceSessionParams; @@ -14,7 +13,6 @@ import static java.util.Objects.requireNonNull; /** * @author Einar M R Rosenvinge */ -@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public class MessageBusParams extends DocumentAccessParams { private String routingConfigId = null; @@ -27,29 +25,8 @@ public class MessageBusParams extends DocumentAccessParams { private RPCNetworkParams rpcNetworkParams = new RPCNetworkParams(); private com.yahoo.messagebus.MessageBusParams mbusParams = new com.yahoo.messagebus.MessageBusParams(); private SourceSessionParams sourceSessionParams = new SourceSessionParams(); - private LoadTypeSet loadTypes; // TODO remove on Vespa 8 - public MessageBusParams() { - this(new LoadTypeSet()); - } - - /** - * @deprecated load types are deprecated. Use default constructor instead - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - public MessageBusParams(LoadTypeSet loadTypes) { - this.loadTypes = loadTypes; - } - - /** - * - * @return Returns the set of load types accepted by this Vespa installation - * @deprecated load types are deprecated - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - public LoadTypeSet getLoadTypes() { - return loadTypes; - } + public MessageBusParams() {} /** * Returns the id to resolve to routing config. diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java index 1b0a6db3d53..bc6451b4b3a 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusSyncSession.java @@ -6,7 +6,7 @@ import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentUpdate; -import com.yahoo.document.fieldset.AllFields; +import com.yahoo.document.fieldset.DocumentOnly; import com.yahoo.documentapi.AsyncParameters; import com.yahoo.documentapi.DocumentAccessException; import com.yahoo.documentapi.DocumentOperationParameters; @@ -126,17 +126,8 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re } @Override - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - public void put(DocumentPut documentPut, DocumentProtocol.Priority priority) { - put(documentPut, parameters().withPriority(priority)); - } - - @Override - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public void put(DocumentPut documentPut, DocumentOperationParameters parameters) { PutDocumentMessage msg = new PutDocumentMessage(documentPut); - msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_3)); Reply reply = syncSend(msg, parameters); if (reply.hasErrors()) { throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply), reply.getErrorCodes()); @@ -149,18 +140,8 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re } @Override - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - public Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority pri, Duration timeout) { - return get(id, parameters().withFieldSet(fieldSet).withPriority(pri), timeout); - } - - @Override - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public Document get(DocumentId id, DocumentOperationParameters parameters, Duration timeout) { - // TODO Vespa 8: change to DocumentOnly.NAME - GetDocumentMessage msg = new GetDocumentMessage(id, parameters.fieldSet().orElse(AllFields.NAME)); - msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_1)); + GetDocumentMessage msg = new GetDocumentMessage(id, parameters.fieldSet().orElse(DocumentOnly.NAME)); Reply reply = syncSend(msg, timeout != null ? timeout : defaultTimeout, parameters); if (reply.hasErrors()) { @@ -183,17 +164,8 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re } @Override - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - public boolean remove(DocumentRemove documentRemove, DocumentProtocol.Priority pri) { - return remove(documentRemove, parameters().withPriority(pri)); - } - - @Override - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public boolean remove(DocumentRemove documentRemove, DocumentOperationParameters parameters) { RemoveDocumentMessage msg = new RemoveDocumentMessage(documentRemove.getId()); - msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_2)); msg.setCondition(documentRemove.getCondition()); Reply reply = syncSend(msg, parameters); if (reply.hasErrors()) { @@ -211,17 +183,8 @@ public class MessageBusSyncSession implements MessageBusSession, SyncSession, Re } @Override - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - public boolean update(DocumentUpdate update, DocumentProtocol.Priority pri) { - return update(update, parameters().withPriority(pri)); - } - - @Override - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public boolean update(DocumentUpdate update, DocumentOperationParameters parameters) { UpdateDocumentMessage msg = new UpdateDocumentMessage(update); - msg.setPriority(parameters.priority().orElse(DocumentProtocol.Priority.NORMAL_2)); Reply reply = syncSend(msg, parameters); if (reply.hasErrors()) { throw new DocumentAccessException(MessageBusAsyncSession.getErrorMessage(reply), reply.getErrorCodes()); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java index 5e4e87759cb..3607f652a51 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java @@ -638,7 +638,7 @@ public class MessageBusVisitorSession implements VisitorSession { return sb.toString(); } - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 + @SuppressWarnings("removal") // TODO: Remove on Vespa 9 private CreateVisitorMessage createMessage(VisitorIterator.BucketProgress bucket) { CreateVisitorMessage msg = new CreateVisitorMessage( params.getVisitorLibrary(), @@ -660,8 +660,7 @@ public class MessageBusVisitorSession implements VisitorSession { msg.setParameters(params.getLibraryParameters()); msg.setRoute(params.getRoute()); msg.setMaxBucketsPerVisitor(params.getMaxBucketsPerVisitor()); - msg.setLoadType(params.getLoadType()); - msg.setPriority(params.getPriority()); // TODO: remove on Vespa 8 + msg.setPriority(params.getPriority()); // TODO: remove on Vespa 9 msg.setRetryEnabled(false); @@ -998,22 +997,8 @@ public class MessageBusVisitorSession implements VisitorSession { progress.getIterator().update(bucket, ProgressToken.FINISHED_BUCKET); } - @SuppressWarnings("removal") // TODO: Vespa 8: remove private boolean enoughHitsReceived() { - // TODO: Vespa 8: remove "Nth pass" concept entirely from API and internals - if (params.getMaxFirstPassHits() != -1 - && statistics.getDocumentsReturned() >= params.getMaxFirstPassHits()) - { - return true; - } - if (params.getMaxTotalHits() != -1 - && ((statistics.getDocumentsReturned() - + statistics.getSecondPassDocumentsReturned()) // TODO: Vespa 8: remove - >= params.getMaxTotalHits())) - { - return true; - } - return false; + return params.getMaxTotalHits() != -1 && (statistics.getDocumentsReturned() >= params.getMaxTotalHits()); } /** @@ -1088,7 +1073,6 @@ public class MessageBusVisitorSession implements VisitorSession { return scheduleSendCreateVisitorsIfApplicable(0, TimeUnit.MILLISECONDS); } - @SuppressWarnings("removal")// TODO: Vespa 8: remove private void handleCreateVisitorReply(CreateVisitorReply reply) { CreateVisitorMessage msg = (CreateVisitorMessage)reply.getMessage(); @@ -1108,19 +1092,6 @@ public class MessageBusVisitorSession implements VisitorSession { trace.getRoot().addChild(reply.getTrace().getRoot()); } - // TODO: Vespa 8 remove this unused functionality - if (params.getDynamicallyIncreaseMaxBucketsPerVisitor() - && (reply.getVisitorStatistics().getDocumentsReturned() - < params.getMaxFirstPassHits() / 2.0)) - { - // Attempt to increase parallelism to reduce latency of visiting - // Ensure new count is within [1, 128] - int newMaxBuckets = Math.max(Math.min((int)(params.getMaxBucketsPerVisitor() - * params.getDynamicMaxBucketsIncreaseFactor()), 128), 1); - params.setMaxBucketsPerVisitor(newMaxBuckets); - log.log(Level.FINE, () -> sessionName + ": increasing max buckets per visitor to " - + params.getMaxBucketsPerVisitor()); - } } private void handleWrongDistributionReply(WrongDistributionReply reply) { @@ -1248,4 +1219,5 @@ public class MessageBusVisitorSession implements VisitorSession { log.log(Level.FINE, () -> sessionName + ": synchronous destroy() done"); } } + } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadType.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadType.java deleted file mode 100644 index 133736a8542..00000000000 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadType.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.loadtypes; - -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; - -/** - * @deprecated load types are deprecated - * @author thomasg - */ -@Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 -public class LoadType { - private int id; - private String name; - private DocumentProtocol.Priority priority; - - public static LoadType DEFAULT = new LoadType(0, "default", DocumentProtocol.Priority.NORMAL_3); - - public LoadType(int id, String name, DocumentProtocol.Priority priority) { - this.id = id; - this.name = name; - this.priority = priority; - } - - @Override - public boolean equals(Object other) { - if (!(other instanceof LoadType)) { - return false; - } - - LoadType o = (LoadType)other; - - return name.equals(o.getName()) && id == o.getId() && priority == o.getPriority(); - } - - @Override - public int hashCode() { - return java.util.Objects.hash(name, id, priority); - } - - public String getName() { return name; } - - public String toString() { return name + " (id " + id + ")"; } - - public DocumentProtocol.Priority getPriority() { return priority; } - - public int getId() { return id; } -} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java deleted file mode 100644 index a3fbed472f0..00000000000 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.loadtypes; - -import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.HashMap; -import java.util.Map; -import java.util.TreeMap; - -/** - * This class keeps track of all configured load types. - * - * For production use, you should only use the String constructor, - * and supply a valid config id. Only the load types configured will - * be propagated throughout the system, so there is no point in using other - * load types. - * - * For testing, you may want to use the empty constructor and add - * load types yourself with addType(). - * - * @deprecated load types are deprecated - */ -@Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 -@SuppressWarnings("removal") // TODO: Remove on Vespa 8 -public class LoadTypeSet { - - public static final LoadTypeSet EMPTY = new LoadTypeSet(); - - class DualMap { - Map<String, LoadType> nameMap = new TreeMap<String, LoadType>(); - Map<Integer, LoadType> idMap = new HashMap<Integer, LoadType>(); - - void put(LoadType l) { - if (nameMap.containsKey(l.getName()) || idMap.containsKey(l.getId())) { - throw new IllegalArgumentException( - "ID or name conflict when adding " + l); - } - - nameMap.put(l.getName(), l); - idMap.put(l.getId(), l); - } - } - - DualMap map; - - public LoadTypeSet() { - map = new DualMap(); - map.put(LoadType.DEFAULT); - } - - public LoadTypeSet(String configId) { - configure(new ConfigGetter<>(LoadTypeConfig.class).getConfig(configId)); - } - - public LoadTypeSet(LoadTypeConfig loadTypeConfig) { - configure(loadTypeConfig); - } - - public Map<String, LoadType> getNameMap() { - return map.nameMap; - } - - public Map<Integer, LoadType> getIdMap() { - return map.idMap; - } - - /** - * Used by config to generate priorities for a name, and add them to the load type set. - */ - public void addType(String name, String priority) { - try { - MessageDigest algorithm = MessageDigest.getInstance("MD5"); - algorithm.reset(); - algorithm.update(name.getBytes()); - byte messageDigest[] = algorithm.digest(); - - int id = 0; - for (int i = 0; i < 4; i++) { - int temp = ((int)messageDigest[i] & 0xff); - id <<= 8; - id |= temp; - } - - map.put(new LoadType(id, name, DocumentProtocol.Priority.valueOf(priority != null ? priority : "NORMAL_3"))); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } - } - - public void addLoadType(int id, String name, DocumentProtocol.Priority priority) { - map.put(new LoadType(id, name, priority)); - } - - public void configure(LoadTypeConfig config) { - DualMap newMap = new DualMap(); - - // Default should always be available. - newMap.put(LoadType.DEFAULT); - - for (LoadTypeConfig.Type t : config.type()) { - newMap.put(new LoadType(t.id(), t.name(), DocumentProtocol.Priority.valueOf(t.priority()))); - } - - map = newMap; - } -} - diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/package-info.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/package-info.java deleted file mode 100644 index be0f91f0f29..00000000000 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.documentapi.messagebus.loadtypes; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/CreateVisitorMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/CreateVisitorMessage.java index 097632f609f..2ba764cd504 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/CreateVisitorMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/CreateVisitorMessage.java @@ -3,7 +3,6 @@ package com.yahoo.documentapi.messagebus.protocol; import com.yahoo.document.BucketId; import com.yahoo.document.FixedBucketSpaces; -import com.yahoo.document.fieldset.AllFields; import com.yahoo.document.fieldset.DocumentOnly; import java.util.ArrayList; @@ -25,8 +24,7 @@ public class CreateVisitorMessage extends DocumentMessage { private long fromTime = 0; private long toTime = 0; private boolean visitRemoves = false; - // TODO Vespa 8: change to DocumentOnly.NAME - private String fieldSet = AllFields.NAME; + private String fieldSet = DocumentOnly.NAME; private boolean visitInconsistentBuckets = false; private Map<String, byte[]> params = new TreeMap<>(); private int maxBucketsPerVisitor = 1; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java index 21f7c243c6f..4ad017a1a83 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentMessage.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi.messagebus.protocol; -import com.yahoo.documentapi.messagebus.loadtypes.LoadType; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.Routable; import com.yahoo.text.Utf8String; @@ -9,11 +8,9 @@ import com.yahoo.text.Utf8String; /** * @author Simon Thoresen Hult */ -@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public abstract class DocumentMessage extends Message { private DocumentProtocol.Priority priority = DocumentProtocol.Priority.NORMAL_3; - private LoadType loadType = LoadType.DEFAULT; // TODO: Remove on Vespa 8 /** * Constructs a new message with no content. @@ -38,10 +35,6 @@ public abstract class DocumentMessage extends Message { DocumentProtocol.Priority pri = this.priority; this.priority = msg.priority; msg.priority = pri; - - LoadType lt = this.loadType; - this.loadType = msg.loadType; - msg.loadType = lt; } } @@ -52,7 +45,7 @@ public abstract class DocumentMessage extends Message { * @return The priority. * @deprecated explicit operation priority is deprecated */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 9 public DocumentProtocol.Priority getPriority() { return priority; } /** @@ -61,36 +54,14 @@ public abstract class DocumentMessage extends Message { * @param priority The priority to set. * @deprecated specifying explicit operation priority is deprecated */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 + @Deprecated(forRemoval = true) // TODO: Remove on Vespa 9 public void setPriority(DocumentProtocol.Priority priority) { this.priority = priority; } - /** - * @deprecated load types are deprecated - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - public LoadType getLoadType() { - return loadType; - } - - /** - * @deprecated load types are deprecated - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - public void setLoadType(LoadType loadType) { - if (loadType != null) { - this.loadType = loadType; - } else { - this.loadType = LoadType.DEFAULT; - } - } - @Override public int getApproxSize() { - return 4 + 1; // type + priority // TODO update on Vespa 8 to not include deprecated fields + return 4 + 1; // type + priority // TODO update on Vespa 9 to not include deprecated fields } @Override diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java index 5db426a5db4..0ff578b64d7 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java @@ -6,7 +6,6 @@ import com.yahoo.component.Version; import com.yahoo.component.VersionSpecification; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentTypeManagerConfigurer; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.messagebus.ErrorCode; import com.yahoo.messagebus.Protocol; import com.yahoo.messagebus.Reply; @@ -32,7 +31,6 @@ import static java.util.Objects.requireNonNull; * * @author Simon Thoresen Hult */ -@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public class DocumentProtocol implements Protocol { private static final Logger log = Logger.getLogger(DocumentProtocol.class.getName()); @@ -172,7 +170,7 @@ public class DocumentProtocol implements Protocol { */ public static final int ERROR_BUCKET_DELETED = ErrorCode.APP_TRANSIENT_ERROR + 1012; - /** Storage node received a timestamp that is stale. Likely clock skew. */ + /** Content node received a timestamp that is stale. Likely clock skew. */ public static final int ERROR_STALE_TIMESTAMP = ErrorCode.APP_TRANSIENT_ERROR + 1013; /** The given node have gotten a critical error and have suspended itself. */ @@ -240,39 +238,21 @@ public class DocumentProtocol implements Protocol { } public DocumentProtocol(DocumentTypeManager docMan) { - this(docMan, null, new LoadTypeSet()); - } - - public DocumentProtocol(DocumentTypeManager docMan, String configId) { - this(docMan, configId, new LoadTypeSet()); + this(docMan, null); } public DocumentProtocol(DocumentTypeManager documentTypeManager, DocumentProtocolPoliciesConfig policiesConfig, DistributionConfig distributionConfig) { - this(requireNonNull(documentTypeManager), null, new LoadTypeSet(), + this(requireNonNull(documentTypeManager), null, requireNonNull(policiesConfig), requireNonNull(distributionConfig)); } - /** - * @deprecated load types are deprecated. Use constructor without LoadTypeSet instead. - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - public DocumentProtocol(DocumentTypeManager documentTypeManager, LoadTypeSet loadTypes, - DocumentProtocolPoliciesConfig policiesConfig, DistributionConfig distributionConfig) { - this(requireNonNull(documentTypeManager), null, requireNonNull(loadTypes), - requireNonNull(policiesConfig), requireNonNull(distributionConfig)); - } - - /** - * @deprecated load types are deprecated. Use constructor without LoadTypeSet instead. - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - public DocumentProtocol(DocumentTypeManager docMan, String configId, LoadTypeSet set) { - this(docMan, configId == null ? "client" : configId, set, null, null); + public DocumentProtocol(DocumentTypeManager docMan, String configId) { + this(docMan, configId == null ? "client" : configId, null, null); } - private DocumentProtocol(DocumentTypeManager docMan, String configId, LoadTypeSet set, + private DocumentProtocol(DocumentTypeManager docMan, String configId, DocumentProtocolPoliciesConfig policiesConfig, DistributionConfig distributionConfig) { if (docMan != null) this.docMan = docMan; @@ -280,7 +260,7 @@ public class DocumentProtocol implements Protocol { this.docMan = new DocumentTypeManager(); DocumentTypeManagerConfigurer.configure(this.docMan, configId); } - this.routableRepository = new RoutableRepository(set); + this.routableRepository = new RoutableRepository(); // When adding factories to this list, please KEEP THEM ORDERED alphabetically like they are now. putRoutingPolicyFactory("AND", new RoutingPolicyFactories.AndPolicyFactory()); @@ -291,7 +271,6 @@ public class DocumentProtocol implements Protocol { putRoutingPolicyFactory("MessageType", new RoutingPolicyFactories.MessageTypePolicyFactory(configId, policiesConfig)); putRoutingPolicyFactory("RoundRobin", new RoutingPolicyFactories.RoundRobinPolicyFactory()); putRoutingPolicyFactory("LoadBalancer", new RoutingPolicyFactories.LoadBalancerPolicyFactory()); - putRoutingPolicyFactory("Storage", new RoutingPolicyFactories.ContentPolicyFactory(distributionConfig)); putRoutingPolicyFactory("SubsetService", new RoutingPolicyFactories.SubsetServicePolicyFactory()); // Prepare version specifications to use when adding routable factories. diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java index a785438da80..fbc8c9deb8e 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java @@ -23,11 +23,10 @@ import java.util.logging.Logger; * * @author Simon Thoresen Hult */ -@SuppressWarnings("removal") // TODO Vespa 8: remove public class DocumentRouteSelectorPolicy implements DocumentProtocolRoutingPolicy, ConfigSubscriber.SingleSubscriber<DocumentrouteselectorpolicyConfig> { - private static Logger log = Logger.getLogger(DocumentRouteSelectorPolicy.class.getName()); + private static final Logger log = Logger.getLogger(DocumentRouteSelectorPolicy.class.getName()); private Map<String, DocumentSelector> config; private String error = "Not configured."; private ConfigSubscriber subscriber; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentMessage.java index 0f6e738cb86..34ec26c2259 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentMessage.java @@ -2,7 +2,6 @@ package com.yahoo.documentapi.messagebus.protocol; import com.yahoo.document.DocumentId; -import com.yahoo.document.fieldset.AllFields; import com.yahoo.document.fieldset.DocumentOnly; import java.util.Arrays; @@ -12,8 +11,7 @@ import java.util.Arrays; */ public class GetDocumentMessage extends DocumentMessage { - // TODO Vespa 8: change to DocumentOnly.NAME - final static String DEFAULT_FIELD_SET = AllFields.NAME; + final static String DEFAULT_FIELD_SET = DocumentOnly.NAME; private DocumentId documentId = null; private String fieldSet = DEFAULT_FIELD_SET; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java index 095d1f433d2..57cd70cf4b4 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java @@ -15,7 +15,6 @@ import static java.util.stream.Collectors.toUnmodifiableMap; /** * @author baldersheim */ -@SuppressWarnings("removal") // TODO Vespa 8: remove public class MessageTypePolicy implements DocumentProtocolRoutingPolicy, ConfigSubscriber.SingleSubscriber<MessagetyperouteselectorpolicyConfig> { private final AtomicReference<Map<Integer, Route>> configRef = new AtomicReference<>(); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java index 42172b04b90..c4d4fb216be 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java @@ -9,7 +9,6 @@ import com.yahoo.document.DocumentUpdate; import com.yahoo.document.TestAndSetCondition; import com.yahoo.document.serialization.DocumentDeserializer; import com.yahoo.document.serialization.DocumentSerializer; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.messagebus.Routable; import com.yahoo.vdslib.DocumentSummary; import com.yahoo.vdslib.SearchResult; @@ -61,7 +60,7 @@ public abstract class RoutableFactories60 { */ protected abstract DocumentMessage doDecode(DocumentDeserializer deserializer); - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 + @SuppressWarnings("removal") // TODO: Remove on Vespa 9 public boolean encode(Routable obj, DocumentSerializer out) { if (!(obj instanceof DocumentMessage)) { throw new AssertionError( @@ -69,19 +68,18 @@ public abstract class RoutableFactories60 { "routable type " + obj.getType() + "(" + obj.getClass().getName() + ")."); } DocumentMessage msg = (DocumentMessage)obj; - out.putByte(null, (byte)(msg.getPriority().getValue())); // TODO: encode default value on Vespa 8 - out.putInt(null, msg.getLoadType().getId()); + out.putByte(null, (byte)(msg.getPriority().getValue())); // TODO: encode default value on Vespa 9 + out.putInt(null, 0); // Ignored load type. 0 is legacy "default" load type ID. return doEncode(msg, out); } - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - public Routable decode(DocumentDeserializer in, LoadTypeSet loadTypes) { - byte pri = in.getByte(null); // TODO: ignore on Vespa 8 - int loadType = in.getInt(null); + @SuppressWarnings("removal") // TODO: Remove on Vespa 9 + public Routable decode(DocumentDeserializer in) { + byte pri = in.getByte(null); // TODO: ignore on Vespa 9 + in.getInt(null); // Ignored load type DocumentMessage msg = doDecode(in); if (msg != null) { msg.setPriority(DocumentProtocol.getPriority(pri)); - msg.setLoadType(loadTypes.getIdMap().get(loadType)); // TODO: ignore on Vespa 8 } return msg; } @@ -131,8 +129,7 @@ public abstract class RoutableFactories60 { return doEncode(reply, out); } - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - public Routable decode(DocumentDeserializer in, LoadTypeSet loadTypes) { + public Routable decode(DocumentDeserializer in) { byte pri = in.getByte(null); DocumentReply reply = doDecode(in); if (reply != null) { @@ -235,7 +232,6 @@ public abstract class RoutableFactories60 { public static class CreateVisitorReplyFactory extends DocumentReplyFactory { @Override - @SuppressWarnings("removal")// TODO: Vespa 8: remove protected DocumentReply doDecode(DocumentDeserializer buf) { CreateVisitorReply reply = new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR); reply.setLastBucket(new BucketId(buf.getLong(null))); @@ -246,14 +242,13 @@ public abstract class RoutableFactories60 { vs.setBytesVisited(buf.getLong(null)); vs.setDocumentsReturned(buf.getLong(null)); vs.setBytesReturned(buf.getLong(null)); - vs.setSecondPassDocumentsReturned(buf.getLong(null)); // TODO: on Vespa 8 remove setter (_not_ getLong()) - vs.setSecondPassBytesReturned(buf.getLong(null)); // TODO: on Vespa 8 remove setter (_not_ getLong()) + buf.getLong(null); // unused + buf.getLong(null); // unused reply.setVisitorStatistics(vs); return reply; } @Override - @SuppressWarnings("removal")// TODO: Vespa 8: remove protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { CreateVisitorReply reply = (CreateVisitorReply)obj; buf.putLong(null, reply.getLastBucket().getRawId()); @@ -262,8 +257,8 @@ public abstract class RoutableFactories60 { buf.putLong(null, reply.getVisitorStatistics().getBytesVisited()); buf.putLong(null, reply.getVisitorStatistics().getDocumentsReturned()); buf.putLong(null, reply.getVisitorStatistics().getBytesReturned()); - buf.putLong(null, reply.getVisitorStatistics().getSecondPassDocumentsReturned()); // TODO: on Vespa 8 remove getter (_not_ putLong()) - buf.putLong(null, reply.getVisitorStatistics().getSecondPassBytesReturned()); // TODO: on Vespa 8 remove getter (_not_ putLong()) + buf.putLong(null, 0); // was SecondPassDocumentsReturned + buf.putLong(null, 0); // was SecondPassBytesReturned return true; } } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactory.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactory.java index 7baa41e5c6a..bff6cac548e 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactory.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactory.java @@ -3,7 +3,6 @@ package com.yahoo.documentapi.messagebus.protocol; import com.yahoo.document.serialization.DocumentDeserializer; import com.yahoo.document.serialization.DocumentSerializer; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.messagebus.Routable; /** @@ -12,7 +11,7 @@ import com.yahoo.messagebus.Routable; * com.yahoo.component.VersionSpecification)} method. </p> * * <p>Notice that no routable type is passed to the - * {@link #decode(DocumentDeserializer, com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet)} method, so + * {@link #decode(DocumentDeserializer)} method, so * you may NOT share a factory across multiple routable types. To share serialization logic between factory use a common * superclass or composition with a common serialization utility.</p> * @@ -32,21 +31,12 @@ public interface RoutableFactory { boolean encode(Routable obj, DocumentSerializer out); /** - * <p>This method decodes the given byte bufer to a routable.</p> <p>Return false to signal failure.</p> <p>This + * <p>This method decodes the given byte buffer to a routable.</p> <p>Return false to signal failure.</p> <p>This * method is NOT exception safe.</p> * * @param in The buffer to read from. - * @param loadTypes The LoadTypeSet to inject into the Routable. * @return The decoded routable. - * @deprecated load types are deprecated. Use method without LoadTypeSet instead */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - Routable decode(DocumentDeserializer in, LoadTypeSet loadTypes); - - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - default Routable decode(DocumentDeserializer in) { - return decode(in, LoadTypeSet.EMPTY); - } + Routable decode(DocumentDeserializer in); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java index 2360cbe8bc3..5323c1f5226 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java @@ -9,7 +9,6 @@ import com.yahoo.document.serialization.DocumentDeserializer; import com.yahoo.document.serialization.DocumentDeserializerFactory; import com.yahoo.document.serialization.DocumentSerializer; import com.yahoo.document.serialization.DocumentSerializerFactory; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.io.GrowableByteBuffer; import java.util.logging.Level; import com.yahoo.messagebus.Routable; @@ -28,26 +27,15 @@ import java.util.logging.Logger; * * @author Simon Thoresen Hult */ -@SuppressWarnings("removal") // TODO: Remove on Vespa 8 final class RoutableRepository { private static final Logger log = Logger.getLogger(RoutableRepository.class.getName()); private final CopyOnWriteHashMap<Integer, VersionMap> factoryTypes = new CopyOnWriteHashMap<>(); private final CopyOnWriteHashMap<CacheKey, RoutableFactory> cache = new CopyOnWriteHashMap<>(); - private LoadTypeSet loadTypes; // TODO remove on Vespa 8 public RoutableRepository() {} /** - * @deprecated load types are deprecated. Use default constructor instead. - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - public RoutableRepository(LoadTypeSet set) { - loadTypes = set; - } - - /** * Decodes a {@link Routable} from the given byte array. This uses the content of the byte array to dispatch the * decode request to the appropriate {@link RoutableFactory} that was previously registered. * @@ -75,7 +63,7 @@ final class RoutableRepository { log.log(Level.SEVERE,"No routable factory found for routable type " + type + " (version " + version + ")."); return null; } - Routable ret = factory.decode(in, loadTypes); + Routable ret = factory.decode(in); if (ret == null) { log.log(Level.SEVERE,"Routable factory " + factory.getClass().getName() + " failed to deserialize " + "routable of type " + type + " (version " + version + ").\nData = " + Arrays.toString(data)); diff --git a/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java index b1187d48374..0a0191a9815 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/VisitorParametersTestCase.java @@ -2,17 +2,12 @@ package com.yahoo.documentapi; import com.yahoo.document.fieldset.AllFields; -import com.yahoo.documentapi.messagebus.loadtypes.LoadType; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import org.junit.Test; import static org.junit.Assert.*; -@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public class VisitorParametersTestCase { - // TODO: Remove on Vespa 8 - private LoadType loadType = new LoadType(3, "samnmax", DocumentProtocol.Priority.HIGH_3); - @SuppressWarnings("removal")// TODO: Vespa 8: remove private VisitorParameters createVisitorParameters() { VisitorParameters params = new VisitorParameters(""); params.setDocumentSelection("id.user==5678"); @@ -23,12 +18,11 @@ public class VisitorParametersTestCase { params.setLibraryParameter("groovy", "dudes"); params.setLibraryParameter("ninja", "turtles"); params.setMaxBucketsPerVisitor(55); - params.setPriority(DocumentProtocol.Priority.HIGHEST); // TODO: Remove on Vespa 8 + params.setPriority(DocumentProtocol.Priority.HIGHEST); params.setRoute("extraterrestrial/highway"); params.setTimeoutMs(1337); params.setMaxPending(111); params.setFieldSet(AllFields.NAME); - params.setLoadType(loadType); // TODO: Remove on Vespa 8 params.setVisitRemoves(true); params.setVisitInconsistentBuckets(true); params.setTraceLevel(9); @@ -36,16 +30,12 @@ public class VisitorParametersTestCase { params.setResumeToken(new ProgressToken()); params.setRemoteDataHandler("mars_rover"); params.setControlHandler(new VisitorControlHandler()); - params.setMaxFirstPassHits(555); params.setMaxTotalHits(777); - params.setDynamicallyIncreaseMaxBucketsPerVisitor(true); - params.setDynamicMaxBucketsIncreaseFactor(2.5f); params.skipBucketsOnFatalErrors(true); return params; } - @SuppressWarnings("removal")// TODO: Vespa 8: remove @Test public void testCopyConstructor() { VisitorParameters params = createVisitorParameters(); @@ -65,7 +55,6 @@ public class VisitorParametersTestCase { assertEquals(1337, copy.getTimeoutMs()); assertEquals(111, copy.getMaxPending()); assertEquals(AllFields.NAME, copy.getFieldSet()); - assertEquals(loadType, copy.getLoadType()); assertEquals(true, copy.getVisitRemoves()); assertEquals(true, copy.getVisitInconsistentBuckets()); assertEquals(9, copy.getTraceLevel()); @@ -73,10 +62,7 @@ public class VisitorParametersTestCase { assertEquals(params.getResumeToken(), copy.getResumeToken()); // instance compare assertEquals("mars_rover", copy.getRemoteDataHandler()); assertEquals(params.getControlHandler(), copy.getControlHandler()); - assertEquals(555, copy.getMaxFirstPassHits()); assertEquals(777, copy.getMaxTotalHits()); - assertEquals(true, copy.getDynamicallyIncreaseMaxBucketsPerVisitor()); - assertEquals(2.5f, copy.getDynamicMaxBucketsIncreaseFactor(), 0.0001); assertEquals(true, copy.skipBucketsOnFatalErrors()); // Test local data handler copy @@ -106,13 +92,11 @@ public class VisitorParametersTestCase { " Field set: [all]\n" + " Route: extraterrestrial/highway\n" + " Weight: 1.0\n" + - " Max firstpass hits: 555\n" + " Max total hits: 777\n" + " Max buckets: 55\n" + " Priority: HIGHEST\n" + - " Dynamically increasing max buckets per visitor\n" + - " Increase factor: 2.5\n" + ")", params.toString()); } + } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSessionTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSessionTestCase.java index 53545510ae7..43dd0ee392e 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSessionTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSessionTestCase.java @@ -3,8 +3,8 @@ package com.yahoo.documentapi.messagebus; import com.yahoo.document.BucketId; import com.yahoo.document.DocumentId; -import com.yahoo.document.fieldset.AllFields; import com.yahoo.document.fieldset.DocIdOnly; +import com.yahoo.document.fieldset.DocumentOnly; import com.yahoo.document.select.parser.ParseException; import com.yahoo.documentapi.AckToken; import com.yahoo.documentapi.ProgressToken; @@ -14,7 +14,6 @@ import com.yahoo.documentapi.VisitorDataHandler; import com.yahoo.documentapi.VisitorDataQueue; import com.yahoo.documentapi.VisitorParameters; import com.yahoo.documentapi.VisitorResponse; -import com.yahoo.documentapi.messagebus.loadtypes.LoadType; import com.yahoo.documentapi.messagebus.protocol.CreateVisitorMessage; import com.yahoo.documentapi.messagebus.protocol.CreateVisitorReply; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; @@ -477,7 +476,7 @@ public class MessageBusVisitorSessionTestCase { return params; } - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 + @SuppressWarnings("removal") // TODO: Remove on Vespa 9 private String createVisitorToString(CreateVisitorMessage msg) { StringBuilder sb = new StringBuilder(); sb.append("CreateVisitorMessage(buckets=[\n"); @@ -500,7 +499,7 @@ public class MessageBusVisitorSessionTestCase { if (msg.getMaxPendingReplyCount() != 32) { sb.append("max pending=").append(msg.getMaxPendingReplyCount()).append("\n"); } - if (!AllFields.NAME.equals(msg.getFieldSet())) { + if (!DocumentOnly.NAME.equals(msg.getFieldSet())) { sb.append("fieldset=").append(msg.getFieldSet()).append("\n"); } if (msg.getVisitInconsistentBuckets()) { @@ -524,10 +523,7 @@ public class MessageBusVisitorSessionTestCase { if (msg.getMaxBucketsPerVisitor() != 1) { sb.append("max buckets per visitor=").append(msg.getMaxBucketsPerVisitor()).append("\n"); } - if (msg.getLoadType() != LoadType.DEFAULT) { - sb.append("load type=").append(msg.getLoadType().getName()).append("\n"); - } - if (msg.getPriority() != DocumentProtocol.Priority.NORMAL_3) { // TODO: remove on Vespa 8 + if (msg.getPriority() != DocumentProtocol.Priority.NORMAL_3) { // TODO: remove on Vespa 9 sb.append("priority=").append(msg.getPriority()).append("\n"); } if (!"DumpVisitor".equals(msg.getLibraryName())) { @@ -722,7 +718,6 @@ public class MessageBusVisitorSessionTestCase { } @Test - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public void testMessageParameters() { MockSender sender = new MockSender(); MockReceiver receiver = new MockReceiver(); @@ -742,7 +737,6 @@ public class MessageBusVisitorSessionTestCase { params.setTimeoutMs(1337); params.setMaxPending(111); params.setFieldSet(DocIdOnly.NAME); - params.setLoadType(new LoadType(3, "samnmax", DocumentProtocol.Priority.HIGH_3)); // TODO: Remove on Vespa 8 params.setVisitRemoves(true); params.setVisitInconsistentBuckets(true); params.setTraceLevel(9); @@ -772,7 +766,6 @@ public class MessageBusVisitorSessionTestCase { "]\n" + "route=extraterrestrial/highway\n" + "max buckets per visitor=55\n" + - "load type=samnmax\n" + "priority=HIGHEST\n" + "visitor library=CoolVisitor\n" + "trace level=9\n" + @@ -975,7 +968,7 @@ public class MessageBusVisitorSessionTestCase { builder.append("onVisitorStatistics : "); // Only bother with a couple of fields. builder.append(vs.getBucketsVisited()).append(" buckets visited, "); - builder.append(vs.getDocumentsReturned() + vs.getSecondPassDocumentsReturned()).append(" docs returned\n"); + builder.append(vs.getDocumentsReturned()).append(" docs returned\n"); } @Override @@ -1439,51 +1432,6 @@ public class MessageBusVisitorSessionTestCase { doTestEarlyCompletion(visitorParameters, replyModifier1, replyModifier2); } - @SuppressWarnings("removal")// TODO: Vespa 8: remove - @Test - public void testVisitingCompletedFromSufficientFirstPassHits() { - VisitorParameters visitorParameters = createVisitorParameters("id.user==1234"); - visitorParameters.setMaxFirstPassHits(10); - ReplyModifier replyModifier1 = (reply) -> { - VisitorStatistics stats = new VisitorStatistics(); - stats.setBucketsVisited(1); - stats.setDocumentsReturned(9); - reply.setVisitorStatistics(stats); - reply.setLastBucket(new BucketId(33, 1234 | (1L << 32))); - }; - ReplyModifier replyModifier2 = (reply) -> { - VisitorStatistics stats = new VisitorStatistics(); - stats.setBucketsVisited(1); - stats.setDocumentsReturned(1); - reply.setVisitorStatistics(stats); - reply.setLastBucket(new BucketId(34, 1234 | (1L << 33))); - }; - doTestEarlyCompletion(visitorParameters, replyModifier1, replyModifier2); - } - - @SuppressWarnings("removal")// TODO: Vespa 8: remove test - @Test - public void testVisitingCompletedFromSecondPassHits() { - VisitorParameters visitorParameters = createVisitorParameters("id.user==1234"); - visitorParameters.setMaxTotalHits(10); - ReplyModifier replyModifier1 = (reply) -> { - VisitorStatistics stats = new VisitorStatistics(); - stats.setBucketsVisited(1); - stats.setDocumentsReturned(5); - stats.setSecondPassDocumentsReturned(4); - reply.setVisitorStatistics(stats); - reply.setLastBucket(new BucketId(33, 1234 | (1L << 32))); - }; - ReplyModifier replyModifier2 = (reply) -> { - VisitorStatistics stats = new VisitorStatistics(); - stats.setBucketsVisited(1); - stats.setSecondPassDocumentsReturned(1); - reply.setVisitorStatistics(stats); - reply.setLastBucket(new BucketId(34, 1234 | (1L << 33))); - }; - doTestEarlyCompletion(visitorParameters, replyModifier1, replyModifier2); - } - /** * Test that waitUntilDone on the session is forwarded to the control handler. */ @@ -2342,50 +2290,6 @@ public class MessageBusVisitorSessionTestCase { mc.controlHandler.toString()); } - @SuppressWarnings("removal")// TODO: Vespa 8: remove test - @Test - public void testDynamicallyIncreaseMaxBucketsPerVisitorOption() { - VisitorParameters visitorParameters = createVisitorParameters("id.user==1234"); - visitorParameters.setDynamicallyIncreaseMaxBucketsPerVisitor(true); - visitorParameters.setMaxBucketsPerVisitor(2); - visitorParameters.setDynamicMaxBucketsIncreaseFactor(10); - visitorParameters.setMaxFirstPassHits(10); - MockComponents mc = createDefaultMock(visitorParameters); - - mc.visitorSession.start(); - mc.executor.expectAndProcessTasks(1); - - assertEquals("CreateVisitorMessage(buckets=[\n" + - "BucketId(0x80000000000004d2)\n" + - "BucketId(0x0000000000000000)\n" + - "]\n" + - "selection='id.user==1234'\n" + - "max buckets per visitor=2\n)", - replyToCreateVisitor(mc.sender, new BucketId(33, 1234 | (1L << 32)))); - mc.executor.expectAndProcessTasks(1); // reply - mc.executor.expectAndProcessTasks(1); // send create visitors - - assertEquals("CreateVisitorMessage(buckets=[\n" + - "BucketId(0x80000000000004d2)\n" + - "BucketId(0x84000001000004d2)\n" + - "]\n" + - "selection='id.user==1234'\n" + - "max buckets per visitor=20\n)", - replyToCreateVisitor(mc.sender, new BucketId(34, 1234 | (1L << 33)))); - - mc.executor.expectAndProcessTasks(1); // reply - mc.executor.expectAndProcessTasks(1); // send create visitors - - // Saturate at 128 - assertEquals("CreateVisitorMessage(buckets=[\n" + - "BucketId(0x80000000000004d2)\n" + - "BucketId(0x88000002000004d2)\n" + - "]\n" + - "selection='id.user==1234'\n" + - "max buckets per visitor=128\n)", - replyToCreateVisitor(mc.sender, ProgressToken.FINISHED_BUCKET)); - } - @Test public void testVisitorTimeoutsNotConsideredFatal() { VisitorParameters visitorParameters = createVisitorParameters("id.user==1234"); diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/loadtypes/test/LoadTypesTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/loadtypes/test/LoadTypesTestCase.java deleted file mode 100644 index 18269971f88..00000000000 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/loadtypes/test/LoadTypesTestCase.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.loadtypes.test; - -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author thomasg - */ -@SuppressWarnings("removal") // TODO: Remove on Vespa 8 -// TODO Vespa 8: remove test case once load types are gone -public class LoadTypesTestCase { - - @Test - public void testIdGeneration() { - LoadTypeSet set = new LoadTypeSet(); - set.addType("vespagrim", "VERY_HIGH"); - set.addType("slow", "VERY_LOW"); - set.addType("test", null); - - assertEquals("vespagrim", set.getNameMap().get("vespagrim").getName()); - assertEquals("slow", set.getNameMap().get("slow").getName()); - assertEquals("test", set.getNameMap().get("test").getName()); - assertEquals("default", set.getNameMap().get("default").getName()); - - assertEquals(0xc21803d4, set.getNameMap().get("vespagrim").getId()); - } - -} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java index deecba96aa6..35f8e7cf0c8 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java @@ -143,7 +143,6 @@ public class Messages60TestCase extends MessagesTestBase { private static final String BUCKET_SPACE = "beartato"; @Override - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public void run() { GetBucketListMessage msg = new GetBucketListMessage(new BucketId(16, 123)); msg.setBucketSpace(BUCKET_SPACE); @@ -152,7 +151,6 @@ public class Messages60TestCase extends MessagesTestBase { for (Language lang : LANGUAGES) { msg = (GetBucketListMessage)deserialize("GetBucketListMessage", DocumentProtocol.MESSAGE_GETBUCKETLIST, lang); assertEquals(new BucketId(16, 123), msg.getBucketId()); - assertEquals("default", msg.getLoadType().getName()); // TODO: Remove on Vespa 8 assertEquals(BUCKET_SPACE, msg.getBucketSpace()); } } @@ -163,10 +161,8 @@ public class Messages60TestCase extends MessagesTestBase { private static final String BUCKET_SPACE = "andrei"; @Override - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 public void run() { StatBucketMessage msg = new StatBucketMessage(new BucketId(16, 123), "id.user=123"); - msg.setLoadType(null); // TODO: Remove on Vespa 8 msg.setBucketSpace(BUCKET_SPACE); assertEquals(BASE_MESSAGE_LENGTH + 27 + serializedLength(BUCKET_SPACE), serialize("StatBucketMessage", msg)); @@ -174,7 +170,6 @@ public class Messages60TestCase extends MessagesTestBase { msg = (StatBucketMessage)deserialize("StatBucketMessage", DocumentProtocol.MESSAGE_STATBUCKET, lang); assertEquals(new BucketId(16, 123), msg.getBucketId()); assertEquals("id.user=123", msg.getDocumentSelection()); - assertEquals("default", msg.getLoadType().getName()); // TODO: Remove on Vespa 8 assertEquals(BUCKET_SPACE, msg.getBucketSpace()); } } @@ -276,7 +271,6 @@ public class Messages60TestCase extends MessagesTestBase { public class testCreateVisitorReply implements RunnableTest { @Override - @SuppressWarnings("removal") public void run() { CreateVisitorReply reply = new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR); reply.setLastBucket(new BucketId(16, 123)); @@ -285,8 +279,6 @@ public class Messages60TestCase extends MessagesTestBase { reply.getVisitorStatistics().setBytesVisited(1024000); reply.getVisitorStatistics().setDocumentsReturned(123); reply.getVisitorStatistics().setBytesReturned(512000); - reply.getVisitorStatistics().setSecondPassDocumentsReturned(456); - reply.getVisitorStatistics().setSecondPassBytesReturned(789100); assertEquals(65, serialize("CreateVisitorReply", reply)); @@ -299,8 +291,6 @@ public class Messages60TestCase extends MessagesTestBase { assertEquals(1024000, reply.getVisitorStatistics().getBytesVisited()); assertEquals(123, reply.getVisitorStatistics().getDocumentsReturned()); assertEquals(512000, reply.getVisitorStatistics().getBytesReturned()); - assertEquals(456, reply.getVisitorStatistics().getSecondPassDocumentsReturned()); // TODO remove on Vespa 8 - assertEquals(789100, reply.getVisitorStatistics().getSecondPassBytesReturned()); } } } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java index 74d06c05383..f43449646be 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/MessagesTestBase.java @@ -4,7 +4,6 @@ package com.yahoo.documentapi.messagebus.protocol.test; import com.yahoo.component.Version; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentTypeManagerConfigurer; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.messagebus.Routable; import org.junit.Test; @@ -19,7 +18,6 @@ import static org.junit.Assert.*; /** * @author Simon Thoresen Hult */ -@SuppressWarnings("removal") // TODO: Remove on Vespa 8 public abstract class MessagesTestBase { protected enum Language { @@ -29,12 +27,10 @@ public abstract class MessagesTestBase { protected static final Set<Language> LANGUAGES = EnumSet.allOf(Language.class); protected final DocumentTypeManager docMan = new DocumentTypeManager(); - protected final LoadTypeSet loadTypes = new LoadTypeSet(); // TODO remove on Vespa 8 - protected final DocumentProtocol protocol = new DocumentProtocol(docMan, null, loadTypes); + protected final DocumentProtocol protocol = new DocumentProtocol(docMan, null); public MessagesTestBase() { DocumentTypeManagerConfigurer.configure(docMan, "file:./test/cfg/testdoc.cfg"); - loadTypes.addLoadType(34, "foo", DocumentProtocol.Priority.NORMAL_2); } @Test diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java index 07456870977..1d92bcf4657 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java @@ -7,7 +7,7 @@ import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentTypeManagerConfigurer; import com.yahoo.document.DocumentUpdate; -import com.yahoo.document.fieldset.AllFields; +import com.yahoo.document.fieldset.DocumentOnly; import com.yahoo.documentapi.messagebus.protocol.ANDPolicy; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.messagebus.protocol.DocumentRouteSelectorPolicy; @@ -378,7 +378,7 @@ public class PolicyTestCase { PolicyTestFrame frame = new PolicyTestFrame(manager); frame.setHop(new HopSpec("test", getDocumentRouteSelectorRawConfig()) .addRecipient("foo").addRecipient("bar")); - frame.setMessage(new GetDocumentMessage(new DocumentId("id:ns:testdoc::yarn"), AllFields.NAME)); + frame.setMessage(new GetDocumentMessage(new DocumentId("id:ns:testdoc::yarn"), DocumentOnly.NAME)); List<RoutingNode> selected = frame.select(2); for (int i = 0, len = selected.size(); i < len; ++i) { Document doc = null; diff --git a/documentapi/src/tests/messages/messages60test.cpp b/documentapi/src/tests/messages/messages60test.cpp index f51b071fc70..ab300794307 100644 --- a/documentapi/src/tests/messages/messages60test.cpp +++ b/documentapi/src/tests/messages/messages60test.cpp @@ -366,8 +366,6 @@ Messages60Test::testCreateVisitorReply() vs.setBytesVisited(1024000); vs.setDocumentsReturned(123); vs.setBytesReturned(512000); - vs.setSecondPassDocumentsReturned(456); - vs.setSecondPassBytesReturned(789100); reply.setVisitorStatistics(vs); EXPECT_EQUAL(65u, serialize("CreateVisitorReply", reply)); @@ -383,8 +381,6 @@ Messages60Test::testCreateVisitorReply() EXPECT_EQUAL(ref.getVisitorStatistics().getBytesVisited(), (uint64_t)1024000); EXPECT_EQUAL(ref.getVisitorStatistics().getDocumentsReturned(), (uint64_t)123); EXPECT_EQUAL(ref.getVisitorStatistics().getBytesReturned(), (uint64_t)512000); - EXPECT_EQUAL(ref.getVisitorStatistics().getSecondPassDocumentsReturned(), (uint64_t)456); // TODO remove on Vespa 8 - EXPECT_EQUAL(ref.getVisitorStatistics().getSecondPassBytesReturned(), (uint64_t)789100); // TODO remove on Vespa 8 } } return true; diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp index d5eef700a15..d751d11177c 100644 --- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp @@ -32,7 +32,6 @@ DocumentProtocol::DocumentProtocol(std::shared_ptr<const DocumentTypeRepo> repo, // When adding factories to this list, please KEEP THEM ORDERED alphabetically like they are now. putRoutingPolicyFactory("AND", std::make_shared<RoutingPolicyFactories::AndPolicyFactory>()); putRoutingPolicyFactory("Content", std::make_shared<RoutingPolicyFactories::ContentPolicyFactory>()); - putRoutingPolicyFactory("Storage", std::make_shared<RoutingPolicyFactories::ContentPolicyFactory>()); // TODO Vespa 8: remove putRoutingPolicyFactory("DocumentRouteSelector", std::make_shared<RoutingPolicyFactories::DocumentRouteSelectorPolicyFactory>(*_repo, cfg)); putRoutingPolicyFactory("Extern", std::make_shared<RoutingPolicyFactories::ExternPolicyFactory>()); putRoutingPolicyFactory("LoadBalancer", std::make_shared<RoutingPolicyFactories::LoadBalancerPolicyFactory>()); diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp index ad32a5b4fa7..0757db9f5fc 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp @@ -171,11 +171,9 @@ RoutableFactories60::CreateVisitorReplyFactory::doDecode(document::ByteBuffer &b vs.setBytesVisited(decodeLong(buf)); vs.setDocumentsReturned(decodeLong(buf)); vs.setBytesReturned(decodeLong(buf)); - // TODO remove second pass concept on Vespa 8 - vs.setSecondPassDocumentsReturned(decodeLong(buf)); - vs.setSecondPassBytesReturned(decodeLong(buf)); + decodeLong(buf); // was SecondPassDocumentsReturned + decodeLong(buf); // was SecondPassBytesReturned reply->setVisitorStatistics(vs); - return reply; } @@ -189,9 +187,8 @@ RoutableFactories60::CreateVisitorReplyFactory::doEncode(const DocumentReply &ob buf.putLong(reply.getVisitorStatistics().getBytesVisited()); buf.putLong(reply.getVisitorStatistics().getDocumentsReturned()); buf.putLong(reply.getVisitorStatistics().getBytesReturned()); - // TODO remove second pass concept on Vespa 8 - buf.putLong(reply.getVisitorStatistics().getSecondPassDocumentsReturned()); - buf.putLong(reply.getVisitorStatistics().getSecondPassBytesReturned()); + buf.putLong(0); // was SecondPassDocumentsReturned + buf.putLong(0); // was SecondPassBytesReturned return true; } diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-CreateVisitorReply.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-CreateVisitorReply.dat Binary files differindex e13917227d1..493fa3291c5 100644 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-CreateVisitorReply.dat +++ b/documentapi/test/crosslanguagefiles/6.221-cpp-CreateVisitorReply.dat diff --git a/documentapi/test/crosslanguagefiles/6.221-java-CreateVisitorReply.dat b/documentapi/test/crosslanguagefiles/6.221-java-CreateVisitorReply.dat Binary files differindex e13917227d1..493fa3291c5 100644 --- a/documentapi/test/crosslanguagefiles/6.221-java-CreateVisitorReply.dat +++ b/documentapi/test/crosslanguagefiles/6.221-java-CreateVisitorReply.dat diff --git a/documentgen-test/etc/complex/book.sd b/documentgen-test/etc/complex/book.sd index ace6c33e440..5047775aa0a 100644 --- a/documentgen-test/etc/complex/book.sd +++ b/documentgen-test/etc/complex/book.sd @@ -24,7 +24,7 @@ search book { field mystruct2 type ss2 { } - field mywsfloat type weightedset<float> { + field mywsinteger type weightedset<int> { indexing: attribute } field myarrayint type array<int> { @@ -64,7 +64,6 @@ search book { } field vector type tensor(x{}) { indexing: attribute | summary - attribute: tensor(x{}) } } diff --git a/documentgen-test/pom.xml b/documentgen-test/pom.xml index 210b4c10570..1898872c1a0 100644 --- a/documentgen-test/pom.xml +++ b/documentgen-test/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>documentgen-test</artifactId> <packaging>jar</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>documentgen_test</name> <description>Tests for vespa-documentgen-plugin</description> <dependencies> @@ -104,7 +104,7 @@ <artifactId>vespa-documentgen-plugin</artifactId> <version>${project.version}</version> <configuration> - <sdDirectory>etc/complex</sdDirectory> + <schemasDirectory>etc/complex</schemasDirectory> <packageName>com.yahoo.vespa.documentgen.test</packageName> <abztract> <annotation><type>emptyannotation</type></annotation> diff --git a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java index 189949ceca9..680853ef687 100644 --- a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java +++ b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java @@ -204,13 +204,13 @@ public class DocumentGenPluginTest { Array old = (Array) book.removeFieldValue("myarrayint"); assertEquals(old.get(0), new IntegerFieldValue(10)); book.removeFieldValue("stringmap"); - book.removeFieldValue("mywsfloat"); + book.removeFieldValue("mywsinteger"); assertNull(book.getIsbn()); assertNull(book.getYear()); assertNull(book.getDescription()); assertNull(book.getStringmap()); assertNull(book.getMyarrayint()); - assertNull(book.getMywsfloat()); + assertNull(book.getMywsinteger()); Music music = getMusicBasic(); Field artist = music.getField("artist"); @@ -349,30 +349,30 @@ public class DocumentGenPluginTest { @Test public void testWeightedSets() { Book book = getBook(); - assertTrue(book.getField("mywsfloat").getDataType() instanceof WeightedSetDataType); - Field ws = book.getField("mywsfloat"); - assertEquals(book.getMywsfloat().get(2.2f), (Integer)200); - WeightedSet floatWs = (WeightedSet) book.getFieldValue("mywsfloat"); - assertEquals(floatWs.get(new FloatFieldValue(2.2f)), (Integer)200); - floatWs.remove(new FloatFieldValue(2.2f)); - assertNull(book.getMywsfloat().get(2.2f)); - assertEquals(((WeightedSet)book.getFieldValue(ws)).get(new FloatFieldValue(1.1f)), (Integer)100); - - Map<Float, Integer> ws2 = new HashMap<>(); - ws2.put(1.11f, 1000); - ws2.put(2.22f, 2000); - ws2.put(3.33f, 3000); - book.setMywsfloat(ws2); - assertEquals(book.getMywsfloat().get(2.22f), (Integer)2000); - floatWs = (WeightedSet) book.getFieldValue("mywsfloat"); - assertEquals(floatWs.get(new FloatFieldValue(2.22f)), (Integer)2000); - assertEquals(floatWs.size(), 3); - ws2.put(4.44f, 4000); - assertEquals(book.getMywsfloat().get(4.44f), (Integer)4000); - book.getMywsfloat().remove(4.44f); - assertNull(book.getMywsfloat().get(4.44f)); - assertNull(ws2.get(4.44f)); - assertEquals(((WeightedSet)book.getFieldValue(ws)).get(new FloatFieldValue(1.11f)), (Integer)1000); + assertTrue(book.getField("mywsinteger").getDataType() instanceof WeightedSetDataType); + Field ws = book.getField("mywsinteger"); + assertEquals(book.getMywsinteger().get(2), (Integer)200); + WeightedSet integerWs = (WeightedSet) book.getFieldValue("mywsinteger"); + assertEquals(integerWs.get(new IntegerFieldValue(2)), (Integer)200); + integerWs.remove(new IntegerFieldValue(2)); + assertNull(book.getMywsinteger().get(2)); + assertEquals(((WeightedSet)book.getFieldValue(ws)).get(new IntegerFieldValue(1)), (Integer)100); + + Map<Integer, Integer> ws2 = new HashMap<>(); + ws2.put(1, 1000); + ws2.put(2, 2000); + ws2.put(3, 3000); + book.setMywsinteger(ws2); + assertEquals(book.getMywsinteger().get(2), (Integer)2000); + integerWs = (WeightedSet) book.getFieldValue("mywsinteger"); + assertEquals(integerWs.get(new IntegerFieldValue(2)), (Integer)2000); + assertEquals(integerWs.size(), 3); + ws2.put(4, 4000); + assertEquals(book.getMywsinteger().get(4), (Integer)4000); + book.getMywsinteger().remove(4); + assertNull(book.getMywsinteger().get(4)); + assertNull(ws2.get(4)); + assertEquals(((WeightedSet)book.getFieldValue(ws)).get(new IntegerFieldValue(1)), (Integer)1000); } @Test @@ -557,10 +557,10 @@ public class DocumentGenPluginTest { book.setTitle("Moby Dick"); book.setYear(1851); book.setMystruct(new Ss1().setSs01(new Ss0().setS0("My s0").setD0(99d)).setS1("My s1").setL1(89L));//.setAl1(myAs1)); - Map<Float, Integer> wsFloat = new HashMap<>(); - wsFloat.put(56f, 55); - wsFloat.put(57f, 54); - book.setMywsfloat(wsFloat); + Map<Integer, Integer> wsInteger = new HashMap<>(); + wsInteger.put(56, 55); + wsInteger.put(57, 54); + book.setMywsinteger(wsInteger); Array<IntegerFieldValue> intArr1 = new Array<>(DataType.getArray(DataType.INT)); intArr1.add(new IntegerFieldValue(1)); @@ -604,10 +604,10 @@ public class DocumentGenPluginTest { myS1.setFieldValue("ss01", myS0); bookGeneric.setFieldValue("mystruct", myS1); - WeightedSet<FloatFieldValue> wsFloat = new WeightedSet<>(DataType.getWeightedSet(DataType.FLOAT)); - wsFloat.put(new FloatFieldValue(56f), 55); - wsFloat.put(new FloatFieldValue(57f), 54); - bookGeneric.setFieldValue("mywsfloat", wsFloat); + WeightedSet<FloatFieldValue> wsInteger = new WeightedSet<>(DataType.getWeightedSet(DataType.FLOAT)); + wsInteger.put(new FloatFieldValue(56), 55); + wsInteger.put(new FloatFieldValue(57), 54); + bookGeneric.setFieldValue("mywsinteger", wsInteger); Array<IntegerFieldValue> intArr1 = new Array<>(DataType.getArray(DataType.INT)); intArr1.add(new IntegerFieldValue(1)); @@ -660,10 +660,10 @@ public class DocumentGenPluginTest { myS1.setFieldValue("ss01", myS0); bookGeneric.setFieldValue("mystruct", myS1); assertEquals(((StructuredFieldValue) bookGeneric.getFieldValue("mystruct")).getFieldValue("s1").getWrappedValue(), "My s1"); - WeightedSet<FloatFieldValue> wsFloat = new WeightedSet<>(DataType.getWeightedSet(DataType.FLOAT)); - wsFloat.put(new FloatFieldValue(56f), 55); - wsFloat.put(new FloatFieldValue(57f), 54); - bookGeneric.setFieldValue("mywsfloat", wsFloat); + WeightedSet<IntegerFieldValue> wsInteger = new WeightedSet<>(DataType.getWeightedSet(DataType.INT)); + wsInteger.put(new IntegerFieldValue(56), 55); + wsInteger.put(new IntegerFieldValue(57), 54); + bookGeneric.setFieldValue("mywsinteger", wsInteger); Array<IntegerFieldValue> intArr1 = new Array<>(DataType.getArray(DataType.INT)); intArr1.add(new IntegerFieldValue(1)); intArr1.add(new IntegerFieldValue(2)); @@ -682,7 +682,7 @@ public class DocumentGenPluginTest { assertEquals(book.getMytriplearray().get(0).get(0).get(0), (Integer)1); assertEquals(book.getMytriplearray().get(0).get(0).get(1), (Integer)2); assertEquals(book.getMytriplearray().get(0).get(0).get(2), (Integer)3); - assertEquals(book.getMywsfloat().get(57f), (Integer)54); + assertEquals(book.getMywsinteger().get(57), (Integer)54); assertEquals(book.getMystruct().getAs1().get(1), "as1_2"); treeCheck = book.titleSpanTrees().values().iterator().next(); titleAnnCheck = treeCheck.iterator().next(); @@ -697,7 +697,7 @@ public class DocumentGenPluginTest { assertEquals(book2.getMytriplearray().get(0).get(0).get(0), (Integer)1); assertEquals(book2.getMytriplearray().get(0).get(0).get(1), (Integer)2); assertEquals(book2.getMytriplearray().get(0).get(0).get(2), (Integer)3); - assertEquals(book2.getMywsfloat().get(57f), (Integer)54); + assertEquals(book2.getMywsinteger().get(57), (Integer)54); assertEquals(book2.getMystruct().getAs1().get(1), "as1_2"); treeCheck = book2.titleSpanTrees().values().iterator().next(); titleAnnCheck = treeCheck.iterator().next(); @@ -816,11 +816,11 @@ public class DocumentGenPluginTest { structMap.put(60, ss1); book.setStructmap(structMap); - Map<Float, Integer> ws = new HashMap<>(); - ws.put(1.1f, 100); - ws.put(2.2f, 200); - ws.put(3.3f, 300); - book.setMywsfloat(ws); + Map<Integer, Integer> ws = new HashMap<>(); + ws.put(1, 100); + ws.put(2, 200); + ws.put(3, 300); + book.setMywsinteger(ws); Ss1 arrayedStruct1 = new Ss1().setS1("YEPS").setI1(789); Ss1 arrayedStruct2 = new Ss1().setS1("JA").setI1(456); @@ -891,11 +891,11 @@ public class DocumentGenPluginTest { assertEquals(a.get(1).getInteger(), 20); assertEquals(a.get(2).getInteger(), 30); - WeightedSet<FloatFieldValue> ws = (WeightedSet<FloatFieldValue>) des.getFieldValue("mywsfloat"); + WeightedSet<IntegerFieldValue> ws = (WeightedSet<IntegerFieldValue>) des.getFieldValue("mywsinteger"); assertEquals(ws.size(), 3); - assertEquals(ws.get(new FloatFieldValue(1.1f)), (Integer)100); - assertEquals(ws.get(new FloatFieldValue(2.2f)), (Integer)200); - assertEquals(ws.get(new FloatFieldValue(3.3f)), (Integer)300); + assertEquals(ws.get(new IntegerFieldValue(1)), (Integer)100); + assertEquals(ws.get(new IntegerFieldValue(2)), (Integer)200); + assertEquals(ws.get(new IntegerFieldValue(3)), (Integer)300); Array<Struct> sstrctArr = (Array<Struct>) des.getFieldValue("mysinglestructarray"); assertEquals(sstrctArr.size(), 2); diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 56db263fed4..3dca80885f7 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -72,6 +72,7 @@ vespa_define_module( src/tests/instruction/inplace_map_function src/tests/instruction/join_with_number src/tests/instruction/l2_distance + src/tests/instruction/mixed_112_dot_product src/tests/instruction/mixed_inner_product_function src/tests/instruction/mixed_simple_join_function src/tests/instruction/pow_as_map_optimizer diff --git a/eval/src/tests/instruction/generic_reduce/generic_reduce_test.cpp b/eval/src/tests/instruction/generic_reduce/generic_reduce_test.cpp index fe531785278..d7298db4b68 100644 --- a/eval/src/tests/instruction/generic_reduce/generic_reduce_test.cpp +++ b/eval/src/tests/instruction/generic_reduce/generic_reduce_test.cpp @@ -75,7 +75,7 @@ void test_generic_reduce_with(const ValueBuilderFactory &factory) { auto input = layout.cpy().cells(ct); if (input.bad_scalar()) continue; SCOPED_TRACE(fmt("tensor type: %s, num_cells: %zu", input.gen().type().c_str(), input.gen().cells().size())); - for (Aggr aggr: {Aggr::SUM, Aggr::AVG, Aggr::MIN, Aggr::MAX}) { + for (Aggr aggr: {Aggr::SUM, Aggr::AVG, Aggr::MIN, Aggr::MAX, Aggr::COUNT}) { SCOPED_TRACE(fmt("aggregator: %s", AggrNames::name_of(aggr)->c_str())); auto t = layout.type(); for (const auto & dim: t.dimensions()) { diff --git a/eval/src/tests/instruction/mixed_112_dot_product/CMakeLists.txt b/eval/src/tests/instruction/mixed_112_dot_product/CMakeLists.txt new file mode 100644 index 00000000000..fae2f185afb --- /dev/null +++ b/eval/src/tests/instruction/mixed_112_dot_product/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_mixed_112_dot_product_test_app TEST + SOURCES + mixed_112_dot_product_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_mixed_112_dot_product_test_app COMMAND eval_mixed_112_dot_product_test_app) diff --git a/eval/src/tests/instruction/mixed_112_dot_product/mixed_112_dot_product_test.cpp b/eval/src/tests/instruction/mixed_112_dot_product/mixed_112_dot_product_test.cpp new file mode 100644 index 00000000000..d3c4d89cf47 --- /dev/null +++ b/eval/src/tests/instruction/mixed_112_dot_product/mixed_112_dot_product_test.cpp @@ -0,0 +1,92 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/instruction/mixed_112_dot_product.h> +#include <vespa/eval/eval/test/eval_fixture.h> +#include <vespa/eval/eval/test/gen_spec.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib::eval; +using namespace vespalib::eval::test; + +using vespalib::make_string_short::fmt; + +//----------------------------------------------------------------------------- + +struct FunInfo { + using LookFor = Mixed112DotProduct; + void verify(const LookFor &fun) const { + EXPECT_TRUE(fun.result_is_mutable()); + } +}; + +void verify_optimized_cell_types(const vespalib::string &expr) +{ + CellTypeSpace stable(CellTypeUtils::list_stable_types(), 3); + CellTypeSpace unstable(CellTypeUtils::list_unstable_types(), 3); + EvalFixture::verify<FunInfo>(expr, {FunInfo()}, CellTypeSpace(stable).same()); + EvalFixture::verify<FunInfo>(expr, {}, CellTypeSpace(stable).different()); + EvalFixture::verify<FunInfo>(expr, {}, unstable); +} + +void verify_optimized(const vespalib::string &expr, size_t num_params = 3) +{ + CellTypeSpace just_float({CellType::FLOAT}, num_params); + EvalFixture::verify<FunInfo>(expr, {FunInfo()}, just_float); +} + +void verify_not_optimized(const vespalib::string &expr) { + CellTypeSpace just_double({CellType::DOUBLE}, 3); + EvalFixture::verify<FunInfo>(expr, {}, just_double); +} + +//----------------------------------------------------------------------------- + +TEST(Mixed112DotProduct, expression_can_be_optimized) +{ + verify_optimized_cell_types("reduce(x5_2*y8*x7_1y8,sum)"); +} + +TEST(Mixed112DotProduct, inverse_dimension_matching_is_handled) { + verify_optimized("reduce(y5_2*x8*x8y7_1,sum)"); +} + +TEST(Mixed112DotProduct, different_input_placement_is_handled) +{ + std::array<vespalib::string,3> params = {"x3_1", "y3", "x3_1y3"}; + for (size_t p1 = 0; p1 < params.size(); ++p1) { + for (size_t p2 = 0; p2 < params.size(); ++p2) { + for (size_t p3 = 0; p3 < params.size(); ++p3) { + if ((p1 != p2) && (p1 != p3) && (p2 != p3)) { + verify_optimized(fmt("reduce((%s*%s)*%s,sum)", params[p1].c_str(), params[p2].c_str(), params[p3].c_str())); + verify_optimized(fmt("reduce(%s*(%s*%s),sum)", params[p1].c_str(), params[p2].c_str(), params[p3].c_str())); + } + } + } + } +} + +TEST(Mixed112DotProduct, expression_can_be_optimized_with_extra_tensors) +{ + verify_optimized("reduce((x5_2*y4)*(x5_1y4*x3_1),sum)", 4); + verify_optimized("reduce((x5_2*x3_1)*(y4*x5_1y4),sum)", 4); +} + +TEST(Mixed112DotProduct, similar_expressions_are_not_optimized) +{ + verify_not_optimized("reduce(x5_2*y4*x5_1y4,prod)"); + verify_not_optimized("reduce(x5_2+y4*x5_1y4,sum)"); + verify_not_optimized("reduce(x5_2*y4+x5_1y4,sum)"); + verify_not_optimized("reduce(x5_2*z4*x5_1y4,sum)"); + verify_not_optimized("reduce(x5_2*y4*x5_1z4,sum)"); + verify_not_optimized("reduce(x5_2*x1_1y4*x5_1y4,sum)"); + verify_not_optimized("reduce(x5_2*y4*x5_1,sum)"); + verify_not_optimized("reduce(x5*y4*x5y4,sum)"); + verify_not_optimized("reduce(x5_1*y4_1*x5_1y4_1,sum)"); +} + +//----------------------------------------------------------------------------- + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/instruction/sparse_112_dot_product/sparse_112_dot_product_test.cpp b/eval/src/tests/instruction/sparse_112_dot_product/sparse_112_dot_product_test.cpp index 9325a203ff3..bab45afe114 100644 --- a/eval/src/tests/instruction/sparse_112_dot_product/sparse_112_dot_product_test.cpp +++ b/eval/src/tests/instruction/sparse_112_dot_product/sparse_112_dot_product_test.cpp @@ -13,8 +13,6 @@ using namespace vespalib::eval::test; using vespalib::make_string_short::fmt; -const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get(); - //----------------------------------------------------------------------------- struct FunInfo { diff --git a/eval/src/vespa/eval/eval/optimize_tensor_function.cpp b/eval/src/vespa/eval/eval/optimize_tensor_function.cpp index f9d3b1c6f54..b6258acc9cb 100644 --- a/eval/src/vespa/eval/eval/optimize_tensor_function.cpp +++ b/eval/src/vespa/eval/eval/optimize_tensor_function.cpp @@ -7,6 +7,7 @@ #include <vespa/eval/instruction/dense_dot_product_function.h> #include <vespa/eval/instruction/sparse_dot_product_function.h> #include <vespa/eval/instruction/sparse_112_dot_product.h> +#include <vespa/eval/instruction/mixed_112_dot_product.h> #include <vespa/eval/instruction/sparse_merge_function.h> #include <vespa/eval/instruction/sparse_no_overlap_join_function.h> #include <vespa/eval/instruction/sparse_full_overlap_join_function.h> @@ -67,6 +68,7 @@ const TensorFunction &optimize_for_factory(const ValueBuilderFactory &, const Te run_optimize_pass(root, [&stash](const Child &child) { child.set(Sparse112DotProduct::optimize(child.get(), stash)); + child.set(Mixed112DotProduct::optimize(child.get(), stash)); child.set(BestSimilarityFunction::optimize(child.get(), stash)); child.set(L2Distance::optimize(child.get(), stash)); }); diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.h b/eval/src/vespa/eval/eval/test/eval_fixture.h index b078a900778..d2244cd4f5f 100644 --- a/eval/src/vespa/eval/eval/test/eval_fixture.h +++ b/eval/src/vespa/eval/eval/test/eval_fixture.h @@ -154,9 +154,9 @@ public: EvalFixture fixture(prod_factory(), expr, param_repo, true, true); EvalFixture slow_fixture(prod_factory(), expr, param_repo, false, false); EvalFixture test_fixture(test_factory(), expr, param_repo, true, true); - REQUIRE_EQ(fixture.result(), EvalFixture::ref(expr, param_repo)); - REQUIRE_EQ(fixture.result(), slow_fixture.result()); REQUIRE_EQ(fixture.result(), test_fixture.result()); + REQUIRE_EQ(fixture.result(), slow_fixture.result()); + REQUIRE_EQ(fixture.result(), EvalFixture::ref(expr, param_repo)); auto info = fixture.find_all<typename FunInfo::LookFor>(); REQUIRE_EQ(info.size(), fun_info.size()); for (size_t i = 0; i < fun_info.size(); ++i) { diff --git a/eval/src/vespa/eval/instruction/CMakeLists.txt b/eval/src/vespa/eval/instruction/CMakeLists.txt index b614606199c..f1ec8aa49a9 100644 --- a/eval/src/vespa/eval/instruction/CMakeLists.txt +++ b/eval/src/vespa/eval/instruction/CMakeLists.txt @@ -31,6 +31,7 @@ vespa_add_library(eval_instruction OBJECT inplace_map_function.cpp join_with_number_function.cpp l2_distance.cpp + mixed_112_dot_product.cpp mixed_inner_product_function.cpp mixed_simple_join_function.cpp pow_as_map_optimizer.cpp diff --git a/eval/src/vespa/eval/instruction/generic_reduce.cpp b/eval/src/vespa/eval/instruction/generic_reduce.cpp index 71eb94f4118..ee74dd49fad 100644 --- a/eval/src/vespa/eval/instruction/generic_reduce.cpp +++ b/eval/src/vespa/eval/instruction/generic_reduce.cpp @@ -154,6 +154,12 @@ void my_generic_dense_reduce_op(State &state, uint64_t param_in) { } }; +template <typename ICT> +void my_count_cells_op(State &state, uint64_t) { + auto cells = state.peek(0).cells().typify<ICT>(); + state.pop_push(state.stash.create<DoubleValue>(cells.size())); +}; + template <typename ICT, typename AGGR> void my_full_reduce_op(State &state, uint64_t) { auto cells = state.peek(0).cells().typify<ICT>(); @@ -194,7 +200,11 @@ struct SelectGenericReduceOp { using OCT = CellValueType<ICM::value.reduce(OIS::value).cell_type>; using AggrType = typename AGGR::template templ<OCT>; if constexpr (OIS::value) { - return my_full_reduce_op<ICT, AggrType>; + if constexpr (AggrType::enum_value() == Aggr::COUNT) { + return my_count_cells_op<ICT>; + } else { + return my_full_reduce_op<ICT, AggrType>; + } } else { if (param.sparse_plan.should_forward_index()) { return my_generic_dense_reduce_op<ICT, OCT, AggrType, true>; diff --git a/eval/src/vespa/eval/instruction/mixed_112_dot_product.cpp b/eval/src/vespa/eval/instruction/mixed_112_dot_product.cpp new file mode 100644 index 00000000000..8bfa4b07980 --- /dev/null +++ b/eval/src/vespa/eval/instruction/mixed_112_dot_product.cpp @@ -0,0 +1,261 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "mixed_112_dot_product.h" +#include <vespa/eval/eval/fast_value.hpp> +#include <vespa/vespalib/util/typify.h> +#include <vespa/vespalib/util/require.h> +#include <vespa/eval/eval/visit_stuff.h> +#include <cblas.h> +#include <algorithm> +#include <optional> + +namespace vespalib::eval { + +using namespace tensor_function; +using namespace operation; +using namespace instruction; + +namespace { + +template <typename CT> double my_dot_product(const CT * lhs, const CT * rhs, size_t count); +template <> double my_dot_product<double>(const double * lhs, const double * rhs, size_t count) { + return cblas_ddot(count, lhs, 1, rhs, 1); +} +template <> double my_dot_product<float>(const float * lhs, const float * rhs, size_t count) { + return cblas_sdot(count, lhs, 1, rhs, 1); +} + +template <typename T, size_t N> +ConstArrayRef<const T *> as_ccar(std::array<T *, N> &array) { + return {array.data(), array.size()}; +} + +template <typename T> +ConstArrayRef<T> as_car(T &value) { + return {&value, 1}; +} + +constexpr std::array<size_t, 1> single_dim = { 0 }; + +template <typename CT> +double my_mixed_112_dot_product_fallback(const Value::Index &a_idx, const Value::Index &c_idx, + const CT *a_cells, const CT *b_cells, const CT *c_cells, + size_t dense_size) __attribute__((noinline)); +template <typename CT> +double my_mixed_112_dot_product_fallback(const Value::Index &a_idx, const Value::Index &c_idx, + const CT *a_cells, const CT *b_cells, const CT *c_cells, + size_t dense_size) +{ + double result = 0.0; + size_t a_space = 0; + size_t c_space = 0; + std::array<string_id, 1> c_addr; + std::array<string_id*, 1> c_addr_ref = {&c_addr[0]}; + auto outer = a_idx.create_view({}); + auto model = c_idx.create_view({&single_dim[0], 1}); + outer->lookup({}); + while (outer->next_result(as_car(c_addr_ref[0]), a_space)) { + model->lookup(as_ccar(c_addr_ref)); + if (model->next_result({}, c_space)) { + result += my_dot_product<CT>(b_cells, c_cells + (c_space * dense_size), dense_size) * a_cells[a_space]; + } + } + return result; +} + +template <typename CT> +double my_fast_mixed_112_dot_product(const FastAddrMap *a_map, const FastAddrMap *c_map, + const CT *a_cells, const CT *b_cells, const CT *c_cells, + size_t dense_size) +{ + double result = 0.0; + const auto &a_labels = a_map->labels(); + for (size_t a_space = 0; a_space < a_labels.size(); ++a_space) { + if (a_cells[a_space] != 0.0) { // handle pseudo-sparse input + auto c_space = c_map->lookup_singledim(a_labels[a_space]); + if (c_space != FastAddrMap::npos()) { + result += my_dot_product<CT>(b_cells, c_cells + (c_space * dense_size), dense_size) * a_cells[a_space]; + } + } + } + return result; +} + +template <typename CT> +void my_mixed_112_dot_product_op(InterpretedFunction::State &state, uint64_t dense_size) { + const auto &a_idx = state.peek(2).index(); + const auto &c_idx = state.peek(0).index(); + const CT *a_cells = state.peek(2).cells().unsafe_typify<CT>().cbegin(); + const CT *b_cells = state.peek(1).cells().unsafe_typify<CT>().cbegin(); + const CT *c_cells = state.peek(0).cells().unsafe_typify<CT>().cbegin(); + double result = __builtin_expect(are_fast(a_idx, c_idx), true) + ? my_fast_mixed_112_dot_product<CT>(&as_fast(a_idx).map, &as_fast(c_idx).map, + a_cells, b_cells, c_cells, dense_size) + : my_mixed_112_dot_product_fallback<CT>(a_idx, c_idx, a_cells, b_cells, c_cells, dense_size); + state.pop_pop_pop_push(state.stash.create<DoubleValue>(result)); +} + +InterpretedFunction::op_function my_select(CellType cell_type) { + if (cell_type == CellType::DOUBLE) { + return my_mixed_112_dot_product_op<double>; + } + if (cell_type == CellType::FLOAT) { + return my_mixed_112_dot_product_op<float>; + } + abort(); +} + +// Try to collect input nodes and organize them into a 3-way dot +// product between one 1d sparse tensor, one 1d dense tensor and one +// 2d mixed tensor. Cell types must be all float or all double. + +struct InputState { + std::optional<CellType> cell_type; + const TensorFunction *sparse = nullptr; + const TensorFunction *dense = nullptr; + const TensorFunction *mixed = nullptr; + bool failed = false; + + void collect_cell_type(CellType ct) { + if (cell_type.has_value()) { + if (ct != cell_type.value()) { + failed = true; + } + } else { + cell_type = ct; + } + } + + void try_save(const TensorFunction *&my_ptr, const TensorFunction &node) { + if (my_ptr == nullptr) { + my_ptr = &node; + } else { + failed = true; + } + } + + void collect(const TensorFunction &node) { + const auto &type = node.result_type(); + collect_cell_type(type.cell_type()); + if (type.is_sparse()) { + try_save(sparse, node); + } else if (type.is_dense()) { + try_save(dense, node); + } else if (type.has_dimensions()) { + try_save(mixed, node); + } else { + failed = true; + } + } + + bool verify() const { + // all parts must have been collected successfully + if (failed || !cell_type.has_value() || (sparse == nullptr) || (dense == nullptr) || (mixed == nullptr)) { + return false; + } + // common cell type must be appropriate + if ((cell_type.value() != CellType::FLOAT) && (cell_type.value() != CellType::DOUBLE)) { + return false; + } + // number of dimensions must match the expected 112 pattern + if ((sparse->result_type().dimensions().size() != 1) || + (dense->result_type().dimensions().size() != 1) || + (mixed->result_type().dimensions().size() != 2)) + { + return false; + } + // the product of the sparse and dense tensors must fully overlap the mixed tensor + const ValueType::Dimension *mapped = &mixed->result_type().dimensions()[0]; + const ValueType::Dimension *indexed = &mixed->result_type().dimensions()[1]; + if (!mapped->is_mapped()) { + std::swap(mapped, indexed); + } + assert(mapped->is_mapped()); + assert(indexed->is_indexed()); + return ((*mapped == sparse->result_type().dimensions()[0]) && + (*indexed == dense->result_type().dimensions()[0])); + } +}; + +// Try to find inputs that form a 112 mixed dot product. + +struct FindInputs { + const TensorFunction *a = nullptr; + const TensorFunction *b = nullptr; + const TensorFunction *c = nullptr; + + bool try_match(const TensorFunction &one, const TensorFunction &two) { + auto join = as<Join>(two); + if (join && (join->function() == Mul::f)) { + InputState state; + state.collect(one); + state.collect(join->lhs()); + state.collect(join->rhs()); + if (state.verify()) { + a = state.sparse; + b = state.dense; + c = state.mixed; + return true; + } + } + return false; + } +}; + +} // namespace <unnamed> + +Mixed112DotProduct::Mixed112DotProduct(const TensorFunction &a_in, + const TensorFunction &b_in, + const TensorFunction &c_in) + : tensor_function::Node(DoubleValue::shared_type()), + _a(a_in), + _b(b_in), + _c(c_in) +{ +} + +InterpretedFunction::Instruction +Mixed112DotProduct::compile_self(const ValueBuilderFactory &, Stash &) const +{ + REQUIRE_EQ(_a.get().result_type().cell_type(), _b.get().result_type().cell_type()); + REQUIRE_EQ(_a.get().result_type().cell_type(), _c.get().result_type().cell_type()); + REQUIRE_EQ(_b.get().result_type().dense_subspace_size(), _c.get().result_type().dense_subspace_size()); + auto op = my_select(_a.get().result_type().cell_type()); + return InterpretedFunction::Instruction(op, _c.get().result_type().dense_subspace_size()); +} + +void +Mixed112DotProduct::push_children(std::vector<Child::CREF> &children) const +{ + children.emplace_back(_a); + children.emplace_back(_b); + children.emplace_back(_c); +} + +void +Mixed112DotProduct::visit_children(vespalib::ObjectVisitor &visitor) const +{ + ::visit(visitor, "a", _a.get()); + ::visit(visitor, "b", _b.get()); + ::visit(visitor, "c", _c.get()); +} + +const TensorFunction & +Mixed112DotProduct::optimize(const TensorFunction &expr, Stash &stash) +{ + auto reduce = as<Reduce>(expr); + if (reduce && (reduce->aggr() == Aggr::SUM) && expr.result_type().is_double()) { + auto join = as<Join>(reduce->child()); + if (join && (join->function() == Mul::f)) { + FindInputs inputs; + if (inputs.try_match(join->lhs(), join->rhs()) || + inputs.try_match(join->rhs(), join->lhs())) + { + return stash.create<Mixed112DotProduct>(*inputs.a, *inputs.b, *inputs.c); + } + } + } + return expr; +} + +} // namespace diff --git a/eval/src/vespa/eval/instruction/mixed_112_dot_product.h b/eval/src/vespa/eval/instruction/mixed_112_dot_product.h new file mode 100644 index 00000000000..c02ccf65032 --- /dev/null +++ b/eval/src/vespa/eval/instruction/mixed_112_dot_product.h @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/tensor_function.h> + +namespace vespalib::eval { + +/** + * Tensor function for the dot product between (the expansion of a 1d + * sparse tensor and a 1d dense tensor) and (a 2d mixed tensor). + */ +class Mixed112DotProduct : public tensor_function::Node +{ +private: + Child _a; // 1d sparse + Child _b; // 1d dense + Child _c; // 2d mixed + +public: + Mixed112DotProduct(const TensorFunction &a_in, + const TensorFunction &b_in, + const TensorFunction &c_in); + InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + bool result_is_mutable() const override { return true; } + void push_children(std::vector<Child::CREF> &children) const final override; + void visit_children(vespalib::ObjectVisitor &visitor) const final override; + static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); +}; + +} // namespace diff --git a/fat-model-dependencies/pom.xml b/fat-model-dependencies/pom.xml index 97e37aa2192..180a39778a8 100644 --- a/fat-model-dependencies/pom.xml +++ b/fat-model-dependencies/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>fat-model-dependencies</artifactId> <packaging>pom</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -131,11 +131,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>statistics</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>container-messagebus</artifactId> <version>${project.version}</version> </dependency> diff --git a/fileacquirer/pom.xml b/fileacquirer/pom.xml index 1a85849fd98..ed02667b2a0 100644 --- a/fileacquirer/pom.xml +++ b/fileacquirer/pom.xml @@ -6,11 +6,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>fileacquirer</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <dependency> <groupId>junit</groupId> diff --git a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java index 5ab2d3d7246..c85a24db9bb 100644 --- a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java +++ b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java @@ -38,10 +38,8 @@ class FileAcquirerImpl implements FileAcquirer { private final Supervisor supervisor = new Supervisor(new Transport("fileaquirer")); - @SuppressWarnings("removal") // TODO Vespa 8: remove private final ConfigSubscriber configSubscriber; - @SuppressWarnings("removal") // TODO Vespa 8: remove private class Connection implements ConfigSubscriber.SingleSubscriber<FiledistributorrpcConfig> { private final Lock targetLock = new ReentrantLock(); private Target target; @@ -125,7 +123,6 @@ class FileAcquirerImpl implements FileAcquirer { } } - @SuppressWarnings("removal") // TODO Vespa 8: remove public FileAcquirerImpl(String configId) { configSubscriber = new ConfigSubscriber(); configSubscriber.subscribe(connection, FiledistributorrpcConfig.class, configId); diff --git a/filedistribution/pom.xml b/filedistribution/pom.xml index 9dca254cebe..7916698b62c 100644 --- a/filedistribution/pom.xml +++ b/filedistribution/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>filedistribution</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionConnectionPool.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionConnectionPool.java index c292080a0e5..3a03e6a87d5 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionConnectionPool.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionConnectionPool.java @@ -22,7 +22,6 @@ import java.util.List; */ public class FileDistributionConnectionPool extends JRTConnectionPool { - @SuppressWarnings("removal") // TODO Vespa 8: remove public FileDistributionConnectionPool(ConfigSourceSet sourceSet, Supervisor supervisor) { super(sourceSet, supervisor); } diff --git a/flags/pom.xml b/flags/pom.xml index 10e0249ce80..cf230c1b7a9 100644 --- a/flags/pom.xml +++ b/flags/pom.xml @@ -8,13 +8,13 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>flags</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>${project.artifactId}</name> <description>Feature flags</description> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index b4f14471af1..21170e6979c 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -215,7 +215,7 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag USE_V8_GEO_POSITIONS = defineFeatureFlag( - "use-v8-geo-positions", false, + "use-v8-geo-positions", true, List.of("arnej"), "2021-11-15", "2022-12-31", "Use Vespa 8 types and formats for geographical positions", "Takes effect at redeployment", @@ -301,14 +301,14 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag USE_QRSERVER_SERVICE_NAME = defineFeatureFlag( - "use-qrserver-service-name", true, + "use-qrserver-service-name", false, List.of("arnej"), "2022-01-18", "2022-12-31", "Use backwards-compatible 'qrserver' service name for containers with only 'search' API", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag AVOID_RENAMING_SUMMARY_FEATURES = defineFeatureFlag( - "avoid-renaming-summary-features", false, + "avoid-renaming-summary-features", true, List.of("arnej"), "2022-01-15", "2023-12-31", "Tell backend about the original name of summary-features that were wrapped in a rankingExpression feature", "Takes effect at redeployment", diff --git a/fsa/pom.xml b/fsa/pom.xml index db863ba5522..5d18bdb666c 100644 --- a/fsa/pom.xml +++ b/fsa/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>fsa</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <description> This is the FSA library and tools, an implementation of finite state automata (FSA) and related algorithms. diff --git a/hosted-api/pom.xml b/hosted-api/pom.xml index a9d8e7eee46..6f7c6186ece 100644 --- a/hosted-api/pom.xml +++ b/hosted-api/pom.xml @@ -8,7 +8,7 @@ <parent> <artifactId>parent</artifactId> <groupId>com.yahoo.vespa</groupId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>hosted-api</artifactId> diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml index fc9257d1db3..2bb292a50f6 100644 --- a/hosted-tenant-base/pom.xml +++ b/hosted-tenant-base/pom.xml @@ -7,7 +7,7 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>hosted-tenant-base</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>Base pom for all tenant base poms</name> <description>Parent POM for all Vespa base poms.</description> <url>https://github.com/vespa-engine</url> @@ -195,10 +195,10 @@ <configuration> <rules> <requireJavaVersion> - <version>[11, )</version> + <version>[17, )</version> </requireJavaVersion> <requireMavenVersion> - <version>[3.5, )</version> + <version>[3.6, )</version> </requireMavenVersion> </rules> </configuration> diff --git a/hosted-zone-api/abi-spec.json b/hosted-zone-api/abi-spec.json index 11375d97972..826f731b4e0 100644 --- a/hosted-zone-api/abi-spec.json +++ b/hosted-zone-api/abi-spec.json @@ -73,7 +73,6 @@ ], "methods": [ "public void <init>(ai.vespa.cloud.ApplicationId, ai.vespa.cloud.Zone, ai.vespa.cloud.Cluster, ai.vespa.cloud.Node)", - "public void <init>(ai.vespa.cloud.Zone, ai.vespa.cloud.Cluster, ai.vespa.cloud.Node)", "public ai.vespa.cloud.ApplicationId application()", "public ai.vespa.cloud.Zone zone()", "public ai.vespa.cloud.Cluster cluster()", @@ -106,7 +105,6 @@ ], "methods": [ "public void <init>(ai.vespa.cloud.ApplicationId, ai.vespa.cloud.Zone)", - "public void <init>(ai.vespa.cloud.Zone)", "public ai.vespa.cloud.ApplicationId application()", "public ai.vespa.cloud.Zone zone()", "public static ai.vespa.cloud.ZoneInfo defaultInfo()" diff --git a/hosted-zone-api/pom.xml b/hosted-zone-api/pom.xml index ea3b035618b..87012df8479 100644 --- a/hosted-zone-api/pom.xml +++ b/hosted-zone-api/pom.xml @@ -5,7 +5,7 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>hosted-zone-api</artifactId> diff --git a/hosted-zone-api/src/main/java/ai/vespa/cloud/SystemInfo.java b/hosted-zone-api/src/main/java/ai/vespa/cloud/SystemInfo.java index 3789c49fe82..752e78b580f 100644 --- a/hosted-zone-api/src/main/java/ai/vespa/cloud/SystemInfo.java +++ b/hosted-zone-api/src/main/java/ai/vespa/cloud/SystemInfo.java @@ -23,12 +23,6 @@ public class SystemInfo { this.node = Objects.requireNonNull(node, "Node cannot be null"); } - /** @deprecated pass an application id */ - @Deprecated // Remove on Vespa 8 - public SystemInfo(Zone zone, Cluster cluster, Node node) { - this(new ApplicationId("default", "default", "default"), zone, cluster, node); - } - /** Returns the application this is running as a part of */ public ApplicationId application() { return application; } diff --git a/hosted-zone-api/src/main/java/ai/vespa/cloud/ZoneInfo.java b/hosted-zone-api/src/main/java/ai/vespa/cloud/ZoneInfo.java index e4b69caa940..f29b921115e 100644 --- a/hosted-zone-api/src/main/java/ai/vespa/cloud/ZoneInfo.java +++ b/hosted-zone-api/src/main/java/ai/vespa/cloud/ZoneInfo.java @@ -25,12 +25,6 @@ public class ZoneInfo { this.zone = Objects.requireNonNull(zone, "Zone cannot be null!"); } - /** @deprecated pass an application id */ - @Deprecated // Remove on Vespa 8 - public ZoneInfo(Zone zone) { - this(new ApplicationId("default", "default", "default"), zone); - } - /** Returns the application this is running as part of */ public ApplicationId application() { return application; } diff --git a/http-client/pom.xml b/http-client/pom.xml index 75e9bdc27cb..49512c09a72 100644 --- a/http-client/pom.xml +++ b/http-client/pom.xml @@ -8,7 +8,7 @@ <parent> <artifactId>parent</artifactId> <groupId>com.yahoo.vespa</groupId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <packaging>container-plugin</packaging> diff --git a/http-utils/pom.xml b/http-utils/pom.xml index 4a84fb820eb..52dac0cfeea 100644 --- a/http-utils/pom.xml +++ b/http-utils/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>http-utils</artifactId> <packaging>jar</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <!-- provided --> diff --git a/indexinglanguage/pom.xml b/indexinglanguage/pom.xml index 48fa89c428e..040a13efc91 100644 --- a/indexinglanguage/pom.xml +++ b/indexinglanguage/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>indexinglanguage</artifactId> <packaging>jar</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>indexinglanguage</name> <description>Interpreter for the Indexing Language</description> <dependencies> diff --git a/integration/intellij/pom.xml b/integration/intellij/pom.xml index dfc44ab9dc2..d7146a60f7a 100644 --- a/integration/intellij/pom.xml +++ b/integration/intellij/pom.xml @@ -5,7 +5,7 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa-intellij</artifactId> <!-- Not used - plugin is build by gradle --> diff --git a/jaxrs_utils/pom.xml b/jaxrs_utils/pom.xml index 492bc095a57..cb7839bc73d 100644 --- a/jaxrs_utils/pom.xml +++ b/jaxrs_utils/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>jaxrs_utils</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc-cloud-aws/pom.xml b/jdisc-cloud-aws/pom.xml index 41259a7fb42..5ae3e056309 100644 --- a/jdisc-cloud-aws/pom.xml +++ b/jdisc-cloud-aws/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>jdisc-cloud-aws</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> diff --git a/jdisc-security-filters/pom.xml b/jdisc-security-filters/pom.xml index bda91f0a812..dcfb61de3cc 100644 --- a/jdisc-security-filters/pom.xml +++ b/jdisc-security-filters/pom.xml @@ -7,12 +7,12 @@ <modelVersion>4.0.0</modelVersion> <artifactId>jdisc-security-filters</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <dependencies> @@ -29,6 +29,12 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>security-utils</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> <!-- test --> <dependency> diff --git a/jdisc_core/abi-spec.json b/jdisc_core/abi-spec.json index 06e76f88898..5beb723465b 100644 --- a/jdisc_core/abi-spec.json +++ b/jdisc_core/abi-spec.json @@ -32,7 +32,6 @@ ], "methods": [ "public abstract com.yahoo.jdisc.handler.RequestHandler resolveHandler(com.yahoo.jdisc.Request)", - "public java.lang.Object getInstance(com.google.inject.Key)", "public abstract java.lang.Object getInstance(java.lang.Class)" ], "fields": [] @@ -179,9 +178,7 @@ "public void <init>(com.yahoo.jdisc.Request, java.net.URI)", "public com.yahoo.jdisc.Container container()", "public java.net.URI getUri()", - "public com.yahoo.jdisc.Request setUri(java.net.URI)", "public boolean isServerRequest()", - "public com.yahoo.jdisc.Request setServerRequest(boolean)", "public com.yahoo.jdisc.application.BindingMatch getBindingMatch()", "public com.yahoo.jdisc.Request setBindingMatch(com.yahoo.jdisc.application.BindingMatch)", "public java.util.Map context()", diff --git a/jdisc_core/pom.xml b/jdisc_core/pom.xml index 1943e2905fa..3ab469983c5 100644 --- a/jdisc_core/pom.xml +++ b/jdisc_core/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>jdisc_core</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> <dependencies> @@ -42,11 +42,6 @@ <version>1.4.01</version> </dependency> <dependency> - <groupId>javax.annotation</groupId> - <artifactId>javax.annotation-api</artifactId> - <version>1.2</version> - </dependency> - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> @@ -79,30 +74,6 @@ <classifier>no_aop</classifier> <!-- Non-AOP version required for Java 8 compatibility --> </dependency> <dependency> - <groupId>com.google.inject.extensions</groupId> - <artifactId>guice-assistedinject</artifactId> - <scope>compile</scope> - <exclusions> - <!-- Depends on AOP version of Guice 3.0, which is not Java 8 compatible --> - <exclusion> - <groupId>com.google.inject</groupId> - <artifactId>guice</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>com.google.inject.extensions</groupId> - <artifactId>guice-multibindings</artifactId> - <scope>compile</scope> - <exclusions> - <!-- Depends on AOP version of Guice 3.0, which is not Java 8 compatible --> - <exclusion> - <groupId>com.google.inject</groupId> - <artifactId>guice</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> <groupId>org.apache.felix</groupId> <artifactId>org.apache.felix.framework</artifactId> <scope>compile</scope> @@ -244,10 +215,6 @@ <argument>${exportPackagesFile}</argument> <argument>__REPLACE_VERSION__${project.build.directory}/dependency/guava.jar</argument> <argument>${project.build.directory}/dependency/guice-no_aop.jar</argument> - <argument>${project.build.directory}/dependency/guice-assistedinject.jar</argument> - <argument>${project.build.directory}/dependency/guice-multibindings.jar</argument> - <argument>${project.build.directory}/dependency/javax.annotation-api.jar</argument> - <argument>${project.build.directory}/dependency/org.apache.felix.log.jar</argument> <argument>${project.build.directory}/dependency/slf4j-api.jar</argument> <argument>${project.build.directory}/dependency/slf4j-jdk14.jar</argument> <argument>${project.build.directory}/dependency/jcl-over-slf4j.jar</argument> diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/Container.java b/jdisc_core/src/main/java/com/yahoo/jdisc/Container.java index e96f7f08fe8..b6deda58226 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/Container.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/Container.java @@ -2,7 +2,6 @@ package com.yahoo.jdisc; import com.google.inject.ConfigurationException; -import com.google.inject.Key; import com.google.inject.ProvisionException; import com.yahoo.jdisc.application.Application; import com.yahoo.jdisc.application.BindingSet; @@ -41,20 +40,6 @@ public interface Container extends SharedResource, Timer { RequestHandler resolveHandler(Request request); /** - * Returns the appropriate instance for the given injection key. When feasible, avoid using this method in favor - * of having Guice inject your dependencies ahead of time. - * - * @param key The key of the instance to return. - * @param <T> The class of the instance to return. - * @return The appropriate instance of the given class. - * @throws ConfigurationException If this injector cannot find or create the provider. - * @throws ProvisionException If there was a runtime failure while providing an instance. - * @deprecated Use {@link #getInstance(Class)} instead. - */ - @Deprecated(forRemoval = true, since = "7") // TODO Vespa 8 remove - default <T> T getInstance(Key<T> key) { throw new UnsupportedOperationException(); } - - /** * Returns the appropriate instance for the given injection type. When feasible, avoid using this method in * favor of having Guice inject your dependencies ahead of time. * diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java b/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java index c94ac5d9c60..c219ae06846 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java @@ -42,12 +42,12 @@ public class Request extends AbstractResource { private final Request parent; private final ResourceReference parentReference; private final long creationTime; + private final boolean serverRequest; + private final URI uri; private volatile boolean cancel = false; private BindingMatch<RequestHandler> bindingMatch; private TimeoutManager timeoutManager; - private boolean serverRequest; // TODO could be final, only used in tests private Long timeout; - private URI uri; // TODO Could be made final, public enum RequestType { READ, WRITE, MONITORING @@ -86,7 +86,7 @@ public class Request extends AbstractResource { parent = null; parentReference = null; serverRequest = isServerRequest; - setUri(uri); + this.uri = uri.normalize(); container = current.newReference(uri, this); creationTime = container.currentTimeMillis(); } @@ -120,7 +120,7 @@ public class Request extends AbstractResource { container = null; creationTime = parent.container().currentTimeMillis(); serverRequest = false; - setUri(uri); + this.uri = uri.normalize(); parentReference = this.parent.refer(this); } @@ -132,27 +132,10 @@ public class Request extends AbstractResource { /** * Returns the Uniform Resource Identifier used by the {@link Container} to resolve the appropriate {@link * RequestHandler} for this Request. - * - * @see #setUri(URI) */ public URI getUri() { return uri; } /** - * Sets the Uniform Resource Identifier used by the {@link Container} to resolve the appropriate {@link - * RequestHandler} for this Request. Because access to the URI is not guarded by any lock, any changes made after - * calling {@link #connect(ResponseHandler)} might never become visible to other threads. - * - * @param uri the URI to set - * @return this, to allow chaining - * @see #getUri() - */ - @Deprecated - public Request setUri(URI uri) { - this.uri = uri.normalize(); - return this; - } - - /** * Returns whether or not this Request was created by a {@link ServerProvider}. The value of this is used by * {@link Container#resolveHandler(Request)} to decide whether to match against server- or client-bindings. * @@ -162,20 +145,6 @@ public class Request extends AbstractResource { return serverRequest; } - /** - * Sets whether or not this Request was created by a {@link ServerProvider}. The constructor that accepts a - * {@link CurrentContainer} sets this to <em>true</em>, whereas the constructor that accepts a parent Request sets - * this to <em>false</em>. - * - * @param serverRequest whether or not this is a server request - * @return this, to allow chaining - * @see #isServerRequest() - */ - @Deprecated - public Request setServerRequest(boolean serverRequest) { - this.serverRequest = serverRequest; - return this; - } /** * Returns the last resolved {@link BindingMatch}, or null if none has been resolved yet. This is set diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ContainerSnapshot.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ContainerSnapshot.java index 808c8e89b1b..8367ea13a1a 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ContainerSnapshot.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ContainerSnapshot.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.core; -import com.google.inject.Key; import com.yahoo.jdisc.AbstractResource; import com.yahoo.jdisc.Container; import com.yahoo.jdisc.Request; @@ -37,14 +36,6 @@ class ContainerSnapshot extends AbstractResource implements Container { this.containerReference = container.refer(context); } - /** @deprecated Use {@link #getInstance(Class)} instead. */ - @Override - @Deprecated(forRemoval = true, since = "7") // TODO Vespa 8 remove - @SuppressWarnings("removal") - public <T> T getInstance(Key<T> key) { - return container.guiceInjector().getInstance(key); - } - @Override public <T> T getInstance(Class<T> type) { return container.guiceInjector().getInstance(type); diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java index 3a706101790..53acd8cbb1b 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java @@ -53,7 +53,8 @@ public class ExportPackages { .append("com.yahoo.jdisc.handler, ") .append("com.yahoo.jdisc.service, ") .append("com.yahoo.jdisc.statistics, ") - .append("javax.inject;version=1.0.0, ") // Included in guice, but not exported. Needed by container-jersey. + + .append("javax.inject;version=1.0.0, ") // TODO Vespa 9: remove. Included in guice, but not exported. Needed by container-jersey. .append("org.aopalliance.intercept, ") .append("org.aopalliance.aop"); diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogHandler.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogHandler.java index b7993de5d82..48fdb2a0293 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogHandler.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogHandler.java @@ -130,7 +130,7 @@ class OsgiLogHandler extends Handler { case SOURCE_METHOD_NAME: return record.getSourceMethodName(); case THREAD_ID: - return record.getThreadID(); + return record.getLongThreadID(); case THROWN: return record.getThrown(); default: diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/FastContentOutputStream.java b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/FastContentOutputStream.java index e001db2ab81..e344dc6a159 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/FastContentOutputStream.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/FastContentOutputStream.java @@ -12,7 +12,7 @@ import java.util.concurrent.TimeoutException; /** * <p>This class extends the {@link AbstractContentOutputStream}, and forwards all write() and close() calls to a {@link * FastContentWriter}. This means that once {@link #close()} has been called, the asynchronous completion of all pending - * operations can be awaited using the ListenableFuture interface of this class. Any asynchronous failure will be + * operations can be awaited using the {@link Future} interface of this class. Any asynchronous failure will be * rethrown when calling either of the get() methods on this class.</p> * <p>Please notice that the Future implementation of this class will NEVER complete unless {@link #close()} has been * called.</p> diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/FastContentWriter.java b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/FastContentWriter.java index 7c278c67d59..fefd2ee6ee7 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/FastContentWriter.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/FastContentWriter.java @@ -11,7 +11,7 @@ import java.util.concurrent.atomic.AtomicInteger; /** * <p>This class provides a non-blocking, awaitable <em>write</em>-interface to a {@link ContentChannel}. - * The ListenableFuture<Boolean> interface can be used to await + * The {@link CompletableFuture<Boolean>} interface can be used to await * the asynchronous completion of all pending operations. Any asynchronous * failure will be rethrown when calling either of the get() methods on * this class.</p> diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/test/ServerProviderConformanceTest.java b/jdisc_core/src/main/java/com/yahoo/jdisc/test/ServerProviderConformanceTest.java index 38d4c70646c..1cf4e3dd858 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/test/ServerProviderConformanceTest.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/test/ServerProviderConformanceTest.java @@ -153,7 +153,7 @@ public abstract class ServerProviderConformanceTest { final Module... config) throws Throwable { runTest(adapter, - Modules.override(Modules.combine()).with(newBindingSetSelector("unknown")), + Modules.override(Modules.EMPTY_MODULE).with(newBindingSetSelector("unknown")), RequestType.WITHOUT_CONTENT, new TestRequestHandler() { @@ -170,7 +170,7 @@ public abstract class ServerProviderConformanceTest { final Module... config) throws Throwable { runTest(adapter, - Modules.override(Modules.combine()).with(newBindingSetSelector(null)), + Modules.override(Modules.EMPTY_MODULE).with(newBindingSetSelector(null)), RequestType.WITHOUT_CONTENT, new TestRequestHandler() { diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/RequestTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/RequestTestCase.java index 75154dd588e..0d665e297da 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/RequestTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/RequestTestCase.java @@ -33,7 +33,6 @@ import static org.junit.Assert.fail; public class RequestTestCase { @Test - @SuppressWarnings("deprecation") public void requireThatAccessorsWork() throws BindingSetNotFoundException { MyTimer timer = new MyTimer(); timer.currentTime = 69; @@ -43,11 +42,7 @@ public class RequestTestCase { Request request = new Request(driver, URI.create("http://foo/bar")); assertNotNull(request); assertEquals(URI.create("http://foo/bar"), request.getUri()); - request.setUri(URI.create("http://baz/cox")); - assertEquals(URI.create("http://baz/cox"), request.getUri()); assertTrue(request.isServerRequest()); - request.setServerRequest(false); - assertFalse(request.isServerRequest()); assertEquals(69, request.creationTime(TimeUnit.MILLISECONDS)); assertNull(request.getTimeout(TimeUnit.MILLISECONDS)); request.setTimeout(10, TimeUnit.MILLISECONDS); @@ -249,14 +244,10 @@ public class RequestTestCase { assertTrue(driver.close()); } - @SuppressWarnings("deprecation") private static void assertUri(CurrentContainer container, String requestUri, String expectedUri) { Request serverReq = new Request(container, URI.create(requestUri)); assertEquals(expectedUri, serverReq.getUri().toString()); - serverReq.setUri(URI.create(requestUri)); - assertEquals(expectedUri, serverReq.getUri().toString()); - Request clientReq = new Request(serverReq, URI.create(requestUri)); assertEquals(expectedUri, clientReq.getUri().toString()); diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/OsgiLogHandlerTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/OsgiLogHandlerTestCase.java index 6ae377a91a4..7c07623814e 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/core/OsgiLogHandlerTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/core/OsgiLogHandlerTestCase.java @@ -86,7 +86,7 @@ public class OsgiLogHandlerTestCase { record.setSequenceNumber(69); record.setSourceClassName("sourceClassName"); record.setSourceMethodName("sourceMethodName"); - record.setThreadID(69); + record.setLongThreadID(69L); Throwable thrown = new Throwable(); record.setThrown(thrown); log.log(record); @@ -116,7 +116,7 @@ public class OsgiLogHandlerTestCase { assertEquals(69L, ref.getProperty("SEQUENCE_NUMBER")); assertEquals("sourceClassName", ref.getProperty("SOURCE_CLASS_NAME")); assertEquals("sourceMethodName", ref.getProperty("SOURCE_METHOD_NAME")); - assertEquals(69, ref.getProperty("THREAD_ID")); + assertEquals(69L, ref.getProperty("THREAD_ID")); assertSame(thrown, ref.getProperty("THROWN")); assertNull(ref.getProperty("unknown")); } diff --git a/jdisc_core_test/integration_test/pom.xml b/jdisc_core_test/integration_test/pom.xml index 7aadee990c6..cee12f1b26a 100644 --- a/jdisc_core_test/integration_test/pom.xml +++ b/jdisc_core_test/integration_test/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>integration-test-parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>integration_test</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/pom.xml b/jdisc_core_test/pom.xml index 4216c25266e..0cf340e907a 100644 --- a/jdisc_core_test/pom.xml +++ b/jdisc_core_test/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>integration-test-parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>pom</packaging> <name>${project.artifactId}</name> <modules> diff --git a/jdisc_core_test/test_bundles/app-a/pom.xml b/jdisc_core_test/test_bundles/app-a/pom.xml index 9b8304284ae..b13b4b699c7 100644 --- a/jdisc_core_test/test_bundles/app-a/pom.xml +++ b/jdisc_core_test/test_bundles/app-a/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>app-a</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/app-b-priv/pom.xml b/jdisc_core_test/test_bundles/app-b-priv/pom.xml index 3b595562718..852f70441cc 100644 --- a/jdisc_core_test/test_bundles/app-b-priv/pom.xml +++ b/jdisc_core_test/test_bundles/app-b-priv/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>app-b-priv</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/app-ca/pom.xml b/jdisc_core_test/test_bundles/app-ca/pom.xml index 6126af80de0..ce2d977c2a5 100644 --- a/jdisc_core_test/test_bundles/app-ca/pom.xml +++ b/jdisc_core_test/test_bundles/app-ca/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>app-ca</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/app-dj/pom.xml b/jdisc_core_test/test_bundles/app-dj/pom.xml index e17020bdd0c..9f6bae624dc 100644 --- a/jdisc_core_test/test_bundles/app-dj/pom.xml +++ b/jdisc_core_test/test_bundles/app-dj/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>app-dj</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/app-ej-priv/pom.xml b/jdisc_core_test/test_bundles/app-ej-priv/pom.xml index 7d55bfbe895..fb05e6bca86 100644 --- a/jdisc_core_test/test_bundles/app-ej-priv/pom.xml +++ b/jdisc_core_test/test_bundles/app-ej-priv/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>app-ej-priv</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/app-f-more/pom.xml b/jdisc_core_test/test_bundles/app-f-more/pom.xml index a488b333092..00fd8977c06 100644 --- a/jdisc_core_test/test_bundles/app-f-more/pom.xml +++ b/jdisc_core_test/test_bundles/app-f-more/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>app-f-more</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/app-g-act/pom.xml b/jdisc_core_test/test_bundles/app-g-act/pom.xml index a8389b8ce33..ea849ad295d 100644 --- a/jdisc_core_test/test_bundles/app-g-act/pom.xml +++ b/jdisc_core_test/test_bundles/app-g-act/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>app-g-act</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/app-h-log/pom.xml b/jdisc_core_test/test_bundles/app-h-log/pom.xml index 99a82c5e9b1..e1e0c2e8de3 100644 --- a/jdisc_core_test/test_bundles/app-h-log/pom.xml +++ b/jdisc_core_test/test_bundles/app-h-log/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>app-h-log</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-a/pom.xml b/jdisc_core_test/test_bundles/cert-a/pom.xml index 68986875b56..17decc78ffd 100644 --- a/jdisc_core_test/test_bundles/cert-a/pom.xml +++ b/jdisc_core_test/test_bundles/cert-a/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-a</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-b/pom.xml b/jdisc_core_test/test_bundles/cert-b/pom.xml index 287d914cebc..587826a64b1 100644 --- a/jdisc_core_test/test_bundles/cert-b/pom.xml +++ b/jdisc_core_test/test_bundles/cert-b/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-b</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-ca/pom.xml b/jdisc_core_test/test_bundles/cert-ca/pom.xml index 32b9b1e58f9..37621a927ef 100644 --- a/jdisc_core_test/test_bundles/cert-ca/pom.xml +++ b/jdisc_core_test/test_bundles/cert-ca/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-ca</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/cert-dc/pom.xml b/jdisc_core_test/test_bundles/cert-dc/pom.xml index 77bb210fc87..114ab11d96d 100644 --- a/jdisc_core_test/test_bundles/cert-dc/pom.xml +++ b/jdisc_core_test/test_bundles/cert-dc/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-dc</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/cert-eab/pom.xml b/jdisc_core_test/test_bundles/cert-eab/pom.xml index 7870603defe..09f432f42dc 100644 --- a/jdisc_core_test/test_bundles/cert-eab/pom.xml +++ b/jdisc_core_test/test_bundles/cert-eab/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-eab</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/cert-fac/pom.xml b/jdisc_core_test/test_bundles/cert-fac/pom.xml index 9bc7c947862..b7beb071edc 100644 --- a/jdisc_core_test/test_bundles/cert-fac/pom.xml +++ b/jdisc_core_test/test_bundles/cert-fac/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-fac</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/cert-gg/pom.xml b/jdisc_core_test/test_bundles/cert-gg/pom.xml index 09c90f2c146..7ecfca57bcc 100644 --- a/jdisc_core_test/test_bundles/cert-gg/pom.xml +++ b/jdisc_core_test/test_bundles/cert-gg/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-gg</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-hi/pom.xml b/jdisc_core_test/test_bundles/cert-hi/pom.xml index b4d66abca2c..82e5d9f1d81 100644 --- a/jdisc_core_test/test_bundles/cert-hi/pom.xml +++ b/jdisc_core_test/test_bundles/cert-hi/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-hi</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-ih/pom.xml b/jdisc_core_test/test_bundles/cert-ih/pom.xml index 5797fdd145b..0d546f96965 100644 --- a/jdisc_core_test/test_bundles/cert-ih/pom.xml +++ b/jdisc_core_test/test_bundles/cert-ih/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-ih</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-j-priv/pom.xml b/jdisc_core_test/test_bundles/cert-j-priv/pom.xml index 65383b2cddb..0d818144820 100644 --- a/jdisc_core_test/test_bundles/cert-j-priv/pom.xml +++ b/jdisc_core_test/test_bundles/cert-j-priv/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-j-priv</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-k-pkgs/pom.xml b/jdisc_core_test/test_bundles/cert-k-pkgs/pom.xml index e9ad347905b..b810c0fb19f 100644 --- a/jdisc_core_test/test_bundles/cert-k-pkgs/pom.xml +++ b/jdisc_core_test/test_bundles/cert-k-pkgs/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-k-pkgs</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-k-pkgs/src/main/java/com/yahoo/jdisc/bundle/k/CertificateK.java b/jdisc_core_test/test_bundles/cert-k-pkgs/src/main/java/com/yahoo/jdisc/bundle/k/CertificateK.java index f8d85233602..34ff4124cd5 100644 --- a/jdisc_core_test/test_bundles/cert-k-pkgs/src/main/java/com/yahoo/jdisc/bundle/k/CertificateK.java +++ b/jdisc_core_test/test_bundles/cert-k-pkgs/src/main/java/com/yahoo/jdisc/bundle/k/CertificateK.java @@ -20,7 +20,6 @@ public class CertificateK { private final com.google.common.reflect.Reflection reflection = null; private final com.google.common.util.concurrent.AbstractIdleService idleService = null; private final com.google.inject.AbstractModule module = null; - private final com.google.inject.assistedinject.Assisted assisted = null; private final com.google.inject.binder.AnnotatedConstantBindingBuilder builder = null; private final com.google.inject.matcher.Matchers matchers = null; private final com.google.inject.multibindings.MapBinder<String, String> mapBinder = null; @@ -32,14 +31,12 @@ public class CertificateK { private final com.yahoo.jdisc.application.AbstractApplication application = null; private final com.yahoo.jdisc.handler.AbstractRequestHandler handler = null; private final com.yahoo.jdisc.service.AbstractClientProvider client = null; - private final com.yahoo.yolean.chain.Chain<?> chain = null; private final com.yahoo.yolean.concurrent.ThreadRobustList<?> robustList = null; private final com.yahoo.yolean.trace.TraceNode traceNode = null; private final com.sun.security.auth.LdapPrincipal principal = null; private final com.sun.security.auth.module.JndiLoginModule jndiLoginModule = null; private final java.security.cert.Certificate certificate = null; private final javax.accessibility.Accessible accessible = null; - private final javax.annotation.PostConstruct postConstruct = null; private final javax.annotation.processing.FilerException filerException = null; private final javax.crypto.BadPaddingException badPaddingException = null; private final javax.crypto.interfaces.DHKey dhKey = null; @@ -144,7 +141,6 @@ public class CertificateK { private final org.osgi.framework.AdminPermission permission = null; private final org.osgi.framework.hooks.service.FindHook hook = null; private final org.osgi.framework.launch.Framework framework = null; - private final org.osgi.service.log.LogEntry entry = null; private final org.osgi.service.packageadmin.ExportedPackage pkg = null; private final org.osgi.service.startlevel.StartLevel level = null; private final org.osgi.service.url.URLConstants constants = null; diff --git a/jdisc_core_test/test_bundles/cert-l1-dup/pom.xml b/jdisc_core_test/test_bundles/cert-l1-dup/pom.xml index 3eef44629ab..f48fd1e66f6 100644 --- a/jdisc_core_test/test_bundles/cert-l1-dup/pom.xml +++ b/jdisc_core_test/test_bundles/cert-l1-dup/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-l1-dup</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-l1/pom.xml b/jdisc_core_test/test_bundles/cert-l1/pom.xml index e89eeae449e..afee36532d4 100644 --- a/jdisc_core_test/test_bundles/cert-l1/pom.xml +++ b/jdisc_core_test/test_bundles/cert-l1/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-l1</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-l2/pom.xml b/jdisc_core_test/test_bundles/cert-l2/pom.xml index b00e1ecaec2..31fe9b1a6f2 100644 --- a/jdisc_core_test/test_bundles/cert-l2/pom.xml +++ b/jdisc_core_test/test_bundles/cert-l2/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-l2</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-ml-dup/pom.xml b/jdisc_core_test/test_bundles/cert-ml-dup/pom.xml index 021d14a1336..703c1ff73cb 100644 --- a/jdisc_core_test/test_bundles/cert-ml-dup/pom.xml +++ b/jdisc_core_test/test_bundles/cert-ml-dup/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-ml-dup</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-ml/pom.xml b/jdisc_core_test/test_bundles/cert-ml/pom.xml index f3e9013ce97..30e12cc35aa 100644 --- a/jdisc_core_test/test_bundles/cert-ml/pom.xml +++ b/jdisc_core_test/test_bundles/cert-ml/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-ml</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-nac/pom.xml b/jdisc_core_test/test_bundles/cert-nac/pom.xml index fe88e4cdd4f..73e4e62887e 100644 --- a/jdisc_core_test/test_bundles/cert-nac/pom.xml +++ b/jdisc_core_test/test_bundles/cert-nac/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-nac</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/cert-oa-path/pom.xml b/jdisc_core_test/test_bundles/cert-oa-path/pom.xml index cdc16200b11..621822f76eb 100644 --- a/jdisc_core_test/test_bundles/cert-oa-path/pom.xml +++ b/jdisc_core_test/test_bundles/cert-oa-path/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-oa-path</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/cert-p-jar/pom.xml b/jdisc_core_test/test_bundles/cert-p-jar/pom.xml index 1d6fe7bba4c..817bb7d3451 100644 --- a/jdisc_core_test/test_bundles/cert-p-jar/pom.xml +++ b/jdisc_core_test/test_bundles/cert-p-jar/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-p-jar</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-q-frag/pom.xml b/jdisc_core_test/test_bundles/cert-q-frag/pom.xml index 563e4eb60d4..b57d04be877 100644 --- a/jdisc_core_test/test_bundles/cert-q-frag/pom.xml +++ b/jdisc_core_test/test_bundles/cert-q-frag/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-q-frag</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-rq/pom.xml b/jdisc_core_test/test_bundles/cert-rq/pom.xml index 0070ddadeca..6162cdd6075 100644 --- a/jdisc_core_test/test_bundles/cert-rq/pom.xml +++ b/jdisc_core_test/test_bundles/cert-rq/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-rq</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/cert-s-act/pom.xml b/jdisc_core_test/test_bundles/cert-s-act/pom.xml index ed34708dc8a..e7350f70665 100644 --- a/jdisc_core_test/test_bundles/cert-s-act/pom.xml +++ b/jdisc_core_test/test_bundles/cert-s-act/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-s-act</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-tp/pom.xml b/jdisc_core_test/test_bundles/cert-tp/pom.xml index b657f36c695..6618b22efdd 100644 --- a/jdisc_core_test/test_bundles/cert-tp/pom.xml +++ b/jdisc_core_test/test_bundles/cert-tp/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-tp</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/cert-us/pom.xml b/jdisc_core_test/test_bundles/cert-us/pom.xml index 731b5af098f..b4306daa5e8 100644 --- a/jdisc_core_test/test_bundles/cert-us/pom.xml +++ b/jdisc_core_test/test_bundles/cert-us/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>cert-us</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/my-bundle-activator/pom.xml b/jdisc_core_test/test_bundles/my-bundle-activator/pom.xml index 5daef993b4a..deb8ddb80fc 100644 --- a/jdisc_core_test/test_bundles/my-bundle-activator/pom.xml +++ b/jdisc_core_test/test_bundles/my-bundle-activator/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>my-bundle-activator</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/my-guice-module/pom.xml b/jdisc_core_test/test_bundles/my-guice-module/pom.xml index 296ca98acb8..be8f22f3c40 100644 --- a/jdisc_core_test/test_bundles/my-guice-module/pom.xml +++ b/jdisc_core_test/test_bundles/my-guice-module/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>my-guice-module</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/my-server-provider/pom.xml b/jdisc_core_test/test_bundles/my-server-provider/pom.xml index 3fc2ed2a755..b5b78191010 100644 --- a/jdisc_core_test/test_bundles/my-server-provider/pom.xml +++ b/jdisc_core_test/test_bundles/my-server-provider/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>my-server-provider</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/pom.xml b/jdisc_core_test/test_bundles/pom.xml index ff3c14ee354..63f31c1963b 100644 --- a/jdisc_core_test/test_bundles/pom.xml +++ b/jdisc_core_test/test_bundles/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>integration-test-parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> </parent> <artifactId>test_bundles</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>pom</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_jetty/.gitignore b/jdisc_jetty/.gitignore deleted file mode 100644 index 016c6f704f0..00000000000 --- a/jdisc_jetty/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -pom.xml.build -/target diff --git a/jdisc_jetty/CMakeLists.txt b/jdisc_jetty/CMakeLists.txt deleted file mode 100644 index 7f72c61be20..00000000000 --- a/jdisc_jetty/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_jar(jdisc_jetty.jar) -install_jar_dependencies(jdisc_jetty) diff --git a/jdisc_jetty/OWNERS b/jdisc_jetty/OWNERS deleted file mode 100644 index 78b92e411b4..00000000000 --- a/jdisc_jetty/OWNERS +++ /dev/null @@ -1,2 +0,0 @@ -gjoranv -bjorncs diff --git a/jdisc_jetty/README b/jdisc_jetty/README deleted file mode 100644 index a09ee05e471..00000000000 --- a/jdisc_jetty/README +++ /dev/null @@ -1 +0,0 @@ -Jetty dependencies for JDisc Container. diff --git a/jdisc_jetty/pom.xml b/jdisc_jetty/pom.xml deleted file mode 100644 index 76140f67416..00000000000 --- a/jdisc_jetty/pom.xml +++ /dev/null @@ -1,77 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 - http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> - </parent> - <artifactId>jdisc_jetty</artifactId> - <version>7-SNAPSHOT</version> - <packaging>jar</packaging> - <dependencies> - <dependency> - <groupId>org.eclipse.jetty.alpn</groupId> - <artifactId>alpn-api</artifactId> - </dependency> - <dependency> - <groupId>org.eclipse.jetty.http2</groupId> - <artifactId>http2-server</artifactId> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-alpn-java-server</artifactId> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-client</artifactId> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-continuation</artifactId> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-server</artifactId> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-servlet</artifactId> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-servlets</artifactId> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-jmx</artifactId> - </dependency> - <dependency> - <!-- Required for ServiceLoader to function in OSGi environment. ServiceLoader is used by Jetty --> - <groupId>org.apache.aries.spifly</groupId> - <artifactId>org.apache.aries.spifly.dynamic.bundle</artifactId> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-dependency-plugin</artifactId> - <executions> - <execution> - <id>copy-dependencies</id> - <phase>package</phase> - <goals> - <goal>copy-dependencies</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - </build> -</project> diff --git a/jdisc_jetty/src/main/javadoc/README b/jdisc_jetty/src/main/javadoc/README deleted file mode 100644 index 6695538e308..00000000000 --- a/jdisc_jetty/src/main/javadoc/README +++ /dev/null @@ -1 +0,0 @@ -No javadoc available for module
\ No newline at end of file diff --git a/jrt/pom.xml b/jrt/pom.xml index fea45602e58..c4fb87d24c4 100644 --- a/jrt/pom.xml +++ b/jrt/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>jrt</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>jrt</name> <dependencies> <dependency> diff --git a/linguistics-components/abi-spec.json b/linguistics-components/abi-spec.json index 39666fd93a3..0db6fff0765 100644 --- a/linguistics-components/abi-spec.json +++ b/linguistics-components/abi-spec.json @@ -134,7 +134,6 @@ "public static java.lang.String getDefMd5()", "public static java.lang.String getDefName()", "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", "public void <init>(com.yahoo.language.sentencepiece.SentencePieceConfig$Builder)", "public boolean collapseUnknowns()", "public com.yahoo.language.sentencepiece.SentencePieceConfig$Scoring$Enum scoring()", @@ -145,7 +144,6 @@ "public static final java.lang.String CONFIG_DEF_MD5", "public static final java.lang.String CONFIG_DEF_NAME", "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", "public static final java.lang.String[] CONFIG_DEF_SCHEMA" ] }, @@ -273,7 +271,6 @@ "public static java.lang.String getDefMd5()", "public static java.lang.String getDefName()", "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", "public void <init>(com.yahoo.language.wordpiece.WordPieceConfig$Builder)", "public java.lang.String subwordPrefix()", "public java.util.List model()", @@ -283,7 +280,6 @@ "public static final java.lang.String CONFIG_DEF_MD5", "public static final java.lang.String CONFIG_DEF_NAME", "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", "public static final java.lang.String[] CONFIG_DEF_SCHEMA" ] }, diff --git a/linguistics-components/pom.xml b/linguistics-components/pom.xml index aa68f6590cb..eae50c778b5 100644 --- a/linguistics-components/pom.xml +++ b/linguistics-components/pom.xml @@ -7,12 +7,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>linguistics-components</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <dependency> <groupId>com.google.protobuf</groupId> diff --git a/linguistics/abi-spec.json b/linguistics/abi-spec.json index c3e489b8dd9..0961c87cbd8 100644 --- a/linguistics/abi-spec.json +++ b/linguistics/abi-spec.json @@ -734,8 +734,7 @@ "abstract" ], "methods": [ - "public abstract java.lang.Iterable tokenize(java.lang.String, com.yahoo.language.Language, com.yahoo.language.process.StemMode, boolean)", - "public java.lang.String getReplacementTerm(java.lang.String)" + "public abstract java.lang.Iterable tokenize(java.lang.String, com.yahoo.language.Language, com.yahoo.language.process.StemMode, boolean)" ], "fields": [] }, diff --git a/linguistics/pom.xml b/linguistics/pom.xml index d0314622fcf..763dff1d653 100644 --- a/linguistics/pom.xml +++ b/linguistics/pom.xml @@ -7,12 +7,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>linguistics</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <dependency> <groupId>junit</groupId> diff --git a/linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java b/linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java index 0edf96e009b..8125f6e504a 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/process/Tokenizer.java @@ -22,12 +22,4 @@ public interface Tokenizer { */ Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents); - /** - * Not used. - * - * @deprecated replacements are already applied in tokens returned by tokenize - */ - @Deprecated // Remove on Vespa 8 - default String getReplacementTerm(String tokenString) { return tokenString; } - } diff --git a/logd/pom.xml b/logd/pom.xml index 2baf77e5aa3..4eae0afc77b 100644 --- a/logd/pom.xml +++ b/logd/pom.xml @@ -7,11 +7,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>logd</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/logd/src/main/resources/configdefinitions/logd.def b/logd/src/main/resources/configdefinitions/logd.def index 58cf1913a2b..3bed6e267b2 100644 --- a/logd/src/main/resources/configdefinitions/logd.def +++ b/logd/src/main/resources/configdefinitions/logd.def @@ -10,14 +10,6 @@ logserver.host string default="localhost" ## RPC port of logserver. logserver.rpcport int default=5822 -## Legacy port to contact the logserver on. -## Deprecated (not used). TODO: Remove on Vespa 8. -logserver.port int default=5821 - -## Use RPC protocol -## Deprecated (always true). TODO: Remove on Vespa 8. -logserver.userpc bool default=true - ## Forward to a logserver. Other logserver configuration is irrelevant if false. logserver.use bool default=true diff --git a/logserver/bin/logserver-start.sh b/logserver/bin/logserver-start.sh index 02efda95798..72b31280106 100755 --- a/logserver/bin/logserver-start.sh +++ b/logserver/bin/logserver-start.sh @@ -78,7 +78,9 @@ ROOT=${VESPA_HOME%/} export ROOT cd $ROOT || { echo "Cannot cd to $ROOT" 1>&2; exit 1; } -addopts="-server -Xms32m -Xmx256m -XX:+PreserveFramePointer -XX:+UseTransparentHugePages -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=448 -XX:MaxJavaStackTraceDepth=1000 -XX:ActiveProcessorCount=2 -XX:-OmitStackTraceInFastThrow -Djava.io.tmpdir=${VESPA_HOME}/tmp" +heap_min=32 +heap_max=256 +addopts="-server -Xms${heap_min}m -Xmx${heap_max}m -XX:+PreserveFramePointer $(get_jvm_hugepage_settings $heap_max) -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=448 -XX:MaxJavaStackTraceDepth=1000 -XX:ActiveProcessorCount=2 -XX:-OmitStackTraceInFastThrow -Djava.io.tmpdir=${VESPA_HOME}/tmp" oomopt="-XX:+ExitOnOutOfMemoryError" diff --git a/logserver/pom.xml b/logserver/pom.xml index 2d5eecd01ce..61b777ad86b 100644 --- a/logserver/pom.xml +++ b/logserver/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>logserver</artifactId> <packaging>jar</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>logserver</name> <description>Log server.</description> <dependencies> diff --git a/logserver/src/main/java/com/yahoo/logserver/Server.java b/logserver/src/main/java/com/yahoo/logserver/Server.java index ff9939ff244..cfe2bcf0776 100644 --- a/logserver/src/main/java/com/yahoo/logserver/Server.java +++ b/logserver/src/main/java/com/yahoo/logserver/Server.java @@ -22,7 +22,6 @@ import java.util.logging.Logger; * @author Bjorn Borud * @author Stig Bakken */ -@SuppressWarnings("removal") // TODO Vespa 8: remove (com.yahoo.log.event) public class Server implements Runnable { private final AtomicBoolean signalCaught = new AtomicBoolean(false); static final String APPNAME = "logserver"; diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java index 59475972279..68a32def9ad 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java @@ -24,7 +24,6 @@ import com.yahoo.logserver.handlers.AbstractLogHandler; * * @author hmusum */ -@SuppressWarnings("removal") // TODO Vespa 8: remove (com.yahoo.log.event) public class LogMetricsHandler extends AbstractLogHandler { private static final long EVENTINTERVAL = 5 * 60; // in seconds diff --git a/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java b/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java index 0c9af994c3d..ec99f99b930 100644 --- a/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java +++ b/logserver/src/main/java/com/yahoo/logserver/testutils/VerifyLogfile.java @@ -17,7 +17,7 @@ import com.yahoo.log.LogMessage; * * @author Bjorn Borud */ -@SuppressWarnings({"deprecation","removal"}) // TODO Vespa 8: remove (com.yahoo.log.event) +@SuppressWarnings("deprecation") public class VerifyLogfile { public static void main (String[] args) throws IOException { diff --git a/maven-plugins/pom.xml b/maven-plugins/pom.xml index 806015c3100..6f8b44540e2 100644 --- a/maven-plugins/pom.xml +++ b/maven-plugins/pom.xml @@ -5,13 +5,13 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <groupId>com.yahoo.vespa</groupId> <artifactId>maven-plugins</artifactId> <packaging>pom</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <description>Parent artifact for Vespa maven plugins.</description> <url>http://yahoo.github.io/vespa</url> diff --git a/messagebus/abi-spec.json b/messagebus/abi-spec.json index 83bab8e4734..af298a96d7d 100644 --- a/messagebus/abi-spec.json +++ b/messagebus/abi-spec.json @@ -572,7 +572,6 @@ "public static java.lang.String getDefMd5()", "public static java.lang.String getDefName()", "public static java.lang.String getDefNamespace()", - "public static java.lang.String getDefVersion()", "public void <init>(com.yahoo.messagebus.MessagebusConfig$Builder)", "public java.util.List routingtable()", "public com.yahoo.messagebus.MessagebusConfig$Routingtable routingtable(int)" @@ -581,7 +580,6 @@ "public static final java.lang.String CONFIG_DEF_MD5", "public static final java.lang.String CONFIG_DEF_NAME", "public static final java.lang.String CONFIG_DEF_NAMESPACE", - "public static final java.lang.String CONFIG_DEF_VERSION", "public static final java.lang.String[] CONFIG_DEF_SCHEMA" ] }, diff --git a/messagebus/pom.xml b/messagebus/pom.xml index d26e9b32909..512c001e3ff 100644 --- a/messagebus/pom.xml +++ b/messagebus/pom.xml @@ -7,11 +7,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>messagebus</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>jar</packaging> <dependencies> <dependency> diff --git a/messagebus/src/main/java/com/yahoo/messagebus/ConfigAgent.java b/messagebus/src/main/java/com/yahoo/messagebus/ConfigAgent.java index 10ebb2fc1ae..4db97f0c083 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/ConfigAgent.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/ConfigAgent.java @@ -14,7 +14,6 @@ import com.yahoo.messagebus.routing.RoutingTableSpec; * * @author Simon Thoresen Hult */ -@SuppressWarnings("removal") // TODO Vespa 8: remove public class ConfigAgent implements ConfigSubscriber.SingleSubscriber<MessagebusConfig>{ private final ConfigURI configURI; diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/SlobrokConfigSubscriber.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/SlobrokConfigSubscriber.java index 02472d8f31f..903a31d3f3a 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/SlobrokConfigSubscriber.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/SlobrokConfigSubscriber.java @@ -13,7 +13,6 @@ import java.util.logging.Logger; * * @author Simon Thoresen Hult */ -@SuppressWarnings("removal") // TODO Vespa 8: remove public class SlobrokConfigSubscriber implements ConfigSubscriber.SingleSubscriber<SlobroksConfig>{ private static final Logger log = Logger.getLogger(SlobrokConfigSubscriber.class.getName()); diff --git a/messagebus/src/test/java/com/yahoo/messagebus/ConfigAgentTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/ConfigAgentTestCase.java index 287529ae449..a32175ff414 100755 --- a/messagebus/src/test/java/com/yahoo/messagebus/ConfigAgentTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/ConfigAgentTestCase.java @@ -22,7 +22,6 @@ public class ConfigAgentTestCase { @Rule public TemporaryFolder tmpFolder = new TemporaryFolder(); - @SuppressWarnings("removal") // TODO Vespa 8: remove @Test public void testRoutingConfig() throws InterruptedException { LocalHandler handler = new LocalHandler(); diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml index 76f7e92ad43..fc63d453738 100644 --- a/metrics-proxy/pom.xml +++ b/metrics-proxy/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>metrics-proxy</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <!-- provided --> @@ -142,11 +142,6 @@ <artifactId>junit</artifactId> <scope>test</scope> </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-http</artifactId> - <scope>test</scope> - </dependency> </dependencies> <build> <plugins> diff --git a/metrics/pom.xml b/metrics/pom.xml index b0482e26697..2cf5ccc57d1 100644 --- a/metrics/pom.xml +++ b/metrics/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>metrics</artifactId> <packaging>jar</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>metrics</name> <dependencies> <dependency> diff --git a/model-evaluation/abi-spec.json b/model-evaluation/abi-spec.json index 6728d5cd9b4..4ef00061852 100644 --- a/model-evaluation/abi-spec.json +++ b/model-evaluation/abi-spec.json @@ -85,7 +85,6 @@ "methods": [ "public void <init>(com.yahoo.filedistribution.fileacquirer.FileAcquirer)", "public java.util.Map importFrom(com.yahoo.vespa.config.search.RankProfilesConfig, com.yahoo.vespa.config.search.core.RankingConstantsConfig, com.yahoo.vespa.config.search.core.RankingExpressionsConfig, com.yahoo.vespa.config.search.core.OnnxModelsConfig)", - "public java.util.Map importFrom(com.yahoo.vespa.config.search.RankProfilesConfig, com.yahoo.vespa.config.search.core.RankingConstantsConfig, com.yahoo.vespa.config.search.core.OnnxModelsConfig)", "protected final java.lang.String readExpressionFromFile(java.io.File)", "protected com.yahoo.searchlib.rankingexpression.RankingExpression readExpressionFromFile(java.lang.String, com.yahoo.config.FileReference)", "protected com.yahoo.tensor.Tensor readTensorFromFile(java.lang.String, com.yahoo.tensor.TensorType, com.yahoo.config.FileReference)" diff --git a/model-evaluation/pom.xml b/model-evaluation/pom.xml index 5064d88ae63..caf28199c3d 100644 --- a/model-evaluation/pom.xml +++ b/model-evaluation/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>model-evaluation</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> @@ -79,10 +79,8 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>jdisc_jetty</artifactId> - <version>${project.version}</version> - <scope>test</scope> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> </dependency> </dependencies> <build> diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/LazyArrayContext.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/LazyArrayContext.java index 3314ecb23fc..d030108a17a 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/LazyArrayContext.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/LazyArrayContext.java @@ -260,7 +260,7 @@ public final class LazyArrayContext extends Context implements ContextIndex { } /** - * Extract the feature used to evaluate the onnx model. e.g. onnxModel(name) and add + * Extract the feature used to evaluate the onnx model. e.g. onnx(name) and add * that as a bind target and argument. During evaluation, this will be evaluated before * the rest of the expression and the result is added to the context. Also extract the * inputs to the model and add them as bind targets and arguments. diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java index 107a4c20a1c..c2cb1993fc0 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java @@ -71,13 +71,6 @@ public class RankProfilesConfigImporter { } } - @Deprecated - public Map<String, Model> importFrom(RankProfilesConfig config, - RankingConstantsConfig constantsConfig, - OnnxModelsConfig onnxModelsConfig) { - return importFrom(config, constantsConfig, new RankingExpressionsConfig.Builder().build(), onnxModelsConfig); - } - private Model importProfile(RankProfilesConfig.Rankprofile profile, RankingConstantsConfig constantsConfig, RankingExpressionsConfig expressionsConfig, diff --git a/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java b/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java index b0e2be26f8a..2661b9c2eb2 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java +++ b/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java @@ -90,13 +90,13 @@ public class ModelsEvaluationHandler extends ThreadedHttpRequestHandler { Tensor result = evaluator.evaluate(); Optional<String> format = property(request, "format.tensors"); - if (format.isPresent() && format.get().equalsIgnoreCase("short")) { - return new Response(200, JsonFormat.encodeShortForm(result)); + if (format.isPresent() && format.get().equalsIgnoreCase("long")) { + return new Response(200, JsonFormat.encode(result)); } else if (format.isPresent() && format.get().equalsIgnoreCase("string")) { return new Response(200, result.toString().getBytes(StandardCharsets.UTF_8)); } - return new Response(200, JsonFormat.encode(result)); + return new Response(200, JsonFormat.encodeShortForm(result)); } private HttpResponse listAllModels(HttpRequest request) { diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java index 3000c83b7ec..7790f8a60d0 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java @@ -66,7 +66,7 @@ public class ModelsEvaluationHandlerTest { @Test public void testXgBoostEvaluationWithoutBindings() { String url = "http://localhost/model-evaluation/v1/xgboost_2_2/eval"; // only has a single function - String expected = "{\"cells\":[{\"address\":{},\"value\":-4.376589999999999}]}"; + String expected = "{\"type\":\"tensor()\",\"values\":[-4.376589999999999]}"; handler.assertResponse(url, 200, expected); } @@ -79,7 +79,7 @@ public class ModelsEvaluationHandlerTest { properties.put("f109", "0.4"); properties.put("non-existing-binding", "-1"); String url = "http://localhost/model-evaluation/v1/xgboost_2_2/eval"; - String expected = "{\"cells\":[{\"address\":{},\"value\":-7.936679999999999}]}"; + String expected = "{\"type\":\"tensor()\",\"values\":[-7.936679999999999]}"; handler.assertResponse(url, properties, 200, expected); } @@ -91,6 +91,7 @@ public class ModelsEvaluationHandlerTest { properties.put("f60", "0.3"); properties.put("f109", "0.4"); properties.put("non-existing-binding", "-1"); + properties.put("format.tensors", "long"); String url = "http://localhost/model-evaluation/v1/xgboost_2_2/eval"; String expected = "{\"cells\":[{\"address\":{},\"value\":-7.936679999999999}]}"; handler.assertResponse(url, properties, 200, expected); @@ -99,7 +100,7 @@ public class ModelsEvaluationHandlerTest { @Test public void testLightGBMEvaluationWithoutBindings() { String url = "http://localhost/model-evaluation/v1/lightgbm_regression/eval"; - String expected = "{\"cells\":[{\"address\":{},\"value\":1.9130086820218188}]}"; + String expected = "{\"type\":\"tensor()\",\"values\":[1.9130086820218188]}"; handler.assertResponse(url, 200, expected); } @@ -112,7 +113,7 @@ public class ModelsEvaluationHandlerTest { properties.put("categorical_2", "i"); properties.put("non-existing-binding", "-1"); String url = "http://localhost/model-evaluation/v1/lightgbm_regression/eval"; - String expected = "{\"cells\":[{\"address\":{},\"value\":2.054697758469921}]}"; + String expected = "{\"type\":\"tensor()\",\"values\":[2.054697758469921]}"; handler.assertResponse(url, properties, 200, expected); } @@ -125,7 +126,7 @@ public class ModelsEvaluationHandlerTest { properties.put("categorical_2", "j"); properties.put("non-existing-binding", "-1"); String url = "http://localhost/model-evaluation/v1/lightgbm_regression/eval"; - String expected = "{\"cells\":[{\"address\":{},\"value\":2.0745534018208094}]}"; + String expected = "{\"type\":\"tensor()\",\"values\":[2.0745534018208094]}"; handler.assertResponse(url, properties, 200, expected); } @@ -162,7 +163,7 @@ public class ModelsEvaluationHandlerTest { Map<String, String> properties = new HashMap<>(); properties.put("Placeholder", inputTensor()); String url = "http://localhost/model-evaluation/v1/mnist_softmax/eval"; - String expected = "{\"cells\":[{\"address\":{\"d0\":\"0\",\"d1\":\"0\"},\"value\":-0.3546536862850189},{\"address\":{\"d0\":\"0\",\"d1\":\"1\"},\"value\":0.3759574592113495},{\"address\":{\"d0\":\"0\",\"d1\":\"2\"},\"value\":0.06054411828517914},{\"address\":{\"d0\":\"0\",\"d1\":\"3\"},\"value\":-0.251544713973999},{\"address\":{\"d0\":\"0\",\"d1\":\"4\"},\"value\":0.017951013520359993},{\"address\":{\"d0\":\"0\",\"d1\":\"5\"},\"value\":1.2899067401885986},{\"address\":{\"d0\":\"0\",\"d1\":\"6\"},\"value\":-0.10389615595340729},{\"address\":{\"d0\":\"0\",\"d1\":\"7\"},\"value\":0.6367976665496826},{\"address\":{\"d0\":\"0\",\"d1\":\"8\"},\"value\":-1.4136744737625122},{\"address\":{\"d0\":\"0\",\"d1\":\"9\"},\"value\":-0.2573896050453186}]}"; + String expected = "{\"type\":\"tensor(d0[],d1[10])\",\"values\":[[-0.3546536862850189,0.3759574592113495,0.06054411828517914,-0.251544713973999,0.017951013520359993,1.2899067401885986,-0.10389615595340729,0.6367976665496826,-1.4136744737625122,-0.2573896050453186]]}"; handler.assertResponse(url, properties, 200, expected); } @@ -171,7 +172,7 @@ public class ModelsEvaluationHandlerTest { Map<String, String> properties = new HashMap<>(); properties.put("Placeholder", inputTensor()); String url = "http://localhost/model-evaluation/v1/mnist_softmax/default.add/eval"; - String expected = "{\"cells\":[{\"address\":{\"d0\":\"0\",\"d1\":\"0\"},\"value\":-0.3546536862850189},{\"address\":{\"d0\":\"0\",\"d1\":\"1\"},\"value\":0.3759574592113495},{\"address\":{\"d0\":\"0\",\"d1\":\"2\"},\"value\":0.06054411828517914},{\"address\":{\"d0\":\"0\",\"d1\":\"3\"},\"value\":-0.251544713973999},{\"address\":{\"d0\":\"0\",\"d1\":\"4\"},\"value\":0.017951013520359993},{\"address\":{\"d0\":\"0\",\"d1\":\"5\"},\"value\":1.2899067401885986},{\"address\":{\"d0\":\"0\",\"d1\":\"6\"},\"value\":-0.10389615595340729},{\"address\":{\"d0\":\"0\",\"d1\":\"7\"},\"value\":0.6367976665496826},{\"address\":{\"d0\":\"0\",\"d1\":\"8\"},\"value\":-1.4136744737625122},{\"address\":{\"d0\":\"0\",\"d1\":\"9\"},\"value\":-0.2573896050453186}]}"; + String expected = "{\"type\":\"tensor(d0[],d1[10])\",\"values\":[[-0.3546536862850189,0.3759574592113495,0.06054411828517914,-0.251544713973999,0.017951013520359993,1.2899067401885986,-0.10389615595340729,0.6367976665496826,-1.4136744737625122,-0.2573896050453186]]}"; handler.assertResponse(url, properties, 200, expected); } @@ -179,6 +180,7 @@ public class ModelsEvaluationHandlerTest { public void testMnistSoftmaxEvaluateSpecificFunctionWithBindingsShortForm() { Map<String, String> properties = new HashMap<>(); properties.put("Placeholder", inputTensorShortForm()); + properties.put("format.tensors", "long"); String url = "http://localhost/model-evaluation/v1/mnist_softmax/default.add/eval"; String expected = "{\"cells\":[{\"address\":{\"d0\":\"0\",\"d1\":\"0\"},\"value\":-0.3546536862850189},{\"address\":{\"d0\":\"0\",\"d1\":\"1\"},\"value\":0.3759574592113495},{\"address\":{\"d0\":\"0\",\"d1\":\"2\"},\"value\":0.06054411828517914},{\"address\":{\"d0\":\"0\",\"d1\":\"3\"},\"value\":-0.251544713973999},{\"address\":{\"d0\":\"0\",\"d1\":\"4\"},\"value\":0.017951013520359993},{\"address\":{\"d0\":\"0\",\"d1\":\"5\"},\"value\":1.2899067401885986},{\"address\":{\"d0\":\"0\",\"d1\":\"6\"},\"value\":-0.10389615595340729},{\"address\":{\"d0\":\"0\",\"d1\":\"7\"},\"value\":0.6367976665496826},{\"address\":{\"d0\":\"0\",\"d1\":\"8\"},\"value\":-1.4136744737625122},{\"address\":{\"d0\":\"0\",\"d1\":\"9\"},\"value\":-0.2573896050453186}]}"; handler.assertResponse(url, properties, 200, expected); diff --git a/model-integration/pom.xml b/model-integration/pom.xml index d064a3ff709..7d3ab3f7a5f 100644 --- a/model-integration/pom.xml +++ b/model-integration/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>model-integration</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/ImportedMlModels.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/ImportedMlModels.java index 1003ebd8f3f..05ff1aba877 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/ImportedMlModels.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/ImportedMlModels.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.rankingexpression.importer.configmodelview; -import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.path.Path; import com.yahoo.yolean.Exceptions; @@ -37,12 +36,6 @@ public class ImportedMlModels { importedModels = Collections.emptyMap(); } - /** Will disappear shortly */ - @Deprecated - public ImportedMlModels(File modelsDirectory, Collection<MlModelImporter> importers) { - this(modelsDirectory, new InThreadExecutorService(), importers); - } - public ImportedMlModels(File modelsDirectory, ExecutorService executor, Collection<MlModelImporter> importers) { Map<String, Future<ImportedMlModel>> futureModels = new HashMap<>(); diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/OnnxImporter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/OnnxImporter.java index 144e3e6c03b..e8fc75824be 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/OnnxImporter.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/OnnxImporter.java @@ -37,7 +37,7 @@ public class OnnxImporter extends ModelImporter { for (int i = 0; i < model.getGraph().getOutputCount(); ++i) { Onnx.ValueInfoProto output = model.getGraph().getOutput(i); String outputName = asValidIdentifier(output.getName()); - importedModel.expression(outputName, "onnxModel(" + modelName + ")." + outputName); + importedModel.expression(outputName, "onnx(" + modelName + ")." + outputName); } return importedModel; diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/TensorFlowImporter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/TensorFlowImporter.java index 5316416c4dc..bd9410eecee 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/TensorFlowImporter.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/TensorFlowImporter.java @@ -51,47 +51,7 @@ public class TensorFlowImporter extends ModelImporter { */ @Override public ImportedModel importModel(String modelName, String modelDir) { - return convertToOnnxAndImport(modelName, modelDir); - } - - private ImportedModel convertToOnnxAndImport(String modelName, String modelDir) { - Path tempDir = null; - try { - tempDir = Files.createTempDirectory("tf2onnx"); - String convertedPath = tempDir.toString() + File.separatorChar + "converted.onnx"; - String outputOfLastConversionAttempt = ""; - for (int opset : onnxOpsetsToTry) { - log.info("Converting TensorFlow model '" + modelDir + "' to ONNX with opset " + opset + "..."); - Pair<Integer, String> res = convertToOnnx(modelDir, convertedPath, opset); - if (res.getFirst() == 0) { - log.info("Conversion to ONNX with opset " + opset + " successful."); - - /* - * For now we have to import tensorflow models as native Vespa expressions. - * The temporary ONNX file that is created by conversion needs to be put - * in the application package so it can be file distributed. - */ - return onnxImporter.importModelAsNative(modelName, convertedPath, ImportedMlModel.ModelType.TENSORFLOW); - } - log.fine("Conversion to ONNX with opset " + opset + " failed. Reason: " + res.getSecond()); - outputOfLastConversionAttempt = res.getSecond(); - } - throw new IllegalArgumentException("Unable to convert TensorFlow model in '" + modelDir + "' to ONNX: " + - outputOfLastConversionAttempt); - } catch (IOException e) { - throw new IllegalArgumentException("Conversion from TensorFlow to ONNX failed for '" + modelDir + "'"); - } finally { - if (tempDir != null) { - IOUtils.recursiveDeleteDir(tempDir.toFile()); - } - } - } - - private Pair<Integer, String> convertToOnnx(String savedModel, String output, int opset) throws IOException { - ProcessExecuter executer = new ProcessExecuter(); - String job = "vespa-convert-tf2onnx --saved-model " + savedModel + " --output " + output + " --opset " + opset - + " --use-graph-names"; // for backward compatibility with tf2onnx versions < 1.9.1 - return executer.exec(job); + throw new IllegalArgumentException("Import of TensorFlow models is no longer supported"); } } diff --git a/model-integration/src/main/javacc/ModelParser.jj b/model-integration/src/main/javacc/ModelParser.jj index c850d223612..323522abce8 100644 --- a/model-integration/src/main/javacc/ModelParser.jj +++ b/model-integration/src/main/javacc/ModelParser.jj @@ -248,7 +248,7 @@ void constant() : } } ) - | // Deprecated forms (TODO: Add warning on Vespa 8): + | // Deprecated forms (TODO: Vespa > 8: Add warning ): ( constantValue(name) | constantTensor(name) ) ) ) diff --git a/node-admin/pom.xml b/node-admin/pom.xml index ff1e159de0d..ae7e6ac2bce 100644 --- a/node-admin/pom.xml +++ b/node-admin/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>node-admin</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java index cea8a18b3b4..77c8b80ca91 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java @@ -19,20 +19,16 @@ public class ConfigServerInfo { private final Function<String, URI> configServerHostnameToUriMapper; private final List<URI> configServerURIs; - public ConfigServerInfo(String loadBalancerHostName, List<String> configServerHostNames, - String scheme, int port, AthenzIdentity configServerAthenzIdentity) { - this.loadBalancerEndpoint = createLoadBalancerEndpoint(loadBalancerHostName, scheme, port); + public ConfigServerInfo(URI loadBalancerEndpoint, List<String> configServerHostNames, + AthenzIdentity configServerAthenzIdentity) { + this.loadBalancerEndpoint = loadBalancerEndpoint; this.configServerIdentity = configServerAthenzIdentity; - this.configServerHostnameToUriMapper = hostname -> URI.create(scheme + "://" + hostname + ":" + port); + this.configServerHostnameToUriMapper = hostname -> URI.create("https://" + hostname + ":4443"); this.configServerURIs = configServerHostNames.stream() .map(configServerHostnameToUriMapper) .collect(Collectors.toUnmodifiableList()); } - private static URI createLoadBalancerEndpoint(String loadBalancerHost, String scheme, int port) { - return URI.create(scheme + "://" + loadBalancerHost + ":" + port); - } - public List<URI> getConfigServerUris() { return configServerURIs; } diff --git a/node-repository/pom.xml b/node-repository/pom.xml index 60aa0a83107..52be67024d0 100644 --- a/node-repository/pom.xml +++ b/node-repository/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>node-repository</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <description>Keeps track of node assignment in a multi-application setup.</description> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java index 6ea5df5e3bf..d100e3bdd95 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java @@ -197,9 +197,9 @@ public class MetricsResponse { @Override public List<String> metricResponseNames() { return List.of("feed.http-requests.rate", - "vds.filestor.alldisks.allthreads.put.sum.count.rate", - "vds.filestor.alldisks.allthreads.remove.sum.count.rate", - "vds.filestor.alldisks.allthreads.update.sum.count.rate"); } + "vds.filestor.allthreads.put.count.rate", + "vds.filestor.allthreads.remove.count.rate", + "vds.filestor.allthreads.update.count.rate"); } }; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java index 0e732984227..3dd76c76cac 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/RealDataScenarioTest.java @@ -129,7 +129,7 @@ public class RealDataScenarioTest { private static FlavorsConfig parseFlavors(Path path) { try { var element = XmlHelper.getDocumentBuilder().parse(path.toFile()).getDocumentElement(); - return ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null, null).build(element)).toInstance(FlavorsConfig.class, ""); + return ConfigPayload.fromBuilder(new DomConfigPayloadBuilder(null).build(element)).toInstance(FlavorsConfig.class, ""); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/orchestrator-restapi/pom.xml b/orchestrator-restapi/pom.xml index da40a1a50a1..156e9367760 100644 --- a/orchestrator-restapi/pom.xml +++ b/orchestrator-restapi/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>orchestrator-restapi</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/orchestrator/pom.xml b/orchestrator/pom.xml index 12d87f6bc07..3f8d2bb83db 100644 --- a/orchestrator/pom.xml +++ b/orchestrator/pom.xml @@ -8,33 +8,15 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>orchestrator</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> <dependencies> <dependency> - <!-- Provided by the configserver bundle --> - <groupId>com.fasterxml.jackson.jaxrs</groupId> - <artifactId>jackson-jaxrs-json-provider</artifactId> - <scope>provided</scope> - <exclusions> - <exclusion> - <!-- Conflicts with javax.activation:javax.activation-api:1.2.0, which is "exported" via jdisc_core. --> - <groupId>jakarta.activation</groupId> - <artifactId>jakarta.activation-api</artifactId> - </exclusion> - <exclusion> - <!-- Conflicts with javax.xml.bind:jaxb-api:2.3, which is "exported" via jdisc_core.--> - <groupId>jakarta.xml.bind</groupId> - <artifactId>jakarta.xml.bind-api</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>container-dev</artifactId> <version>${project.version}</version> diff --git a/parent/pom.xml b/parent/pom.xml index b24390ae474..58daca2fe57 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -4,7 +4,7 @@ <modelVersion>4.0.0</modelVersion> <artifactId>parent</artifactId> <packaging>pom</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>parent</name> <description>Parent artifact for all Vespa maven projects.</description> <url>https://github.com/vespa-engine</url> @@ -12,7 +12,7 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>container-dependency-versions</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../container-dependency-versions/pom.xml</relativePath> </parent> @@ -75,7 +75,7 @@ <artifactId>maven-compiler-plugin</artifactId> <version>${maven-compiler-plugin.version}</version> <configuration> - <release>11</release> + <release>17</release> <showWarnings>true</showWarnings> <optimize>true</optimize> <showDeprecation>false</showDeprecation> @@ -474,6 +474,38 @@ <version>2.6.0</version> </dependency> <dependency> + <groupId>com.google.code.findbugs</groupId> + <artifactId>jsr305</artifactId> + <version>${findbugs.version}</version> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <version>${guava.version}</version> + <exclusions> + <exclusion> + <!-- Annotations with retention Runtime. Remove exclusion and provide from jdisc_core if needed. --> + <groupId>com.google.code.findbugs</groupId> + <artifactId>jsr305</artifactId> + </exclusion> + <exclusion> + <!-- Empty artifact for gradle projects, see the artifact's pom.xml --> + <groupId>com.google.guava</groupId> + <artifactId>listenablefuture</artifactId> + </exclusion> + <exclusion> + <!-- Annotations with retention Runtime. Remove exclusion if needed. --> + <groupId>org.checkerframework</groupId> + <artifactId>checker-qual</artifactId> + </exclusion> + <exclusion> + <!-- Annotation with retention class only --> + <groupId>org.codehaus.mojo</groupId> + <artifactId>animal-sniffer-annotations</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> <groupId>com.google.guava</groupId> <artifactId>guava-testlib</artifactId> <version>${guava.version}</version> @@ -635,6 +667,22 @@ <version>1.8</version> </dependency> <dependency> + <groupId>org.apache.felix</groupId> + <artifactId>org.apache.felix.framework</artifactId> + <version>${felix.version}</version> + <exclusions> + <exclusion> + <groupId>org.codehaus.mojo</groupId> + <artifactId>animal-sniffer-annotations</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.felix</groupId> + <artifactId>org.apache.felix.log</artifactId> + <version>${felix.log.version}</version> + </dependency> + <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpcore</artifactId> <version>${apache.httpcore.version}</version> @@ -726,6 +774,16 @@ <version>3.11.1</version> </dependency> <dependency> + <groupId>org.bouncycastle</groupId> + <artifactId>bcpkix-jdk15on</artifactId> + <version>${bouncycastle.version}</version> + </dependency> + <dependency> + <groupId>org.bouncycastle</groupId> + <artifactId>bcprov-jdk15on</artifactId> + <version>${bouncycastle.version}</version> + </dependency> + <dependency> <groupId>org.codehaus.jettison</groupId> <artifactId>jettison</artifactId> <version>1.3.1</version> @@ -797,26 +855,6 @@ <version>2.3.2</version> <!-- 2.3.3 has a BROKEN manifest --> </dependency> <dependency> - <groupId>org.glassfish.jersey.ext</groupId> - <artifactId>jersey-entity-filtering</artifactId> - <version>${jersey2.version}</version> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.ext</groupId> - <artifactId>jersey-proxy-client</artifactId> - <version>${jersey2.version}</version> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.media</groupId> - <artifactId>jersey-media-json-jackson</artifactId> - <version>${jersey2.version}</version> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.media</groupId> - <artifactId>jersey-media-multipart</artifactId> - <version>${jersey2.version}</version> - </dependency> - <dependency> <groupId>org.hamcrest</groupId> <artifactId>hamcrest-all</artifactId> <version>1.3</version> @@ -865,6 +903,11 @@ <version>1.9.6</version> </dependency> <dependency> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> + <version>${org.lz4.version}</version> + </dependency> + <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <version>${mockito.version}</version> @@ -910,7 +953,6 @@ <artifactId>metrics-core</artifactId> <version>3.2.5</version> </dependency> - <dependency> <groupId>uk.co.datumedge</groupId> <artifactId>hamcrest-json</artifactId> @@ -922,6 +964,11 @@ <artifactId>xercesImpl</artifactId> <version>2.12.2</version> </dependency> + <dependency> + <groupId>org.json</groupId> + <artifactId>json</artifactId> + <version>${org.json.version}</version> + </dependency> </dependencies> </dependencyManagement> @@ -947,12 +994,18 @@ find zkfacade/src/main/java/org/apache/curator -name package-info.java | \ xargs perl -pi -e 's/major = [0-9]+, minor = [0-9]+, micro = [0-9]+/major = 5, minor = 3, micro = 0/g' --> + <bouncycastle.version>1.68</bouncycastle.version> <curator.version>5.2.1</curator.version> <commons.codec.version>1.15</commons.codec.version> <commons.math3.version>3.6.1</commons.math3.version> <eclipse-collections.version>11.0.0</eclipse-collections.version> + <felix.version>7.0.1</felix.version> + <felix.log.version>1.0.1</felix.log.version> + <findbugs.version>3.0.2</findbugs.version> <!-- Should be kept in sync with guava --> <gson.version>2.8.9</gson.version> <hdrhistogram.version>2.1.12</hdrhistogram.version> + <jetty.version>9.4.46.v20220331</jetty.version> + <jetty-alpn.version>1.1.3.v20160715</jetty-alpn.version> <jna.version>5.11.0</jna.version> <junit.version>5.8.1</junit.version> <maven-archiver.version>3.5.2</maven-archiver.version> @@ -975,9 +1028,10 @@ <maven-source-plugin.version>3.2.1</maven-source-plugin.version> <mockito.version>4.0.0</mockito.version> <onnxruntime.version>1.8.0</onnxruntime.version> <!-- WARNING: sync cloud-tenant-base-dependencies-enforcer/pom.xml --> + <org.lz4.version>1.8.0</org.lz4.version> <prometheus.client.version>0.6.0</prometheus.client.version> <protobuf.version>3.19.2</protobuf.version> - <spifly.version>1.3.3</spifly.version> + <spifly.version>1.3.5</spifly.version> <surefire.version>2.22.2</surefire.version> <zookeeper.client.version>3.7.0</zookeeper.client.version> @@ -5,7 +5,7 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>vespa</artifactId> <packaging>pom</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <description>Aggregator pom for vespa.</description> <url>https://github.com/vespa-engine</url> @@ -64,6 +64,7 @@ <module>container-search-and-docproc</module> <module>container-search</module> <module>container-search-gui</module> + <module>container-spifly</module> <module>container-test</module> <module>controller-api</module> <module>controller-server</module> @@ -91,7 +92,6 @@ <module>jdisc-security-filters</module> <module>jdisc_core</module> <module>jdisc_core_test</module> - <module>jdisc_jetty</module> <module>jrt</module> <module>linguistics</module> <module>linguistics-components</module> @@ -119,7 +119,6 @@ <module>service-monitor</module> <module>socket_test</module> <module>standalone-container</module> - <module>statistics</module> <module>storage</module> <module>streamingvisitors</module> <module>tenant-base</module> @@ -136,12 +135,11 @@ <module>vespa-feed-client-api</module> <module>vespa-feed-client-cli</module> <module>vespa-hadoop</module> - <module>vespa-http-client</module> <module>vespa-maven-plugin</module> <module>vespa-osgi-testrunner</module> <module>vespa-testrunner-components</module> <module>vespa_feed_perf</module> - <module>vespa_jersey2</module> + <module>vespa-3party-bundles</module> <module>vespajlib</module> <module>vespalog</module> <module>yolean</module> diff --git a/predicate-search-core/pom.xml b/predicate-search-core/pom.xml index 733501122bf..4951b3d40f1 100644 --- a/predicate-search-core/pom.xml +++ b/predicate-search-core/pom.xml @@ -6,11 +6,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>predicate-search-core</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/predicate-search/pom.xml b/predicate-search/pom.xml index 83efe9d9754..75c9350707c 100644 --- a/predicate-search/pom.xml +++ b/predicate-search/pom.xml @@ -6,11 +6,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>predicate-search</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/protocols/getnodestate/slow_host_info.json b/protocols/getnodestate/slow_host_info.json index b792e45d5d6..6e3afe9f8c7 100644 --- a/protocols/getnodestate/slow_host_info.json +++ b/protocols/getnodestate/slow_host_info.json @@ -91,7 +91,7 @@ } }, { - "name":"vds.filestor.alldisks.allthreads.operations", + "name":"vds.filestor.allthreads.operations", "description":"Number of operations processed.", "values": { @@ -103,7 +103,7 @@ } }, { - "name":"vds.filestor.alldisks.allthreads.put.sum.count", + "name":"vds.filestor.allthreads.put.count", "description":"Number of requests processed.", "values": { @@ -115,7 +115,7 @@ } }, { - "name":"vds.filestor.alldisks.allthreads.put.sum.latency", + "name":"vds.filestor.allthreads.put.latency", "description":"Latency of successful requests.", "values": { @@ -131,7 +131,7 @@ } }, { - "name":"vds.filestor.alldisks.allthreads.put.sum.failed", + "name":"vds.filestor.allthreads.put.failed", "description":"Number of failed requests.", "values": { @@ -143,7 +143,7 @@ } }, { - "name":"vds.filestor.alldisks.allthreads.get.sum.count", + "name":"vds.filestor.allthreads.get.count", "description":"Number of requests processed.", "values": { @@ -155,7 +155,7 @@ } }, { - "name":"vds.filestor.alldisks.allthreads.get.sum.latency", + "name":"vds.filestor.allthreads.get.latency", "description":"Latency of successful requests.", "values": { @@ -171,7 +171,7 @@ } }, { - "name":"vds.filestor.alldisks.allthreads.get.sum.failed", + "name":"vds.filestor.allthreads.get.failed", "description":"Number of failed requests.", "values": { @@ -183,7 +183,7 @@ } }, { - "name":"vds.filestor.alldisks.allthreads.get.sum.not_found", + "name":"vds.filestor.allthreads.get.not_found", "description":"Number of requests that could not be completed due to source document not found.", "values": { @@ -195,7 +195,7 @@ } }, { - "name":"vds.filestor.alldisks.allthreads.remove.sum.count", + "name":"vds.filestor.allthreads.remove.count", "description":"Number of requests processed.", "values": { @@ -207,7 +207,7 @@ } }, { - "name":"vds.filestor.alldisks.allthreads.remove.sum.latency", + "name":"vds.filestor.allthreads.remove.latency", "description":"Latency of successful requests.", "values": { @@ -223,7 +223,7 @@ } }, { - "name":"vds.filestor.alldisks.allthreads.remove.sum.failed", + "name":"vds.filestor.allthreads.remove.failed", "description":"Number of failed requests.", "values": { @@ -235,7 +235,7 @@ } }, { - "name":"vds.filestor.alldisks.allthreads.remove.sum.not_found", + "name":"vds.filestor.allthreads.remove.not_found", "description":"Number of requests that could not be completed due to source document not found.", "values": { @@ -247,7 +247,7 @@ } }, { - "name":"vds.filestor.alldisks.allthreads.update.sum.count", + "name":"vds.filestor.allthreads.update.count", "description":"Number of requests processed.", "values": { @@ -259,7 +259,7 @@ } }, { - "name":"vds.filestor.alldisks.allthreads.update.sum.latency", + "name":"vds.filestor.allthreads.update.latency", "description":"Latency of successful requests.", "values": { @@ -275,7 +275,7 @@ } }, { - "name":"vds.filestor.alldisks.allthreads.update.sum.failed", + "name":"vds.filestor.allthreads.update.failed", "description":"Number of failed requests.", "values": { @@ -287,7 +287,7 @@ } }, { - "name":"vds.filestor.alldisks.allthreads.update.sum.not_found", + "name":"vds.filestor.allthreads.update.not_found", "description":"Number of requests that could not be completed due to source document not found.", "values": { @@ -299,7 +299,7 @@ } }, { - "name":"vds.filestor.alldisks.allthreads.update.sum.latency_read", + "name":"vds.filestor.allthreads.update.latency_read", "description":"Latency of the source read in the request.", "values": { @@ -313,2657 +313,6 @@ "dimensions": { } - }, - { - "name":"vds.filestor.disk_0.allthreads.operations", - "description":"Number of operations processed.", - "values": - { - "count":845, - "rate":2.816666 - }, - "dimensions": - { - "disk":"disk_0" - } - }, - { - "name":"vds.filestor.disk_0.allthreads.put.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":307, - "rate":1.023333 - }, - "dimensions": - { - "disk":"disk_0" - } - }, - { - "name":"vds.filestor.disk_0.allthreads.put.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":4.485342019543974, - "count":307, - "rate":1.023333, - "min":0, - "max":128, - "last":8 - }, - "dimensions": - { - "disk":"disk_0" - } - }, - { - "name":"vds.filestor.disk_0.allthreads.put.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_0" - } - }, - { - "name":"vds.filestor.disk_0.allthreads.get.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_0" - } - }, - { - "name":"vds.filestor.disk_0.allthreads.get.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":0.0, - "count":0, - "rate":0.0, - "min":0, - "max":0, - "last":0 - }, - "dimensions": - { - "disk":"disk_0" - } - }, - { - "name":"vds.filestor.disk_0.allthreads.get.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_0" - } - }, - { - "name":"vds.filestor.disk_0.allthreads.get.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_0" - } - }, - { - "name":"vds.filestor.disk_0.allthreads.remove.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":330, - "rate":1.1 - }, - "dimensions": - { - "disk":"disk_0" - } - }, - { - "name":"vds.filestor.disk_0.allthreads.remove.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":10.17878787878788, - "count":330, - "rate":1.1, - "min":0, - "max":71, - "last":3 - }, - "dimensions": - { - "disk":"disk_0" - } - }, - { - "name":"vds.filestor.disk_0.allthreads.remove.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_0" - } - }, - { - "name":"vds.filestor.disk_0.allthreads.remove.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":150, - "rate":0.5 - }, - "dimensions": - { - "disk":"disk_0" - } - }, - { - "name":"vds.filestor.disk_0.allthreads.update.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":208, - "rate":0.693333 - }, - "dimensions": - { - "disk":"disk_0" - } - }, - { - "name":"vds.filestor.disk_0.allthreads.update.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":1.543269230769231, - "count":208, - "rate":0.693333, - "min":0, - "max":13, - "last":5 - }, - "dimensions": - { - "disk":"disk_0" - } - }, - { - "name":"vds.filestor.disk_0.allthreads.update.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_0" - } - }, - { - "name":"vds.filestor.disk_0.allthreads.update.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_0" - } - }, - { - "name":"vds.filestor.disk_0.allthreads.update.sum.latency_read", - "description":"Latency of the source read in the request.", - "values": - { - "average":0.0, - "count":0, - "rate":0.0, - "min":0, - "max":0, - "last":0 - }, - "dimensions": - { - "disk":"disk_0" - } - }, - { - "name":"vds.filestor.disk_1.allthreads.operations", - "description":"Number of operations processed.", - "values": - { - "count":1257, - "rate":4.19 - }, - "dimensions": - { - "disk":"disk_1" - } - }, - { - "name":"vds.filestor.disk_1.allthreads.put.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":404, - "rate":1.346666 - }, - "dimensions": - { - "disk":"disk_1" - } - }, - { - "name":"vds.filestor.disk_1.allthreads.put.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":7.344059405940594, - "count":404, - "rate":1.346666, - "min":0, - "max":35, - "last":8 - }, - "dimensions": - { - "disk":"disk_1" - } - }, - { - "name":"vds.filestor.disk_1.allthreads.put.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_1" - } - }, - { - "name":"vds.filestor.disk_1.allthreads.get.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_1" - } - }, - { - "name":"vds.filestor.disk_1.allthreads.get.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":0.0, - "count":0, - "rate":0.0, - "min":0, - "max":0, - "last":0 - }, - "dimensions": - { - "disk":"disk_1" - } - }, - { - "name":"vds.filestor.disk_1.allthreads.get.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_1" - } - }, - { - "name":"vds.filestor.disk_1.allthreads.get.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_1" - } - }, - { - "name":"vds.filestor.disk_1.allthreads.remove.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":435, - "rate":1.45 - }, - "dimensions": - { - "disk":"disk_1" - } - }, - { - "name":"vds.filestor.disk_1.allthreads.remove.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":7.289655172413793, - "count":435, - "rate":1.45, - "min":0, - "max":72, - "last":7 - }, - "dimensions": - { - "disk":"disk_1" - } - }, - { - "name":"vds.filestor.disk_1.allthreads.remove.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_1" - } - }, - { - "name":"vds.filestor.disk_1.allthreads.remove.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":307, - "rate":1.023333 - }, - "dimensions": - { - "disk":"disk_1" - } - }, - { - "name":"vds.filestor.disk_1.allthreads.update.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":418, - "rate":1.393333 - }, - "dimensions": - { - "disk":"disk_1" - } - }, - { - "name":"vds.filestor.disk_1.allthreads.update.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":13.78947368421053, - "count":418, - "rate":1.393333, - "min":0, - "max":78, - "last":17 - }, - "dimensions": - { - "disk":"disk_1" - } - }, - { - "name":"vds.filestor.disk_1.allthreads.update.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_1" - } - }, - { - "name":"vds.filestor.disk_1.allthreads.update.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_1" - } - }, - { - "name":"vds.filestor.disk_1.allthreads.update.sum.latency_read", - "description":"Latency of the source read in the request.", - "values": - { - "average":0.0, - "count":0, - "rate":0.0, - "min":0, - "max":0, - "last":0 - }, - "dimensions": - { - "disk":"disk_1" - } - }, - { - "name":"vds.filestor.disk_2.allthreads.operations", - "description":"Number of operations processed.", - "values": - { - "count":769, - "rate":2.563333 - }, - "dimensions": - { - "disk":"disk_2" - } - }, - { - "name":"vds.filestor.disk_2.allthreads.put.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":350, - "rate":1.166666 - }, - "dimensions": - { - "disk":"disk_2" - } - }, - { - "name":"vds.filestor.disk_2.allthreads.put.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":4.708571428571428, - "count":350, - "rate":1.166666, - "min":0, - "max":91, - "last":7 - }, - "dimensions": - { - "disk":"disk_2" - } - }, - { - "name":"vds.filestor.disk_2.allthreads.put.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_2" - } - }, - { - "name":"vds.filestor.disk_2.allthreads.get.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_2" - } - }, - { - "name":"vds.filestor.disk_2.allthreads.get.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":0.0, - "count":0, - "rate":0.0, - "min":0, - "max":0, - "last":0 - }, - "dimensions": - { - "disk":"disk_2" - } - }, - { - "name":"vds.filestor.disk_2.allthreads.get.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_2" - } - }, - { - "name":"vds.filestor.disk_2.allthreads.get.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_2" - } - }, - { - "name":"vds.filestor.disk_2.allthreads.remove.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":112, - "rate":0.373333 - }, - "dimensions": - { - "disk":"disk_2" - } - }, - { - "name":"vds.filestor.disk_2.allthreads.remove.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":3.053571428571428, - "count":112, - "rate":0.373333, - "min":0, - "max":14, - "last":1 - }, - "dimensions": - { - "disk":"disk_2" - } - }, - { - "name":"vds.filestor.disk_2.allthreads.remove.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_2" - } - }, - { - "name":"vds.filestor.disk_2.allthreads.remove.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":32, - "rate":0.106666 - }, - "dimensions": - { - "disk":"disk_2" - } - }, - { - "name":"vds.filestor.disk_2.allthreads.update.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":307, - "rate":1.023333 - }, - "dimensions": - { - "disk":"disk_2" - } - }, - { - "name":"vds.filestor.disk_2.allthreads.update.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":3.739413680781759, - "count":307, - "rate":1.023333, - "min":0, - "max":33, - "last":5 - }, - "dimensions": - { - "disk":"disk_2" - } - }, - { - "name":"vds.filestor.disk_2.allthreads.update.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_2" - } - }, - { - "name":"vds.filestor.disk_2.allthreads.update.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_2" - } - }, - { - "name":"vds.filestor.disk_2.allthreads.update.sum.latency_read", - "description":"Latency of the source read in the request.", - "values": - { - "average":0.0, - "count":0, - "rate":0.0, - "min":0, - "max":0, - "last":0 - }, - "dimensions": - { - "disk":"disk_2" - } - }, - { - "name":"vds.filestor.disk_3.allthreads.operations", - "description":"Number of operations processed.", - "values": - { - "count":744, - "rate":2.48 - }, - "dimensions": - { - "disk":"disk_3" - } - }, - { - "name":"vds.filestor.disk_3.allthreads.put.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":286, - "rate":0.953333 - }, - "dimensions": - { - "disk":"disk_3" - } - }, - { - "name":"vds.filestor.disk_3.allthreads.put.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":4.377622377622377, - "count":286, - "rate":0.953333, - "min":0, - "max":10, - "last":5 - }, - "dimensions": - { - "disk":"disk_3" - } - }, - { - "name":"vds.filestor.disk_3.allthreads.put.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_3" - } - }, - { - "name":"vds.filestor.disk_3.allthreads.get.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_3" - } - }, - { - "name":"vds.filestor.disk_3.allthreads.get.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":0.0, - "count":0, - "rate":0.0, - "min":0, - "max":0, - "last":0 - }, - "dimensions": - { - "disk":"disk_3" - } - }, - { - "name":"vds.filestor.disk_3.allthreads.get.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_3" - } - }, - { - "name":"vds.filestor.disk_3.allthreads.get.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_3" - } - }, - { - "name":"vds.filestor.disk_3.allthreads.remove.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":275, - "rate":0.916666 - }, - "dimensions": - { - "disk":"disk_3" - } - }, - { - "name":"vds.filestor.disk_3.allthreads.remove.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":10.04, - "count":275, - "rate":0.916666, - "min":0, - "max":35, - "last":5 - }, - "dimensions": - { - "disk":"disk_3" - } - }, - { - "name":"vds.filestor.disk_3.allthreads.remove.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_3" - } - }, - { - "name":"vds.filestor.disk_3.allthreads.remove.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":168, - "rate":0.5600000000000001 - }, - "dimensions": - { - "disk":"disk_3" - } - }, - { - "name":"vds.filestor.disk_3.allthreads.update.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":183, - "rate":0.61 - }, - "dimensions": - { - "disk":"disk_3" - } - }, - { - "name":"vds.filestor.disk_3.allthreads.update.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":5.754098360655738, - "count":183, - "rate":0.61, - "min":0, - "max":17, - "last":2 - }, - "dimensions": - { - "disk":"disk_3" - } - }, - { - "name":"vds.filestor.disk_3.allthreads.update.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_3" - } - }, - { - "name":"vds.filestor.disk_3.allthreads.update.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_3" - } - }, - { - "name":"vds.filestor.disk_3.allthreads.update.sum.latency_read", - "description":"Latency of the source read in the request.", - "values": - { - "average":0.0, - "count":0, - "rate":0.0, - "min":0, - "max":0, - "last":0 - }, - "dimensions": - { - "disk":"disk_3" - } - }, - { - "name":"vds.filestor.disk_4.allthreads.operations", - "description":"Number of operations processed.", - "values": - { - "count":495, - "rate":1.65 - }, - "dimensions": - { - "disk":"disk_4" - } - }, - { - "name":"vds.filestor.disk_4.allthreads.put.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":258, - "rate":0.86 - }, - "dimensions": - { - "disk":"disk_4" - } - }, - { - "name":"vds.filestor.disk_4.allthreads.put.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":4.682170542635659, - "count":258, - "rate":0.86, - "min":0, - "max":12, - "last":6 - }, - "dimensions": - { - "disk":"disk_4" - } - }, - { - "name":"vds.filestor.disk_4.allthreads.put.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_4" - } - }, - { - "name":"vds.filestor.disk_4.allthreads.get.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_4" - } - }, - { - "name":"vds.filestor.disk_4.allthreads.get.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":0.0, - "count":0, - "rate":0.0, - "min":0, - "max":0, - "last":0 - }, - "dimensions": - { - "disk":"disk_4" - } - }, - { - "name":"vds.filestor.disk_4.allthreads.get.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_4" - } - }, - { - "name":"vds.filestor.disk_4.allthreads.get.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_4" - } - }, - { - "name":"vds.filestor.disk_4.allthreads.remove.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":200, - "rate":0.666666 - }, - "dimensions": - { - "disk":"disk_4" - } - }, - { - "name":"vds.filestor.disk_4.allthreads.remove.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":2.045, - "count":200, - "rate":0.666666, - "min":0, - "max":26, - "last":0 - }, - "dimensions": - { - "disk":"disk_4" - } - }, - { - "name":"vds.filestor.disk_4.allthreads.remove.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_4" - } - }, - { - "name":"vds.filestor.disk_4.allthreads.remove.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":152, - "rate":0.5066659999999999 - }, - "dimensions": - { - "disk":"disk_4" - } - }, - { - "name":"vds.filestor.disk_4.allthreads.update.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":37, - "rate":0.123333 - }, - "dimensions": - { - "disk":"disk_4" - } - }, - { - "name":"vds.filestor.disk_4.allthreads.update.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":4.783783783783784, - "count":37, - "rate":0.123333, - "min":0, - "max":28, - "last":10 - }, - "dimensions": - { - "disk":"disk_4" - } - }, - { - "name":"vds.filestor.disk_4.allthreads.update.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_4" - } - }, - { - "name":"vds.filestor.disk_4.allthreads.update.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_4" - } - }, - { - "name":"vds.filestor.disk_4.allthreads.update.sum.latency_read", - "description":"Latency of the source read in the request.", - "values": - { - "average":0.0, - "count":0, - "rate":0.0, - "min":0, - "max":0, - "last":0 - }, - "dimensions": - { - "disk":"disk_4" - } - }, - { - "name":"vds.filestor.disk_5.allthreads.operations", - "description":"Number of operations processed.", - "values": - { - "count":384, - "rate":1.28 - }, - "dimensions": - { - "disk":"disk_5" - } - }, - { - "name":"vds.filestor.disk_5.allthreads.put.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":202, - "rate":0.673333 - }, - "dimensions": - { - "disk":"disk_5" - } - }, - { - "name":"vds.filestor.disk_5.allthreads.put.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":4.396039603960396, - "count":202, - "rate":0.673333, - "min":0, - "max":9, - "last":8 - }, - "dimensions": - { - "disk":"disk_5" - } - }, - { - "name":"vds.filestor.disk_5.allthreads.put.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_5" - } - }, - { - "name":"vds.filestor.disk_5.allthreads.get.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_5" - } - }, - { - "name":"vds.filestor.disk_5.allthreads.get.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":0.0, - "count":0, - "rate":0.0, - "min":0, - "max":0, - "last":0 - }, - "dimensions": - { - "disk":"disk_5" - } - }, - { - "name":"vds.filestor.disk_5.allthreads.get.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_5" - } - }, - { - "name":"vds.filestor.disk_5.allthreads.get.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_5" - } - }, - { - "name":"vds.filestor.disk_5.allthreads.remove.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":137, - "rate":0.456666 - }, - "dimensions": - { - "disk":"disk_5" - } - }, - { - "name":"vds.filestor.disk_5.allthreads.remove.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":2.445255474452555, - "count":137, - "rate":0.456666, - "min":0, - "max":14, - "last":0 - }, - "dimensions": - { - "disk":"disk_5" - } - }, - { - "name":"vds.filestor.disk_5.allthreads.remove.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_5" - } - }, - { - "name":"vds.filestor.disk_5.allthreads.remove.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":118, - "rate":0.393333 - }, - "dimensions": - { - "disk":"disk_5" - } - }, - { - "name":"vds.filestor.disk_5.allthreads.update.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":45, - "rate":0.15 - }, - "dimensions": - { - "disk":"disk_5" - } - }, - { - "name":"vds.filestor.disk_5.allthreads.update.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":4.288888888888889, - "count":45, - "rate":0.15, - "min":0, - "max":16, - "last":9 - }, - "dimensions": - { - "disk":"disk_5" - } - }, - { - "name":"vds.filestor.disk_5.allthreads.update.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_5" - } - }, - { - "name":"vds.filestor.disk_5.allthreads.update.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_5" - } - }, - { - "name":"vds.filestor.disk_5.allthreads.update.sum.latency_read", - "description":"Latency of the source read in the request.", - "values": - { - "average":0.0, - "count":0, - "rate":0.0, - "min":0, - "max":0, - "last":0 - }, - "dimensions": - { - "disk":"disk_5" - } - }, - { - "name":"vds.filestor.disk_6.allthreads.operations", - "description":"Number of operations processed.", - "values": - { - "count":798, - "rate":2.66 - }, - "dimensions": - { - "disk":"disk_6" - } - }, - { - "name":"vds.filestor.disk_6.allthreads.put.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":263, - "rate":0.8766659999999999 - }, - "dimensions": - { - "disk":"disk_6" - } - }, - { - "name":"vds.filestor.disk_6.allthreads.put.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":5.418250950570342, - "count":263, - "rate":0.8766659999999999, - "min":1, - "max":24, - "last":6 - }, - "dimensions": - { - "disk":"disk_6" - } - }, - { - "name":"vds.filestor.disk_6.allthreads.put.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_6" - } - }, - { - "name":"vds.filestor.disk_6.allthreads.get.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_6" - } - }, - { - "name":"vds.filestor.disk_6.allthreads.get.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":0.0, - "count":0, - "rate":0.0, - "min":0, - "max":0, - "last":0 - }, - "dimensions": - { - "disk":"disk_6" - } - }, - { - "name":"vds.filestor.disk_6.allthreads.get.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_6" - } - }, - { - "name":"vds.filestor.disk_6.allthreads.get.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_6" - } - }, - { - "name":"vds.filestor.disk_6.allthreads.remove.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":346, - "rate":1.153333 - }, - "dimensions": - { - "disk":"disk_6" - } - }, - { - "name":"vds.filestor.disk_6.allthreads.remove.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":12.90751445086705, - "count":346, - "rate":1.153333, - "min":0, - "max":53, - "last":6 - }, - "dimensions": - { - "disk":"disk_6" - } - }, - { - "name":"vds.filestor.disk_6.allthreads.remove.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_6" - } - }, - { - "name":"vds.filestor.disk_6.allthreads.remove.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":230, - "rate":0.766666 - }, - "dimensions": - { - "disk":"disk_6" - } - }, - { - "name":"vds.filestor.disk_6.allthreads.update.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":189, - "rate":0.63 - }, - "dimensions": - { - "disk":"disk_6" - } - }, - { - "name":"vds.filestor.disk_6.allthreads.update.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":3.984126984126984, - "count":189, - "rate":0.63, - "min":0, - "max":24, - "last":0 - }, - "dimensions": - { - "disk":"disk_6" - } - }, - { - "name":"vds.filestor.disk_6.allthreads.update.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_6" - } - }, - { - "name":"vds.filestor.disk_6.allthreads.update.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_6" - } - }, - { - "name":"vds.filestor.disk_6.allthreads.update.sum.latency_read", - "description":"Latency of the source read in the request.", - "values": - { - "average":0.0, - "count":0, - "rate":0.0, - "min":0, - "max":0, - "last":0 - }, - "dimensions": - { - "disk":"disk_6" - } - }, - { - "name":"vds.filestor.disk_7.allthreads.operations", - "description":"Number of operations processed.", - "values": - { - "count":490, - "rate":1.633333 - }, - "dimensions": - { - "disk":"disk_7" - } - }, - { - "name":"vds.filestor.disk_7.allthreads.put.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":279, - "rate":0.93 - }, - "dimensions": - { - "disk":"disk_7" - } - }, - { - "name":"vds.filestor.disk_7.allthreads.put.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":4.802867383512544, - "count":279, - "rate":0.93, - "min":1, - "max":10, - "last":6 - }, - "dimensions": - { - "disk":"disk_7" - } - }, - { - "name":"vds.filestor.disk_7.allthreads.put.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_7" - } - }, - { - "name":"vds.filestor.disk_7.allthreads.get.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_7" - } - }, - { - "name":"vds.filestor.disk_7.allthreads.get.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":0.0, - "count":0, - "rate":0.0, - "min":0, - "max":0, - "last":0 - }, - "dimensions": - { - "disk":"disk_7" - } - }, - { - "name":"vds.filestor.disk_7.allthreads.get.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_7" - } - }, - { - "name":"vds.filestor.disk_7.allthreads.get.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_7" - } - }, - { - "name":"vds.filestor.disk_7.allthreads.remove.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":38, - "rate":0.126666 - }, - "dimensions": - { - "disk":"disk_7" - } - }, - { - "name":"vds.filestor.disk_7.allthreads.remove.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":3.315789473684211, - "count":38, - "rate":0.126666, - "min":0, - "max":13, - "last":7 - }, - "dimensions": - { - "disk":"disk_7" - } - }, - { - "name":"vds.filestor.disk_7.allthreads.remove.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_7" - } - }, - { - "name":"vds.filestor.disk_7.allthreads.remove.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":19, - "rate":0.063333 - }, - "dimensions": - { - "disk":"disk_7" - } - }, - { - "name":"vds.filestor.disk_7.allthreads.update.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":173, - "rate":0.576666 - }, - "dimensions": - { - "disk":"disk_7" - } - }, - { - "name":"vds.filestor.disk_7.allthreads.update.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":3.473988439306358, - "count":173, - "rate":0.576666, - "min":0, - "max":25, - "last":2 - }, - "dimensions": - { - "disk":"disk_7" - } - }, - { - "name":"vds.filestor.disk_7.allthreads.update.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_7" - } - }, - { - "name":"vds.filestor.disk_7.allthreads.update.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_7" - } - }, - { - "name":"vds.filestor.disk_7.allthreads.update.sum.latency_read", - "description":"Latency of the source read in the request.", - "values": - { - "average":0.0, - "count":0, - "rate":0.0, - "min":0, - "max":0, - "last":0 - }, - "dimensions": - { - "disk":"disk_7" - } - }, - { - "name":"vds.filestor.disk_8.allthreads.operations", - "description":"Number of operations processed.", - "values": - { - "count":2115, - "rate":7.05 - }, - "dimensions": - { - "disk":"disk_8" - } - }, - { - "name":"vds.filestor.disk_8.allthreads.put.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":355, - "rate":1.183333 - }, - "dimensions": - { - "disk":"disk_8" - } - }, - { - "name":"vds.filestor.disk_8.allthreads.put.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":5.504225352112676, - "count":355, - "rate":1.183333, - "min":0, - "max":236, - "last":6 - }, - "dimensions": - { - "disk":"disk_8" - } - }, - { - "name":"vds.filestor.disk_8.allthreads.put.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_8" - } - }, - { - "name":"vds.filestor.disk_8.allthreads.get.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_8" - } - }, - { - "name":"vds.filestor.disk_8.allthreads.get.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":0.0, - "count":0, - "rate":0.0, - "min":0, - "max":0, - "last":0 - }, - "dimensions": - { - "disk":"disk_8" - } - }, - { - "name":"vds.filestor.disk_8.allthreads.get.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_8" - } - }, - { - "name":"vds.filestor.disk_8.allthreads.get.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_8" - } - }, - { - "name":"vds.filestor.disk_8.allthreads.remove.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":1635, - "rate":5.45 - }, - "dimensions": - { - "disk":"disk_8" - } - }, - { - "name":"vds.filestor.disk_8.allthreads.remove.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":184.6189602446483, - "count":1635, - "rate":5.45, - "min":0, - "max":913, - "last":0 - }, - "dimensions": - { - "disk":"disk_8" - } - }, - { - "name":"vds.filestor.disk_8.allthreads.remove.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_8" - } - }, - { - "name":"vds.filestor.disk_8.allthreads.remove.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":1511, - "rate":5.036666 - }, - "dimensions": - { - "disk":"disk_8" - } - }, - { - "name":"vds.filestor.disk_8.allthreads.update.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":125, - "rate":0.416666 - }, - "dimensions": - { - "disk":"disk_8" - } - }, - { - "name":"vds.filestor.disk_8.allthreads.update.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":4.336, - "count":125, - "rate":0.416666, - "min":0, - "max":115, - "last":0 - }, - "dimensions": - { - "disk":"disk_8" - } - }, - { - "name":"vds.filestor.disk_8.allthreads.update.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_8" - } - }, - { - "name":"vds.filestor.disk_8.allthreads.update.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_8" - } - }, - { - "name":"vds.filestor.disk_8.allthreads.update.sum.latency_read", - "description":"Latency of the source read in the request.", - "values": - { - "average":0.0, - "count":0, - "rate":0.0, - "min":0, - "max":0, - "last":0 - }, - "dimensions": - { - "disk":"disk_8" - } - }, - { - "name":"vds.filestor.disk_9.allthreads.operations", - "description":"Number of operations processed.", - "values": - { - "count":703, - "rate":2.343333 - }, - "dimensions": - { - "disk":"disk_9" - } - }, - { - "name":"vds.filestor.disk_9.allthreads.put.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":309, - "rate":1.03 - }, - "dimensions": - { - "disk":"disk_9" - } - }, - { - "name":"vds.filestor.disk_9.allthreads.put.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":4.841423948220065, - "count":309, - "rate":1.03, - "min":0, - "max":134, - "last":11 - }, - "dimensions": - { - "disk":"disk_9" - } - }, - { - "name":"vds.filestor.disk_9.allthreads.put.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_9" - } - }, - { - "name":"vds.filestor.disk_9.allthreads.get.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_9" - } - }, - { - "name":"vds.filestor.disk_9.allthreads.get.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":0.0, - "count":0, - "rate":0.0, - "min":0, - "max":0, - "last":0 - }, - "dimensions": - { - "disk":"disk_9" - } - }, - { - "name":"vds.filestor.disk_9.allthreads.get.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_9" - } - }, - { - "name":"vds.filestor.disk_9.allthreads.get.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_9" - } - }, - { - "name":"vds.filestor.disk_9.allthreads.remove.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":69, - "rate":0.23 - }, - "dimensions": - { - "disk":"disk_9" - } - }, - { - "name":"vds.filestor.disk_9.allthreads.remove.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":3.101449275362319, - "count":69, - "rate":0.23, - "min":0, - "max":18, - "last":7 - }, - "dimensions": - { - "disk":"disk_9" - } - }, - { - "name":"vds.filestor.disk_9.allthreads.remove.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_9" - } - }, - { - "name":"vds.filestor.disk_9.allthreads.remove.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":32, - "rate":0.106666 - }, - "dimensions": - { - "disk":"disk_9" - } - }, - { - "name":"vds.filestor.disk_9.allthreads.update.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":325, - "rate":1.083333 - }, - "dimensions": - { - "disk":"disk_9" - } - }, - { - "name":"vds.filestor.disk_9.allthreads.update.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":3.313846153846154, - "count":325, - "rate":1.083333, - "min":0, - "max":37, - "last":4 - }, - "dimensions": - { - "disk":"disk_9" - } - }, - { - "name":"vds.filestor.disk_9.allthreads.update.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_9" - } - }, - { - "name":"vds.filestor.disk_9.allthreads.update.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_9" - } - }, - { - "name":"vds.filestor.disk_9.allthreads.update.sum.latency_read", - "description":"Latency of the source read in the request.", - "values": - { - "average":0.0, - "count":0, - "rate":0.0, - "min":0, - "max":0, - "last":0 - }, - "dimensions": - { - "disk":"disk_9" - } - }, - { - "name":"vds.filestor.disk_10.allthreads.operations", - "description":"Number of operations processed.", - "values": - { - "count":344, - "rate":1.146666 - }, - "dimensions": - { - "disk":"disk_10" - } - }, - { - "name":"vds.filestor.disk_10.allthreads.put.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":233, - "rate":0.776666 - }, - "dimensions": - { - "disk":"disk_10" - } - }, - { - "name":"vds.filestor.disk_10.allthreads.put.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":5.24892703862661, - "count":233, - "rate":0.776666, - "min":0, - "max":65, - "last":4 - }, - "dimensions": - { - "disk":"disk_10" - } - }, - { - "name":"vds.filestor.disk_10.allthreads.put.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_10" - } - }, - { - "name":"vds.filestor.disk_10.allthreads.get.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_10" - } - }, - { - "name":"vds.filestor.disk_10.allthreads.get.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":0.0, - "count":0, - "rate":0.0, - "min":0, - "max":0, - "last":0 - }, - "dimensions": - { - "disk":"disk_10" - } - }, - { - "name":"vds.filestor.disk_10.allthreads.get.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_10" - } - }, - { - "name":"vds.filestor.disk_10.allthreads.get.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_10" - } - }, - { - "name":"vds.filestor.disk_10.allthreads.remove.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":29, - "rate":0.096666 - }, - "dimensions": - { - "disk":"disk_10" - } - }, - { - "name":"vds.filestor.disk_10.allthreads.remove.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":3.344827586206896, - "count":29, - "rate":0.096666, - "min":1, - "max":13, - "last":7 - }, - "dimensions": - { - "disk":"disk_10" - } - }, - { - "name":"vds.filestor.disk_10.allthreads.remove.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_10" - } - }, - { - "name":"vds.filestor.disk_10.allthreads.remove.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":15, - "rate":0.05 - }, - "dimensions": - { - "disk":"disk_10" - } - }, - { - "name":"vds.filestor.disk_10.allthreads.update.sum.count", - "description":"Number of requests processed.", - "values": - { - "count":82, - "rate":0.273333 - }, - "dimensions": - { - "disk":"disk_10" - } - }, - { - "name":"vds.filestor.disk_10.allthreads.update.sum.latency", - "description":"Latency of successful requests.", - "values": - { - "average":4.292682926829269, - "count":82, - "rate":0.273333, - "min":0, - "max":21, - "last":0 - }, - "dimensions": - { - "disk":"disk_10" - } - }, - { - "name":"vds.filestor.disk_10.allthreads.update.sum.failed", - "description":"Number of failed requests.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_10" - } - }, - { - "name":"vds.filestor.disk_10.allthreads.update.sum.not_found", - "description":"Number of requests that could not be completed due to source document not found.", - "values": - { - "count":0, - "rate":0.0 - }, - "dimensions": - { - "disk":"disk_10" - } - }, - { - "name":"vds.filestor.disk_10.allthreads.update.sum.latency_read", - "description":"Latency of the source read in the request.", - "values": - { - "average":0.0, - "count":0, - "rate":0.0, - "min":0, - "max":0, - "last":0 - }, - "dimensions": - { - "disk":"disk_10" - } } ] }, diff --git a/provided-dependencies/pom.xml b/provided-dependencies/pom.xml index a322be55560..da7cc007053 100755 --- a/provided-dependencies/pom.xml +++ b/provided-dependencies/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>provided-dependencies</artifactId> <packaging>jar</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <description>Dependencies that are installed and should not be included in artifacts with dependencies.</description> <dependencies> <!-- jdisc --> @@ -21,22 +21,16 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>jdisc_core</artifactId> <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.bouncycastle</groupId> - <artifactId>bcprov-jdk15on</artifactId> - </dependency> - <dependency> - <groupId>org.bouncycastle</groupId> - <artifactId>bcpkix-jdk15on</artifactId> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-util</artifactId> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-http</artifactId> + <exclusions> + <exclusion> + <groupId>org.apache.felix</groupId> + <artifactId>org.apache.felix.log</artifactId> + </exclusion> + <exclusion> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -57,10 +51,6 @@ <artifactId>guice</artifactId> <classifier>no_aop</classifier> </dependency> - <dependency> - <groupId>org.apache.felix</groupId> - <artifactId>org.apache.felix.main</artifactId> - </dependency> <!-- Dependencies used by container-core --> <dependency> diff --git a/routing-generator/pom.xml b/routing-generator/pom.xml index 3a197b94012..032d58acb51 100644 --- a/routing-generator/pom.xml +++ b/routing-generator/pom.xml @@ -6,14 +6,14 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <groupId>com.yahoo.vespa</groupId> <artifactId>routing-generator</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <!-- test --> @@ -62,6 +62,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>container-dev</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/RoutingGenerator.java b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/RoutingGenerator.java index a1d84873379..ba14bc83137 100644 --- a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/RoutingGenerator.java +++ b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/RoutingGenerator.java @@ -51,7 +51,6 @@ public class RoutingGenerator extends AbstractComponent { private final Router router; private final Clock clock; - @SuppressWarnings("removal") // TODO Vespa 8: remove private final ConfigSubscriber configSubscriber; private final ExecutorService executor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("routing-generator-config-subscriber")); @@ -61,7 +60,6 @@ public class RoutingGenerator extends AbstractComponent { private volatile RoutingTable routingTable = null; @Inject - @SuppressWarnings("removal") // TODO Vespa 8: remove public RoutingGenerator(ZoneConfig zoneConfig, RoutingStatus routingStatus, Metric metric) { this(new ConfigSourceSet(zoneConfig.configserver()), new Nginx(FileSystems.getDefault(), new ProcessExecuter(), @@ -72,7 +70,6 @@ public class RoutingGenerator extends AbstractComponent { Clock.systemUTC()); } - @SuppressWarnings("removal") // TODO Vespa 8: remove RoutingGenerator(ConfigSource configSource, Router router, Clock clock) { this.router = Objects.requireNonNull(router); this.clock = Objects.requireNonNull(clock); @@ -108,7 +105,6 @@ public class RoutingGenerator extends AbstractComponent { load(RoutingTable.from(lbServicesConfig, generation)); } - @SuppressWarnings("removal") // TODO Vespa 8: remove private <T extends ConfigInstance> void subscribeOn(Class<T> clazz, BiConsumer<T, Long> action, ConfigSource configSource, ExecutorService executor) { ConfigHandle<T> configHandle = null; @@ -142,7 +138,6 @@ public class RoutingGenerator extends AbstractComponent { } @Override - @SuppressWarnings("removal") // TODO Vespa 8: remove public void deconstruct() { configSubscriber.close(); // shutdownNow because ConfigSubscriber#nextGeneration blocks until next config, and we don't want to wait for diff --git a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/RoutingTable.java b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/RoutingTable.java index 90a246ad424..8db2387bf66 100644 --- a/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/RoutingTable.java +++ b/routing-generator/src/main/java/com/yahoo/vespa/hosted/routing/RoutingTable.java @@ -255,6 +255,7 @@ public class RoutingTable { Optional<InstanceName> instance, ClusterSpec.Id cluster, ZoneId zone) { if (instance.isEmpty()) { // Application-scoped endpoint if (dnsName.isEmpty()) throw new IllegalArgumentException("dnsName must given for application-scoped endpoint"); + @SuppressWarnings("deprecation") String endpointHash = Hashing.sha1().hashString(dnsName, StandardCharsets.UTF_8).toString(); return "application-" + endpointHash + "." +application.value() + "." + tenant.value(); } else { diff --git a/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/RoutingGeneratorTest.java b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/RoutingGeneratorTest.java index 3e8b9be572f..7c526eae4ed 100644 --- a/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/RoutingGeneratorTest.java +++ b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/RoutingGeneratorTest.java @@ -55,7 +55,6 @@ public class RoutingGeneratorTest { } - @SuppressWarnings("removal") // TODO Vespa 8: remove private static class ConfigSetMock extends ConfigSet { private int attempt = 0; diff --git a/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/TestUtil.java b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/TestUtil.java index 09440cfaac7..ac2db906825 100644 --- a/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/TestUtil.java +++ b/routing-generator/src/test/java/com/yahoo/vespa/hosted/routing/TestUtil.java @@ -18,7 +18,6 @@ public class TestUtil { private static final Path testData = Paths.get("src/test/resources/"); - @SuppressWarnings("removal") // TODO Vespa 8: remove public static RoutingTable readRoutingTable(String filename) { List<String> lines = Exceptions.uncheck(() -> Files.readAllLines(testFile(filename), StandardCharsets.UTF_8)); diff --git a/screwdriver.yaml b/screwdriver.yaml index 937abc30613..0ffc3316f95 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -6,7 +6,7 @@ cache: component-build: [/main_job_cache] shared: - image: vespaengine/vespa-build-centos7:latest + image: vespaengine/vespa-build-centos-stream8:latest environment: USER_SHELL_BIN: bash annotations: @@ -113,10 +113,9 @@ jobs: exit 1 fi - install-dependencies: | - yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo - yum install -y --setopt skip_missing_names_on_install=False docker-ce docker-ce-cli containerd.io + dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + dnf -y install docker-ce docker-ce-cli containerd.io docker system info - ln -sf $(which yum) /usr/local/bin/dnf - checkout: | mkdir -p workdir cd workdir @@ -149,6 +148,7 @@ jobs: time make -j ${NUM_THREADS} install DESTDIR=$WORKDIR/vespa-install - verify-rpm-build: | cd $WORKDIR + ulimit -c 0 time rpmbuild --rebuild --define="_topdir $WORKDIR/vespa-rpmbuild" \ --define "_debugsource_template %{nil}" \ --define "installdir $WORKDIR/vespa-install" *.src.rpm @@ -163,22 +163,22 @@ jobs: cp -a $LOCAL_MVN_REPO docker/repository cd docker docker build --file Dockerfile.systemtest \ - --build-arg VESPA_BASE_IMAGE=vespaengine/vespa-systemtest-base-centos7:latest \ + --build-arg VESPA_BASE_IMAGE=vespaengine/vespa-systemtest-base-centos-stream8:latest \ --build-arg SYSTEMTEST_BASE_IMAGE=vespa --build-arg SKIP_M2_POPULATE=true \ --target systemtest \ - --tag vespaengine/vespa-systemtest-centos7:$VESPA_VERSION \ - --tag vespaengine/vespa-systemtest-centos7:latest . + --tag vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION \ + --tag vespaengine/vespa-systemtest-centos-stream8:latest . - verify-test-image: | - docker run --rm -ti --entrypoint bash vespaengine/vespa-systemtest-centos7:$VESPA_VERSION -lc \ - "source /opt/rh/rh-ruby27/enable; /opt/vespa-systemtests/lib/node_server.rb & sleep 3 && ruby /opt/vespa-systemtests/tests/search/basicsearch/basic_search.rb --run test_basicsearch__ELASTIC" + docker run --rm -ti --entrypoint bash vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION -lc \ + "/opt/vespa-systemtests/lib/node_server.rb & sleep 3 && ruby /opt/vespa-systemtests/tests/search/basicsearch/basic_search.rb --run test_basicsearch__ELASTIC" - publish-test-image: | if [[ -z "$SD_PULL_REQUEST" ]]; then OPT_STATE="$(set +o)" set +x docker login --username aressem --password "$DOCKER_HUB_DEPLOY_KEY" eval "$OPT_STATE" - docker push docker.io/vespaengine/vespa-systemtest-centos7:$VESPA_VERSION - docker push docker.io/vespaengine/vespa-systemtest-centos7:latest + docker push docker.io/vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION + docker push docker.io/vespaengine/vespa-systemtest-centos-stream8:latest fi - *save-cache diff --git a/screwdriver/build-vespa.sh b/screwdriver/build-vespa.sh index c809504bef5..ca9ada81205 100755 --- a/screwdriver/build-vespa.sh +++ b/screwdriver/build-vespa.sh @@ -6,9 +6,7 @@ set -e readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd )" readonly NUM_THREADS=$(( $(nproc) + 2 )) -source /etc/profile.d/enable-devtoolset-11.sh -source /etc/profile.d/enable-rh-maven35.sh -source /etc/profile.d/enable-rh-git227.sh +source /etc/profile.d/enable-gcc-toolset-11.sh export MALLOC_ARENA_MAX=1 export MAVEN_OPTS="-Xss1m -Xms128m -Xmx2g" @@ -54,21 +52,24 @@ case $SHOULD_BUILD in ;; esac -if [[ $SHOULD_BUILD == systemtest ]]; then - yum -y --setopt=skip_missing_names_on_install=False install \ - zstd \ - devtoolset-11-gcc-c++ \ - devtoolset-11-libatomic-devel \ - devtoolset-11-binutils \ - libxml2-devel \ - rh-ruby27-rubygems-devel \ - rh-ruby27-ruby-devel \ - rh-ruby27 \ - rh-ruby27-rubygem-net-telnet - - source /opt/rh/rh-ruby27/enable - gem install libxml-ruby gnuplot distribution test-unit builder concurrent-ruby ffi +if [[ $SHOULD_BUILD == systemtest ]]; then + dnf module enable -y ruby:2.7 + dnf install -y \ + gcc-toolset-11-annobin \ + gcc-toolset-11-annobin-plugin-gcc \ + gcc-toolset-11-binutils \ + gcc-toolset-11-gcc-c++ \ + gcc-toolset-11-libatomic-devel \ + libxml2-devel \ + ruby \ + ruby-devel \ + rubygems-devel \ + rubygem-net-telnet \ + zstd + source /opt/rh/gcc-toolset-11/enable + gem install libxml-ruby gnuplot distribution test-unit builder concurrent-ruby bigdecimal ffi parallel + cd $HOME git clone https://github.com/vespa-engine/system-test export SYSTEM_TEST_DIR=$(pwd)/system-test diff --git a/screwdriver/delete-old-artifactory-artifacts.sh b/screwdriver/delete-old-artifactory-artifacts.sh index 9235575baff..80cee37bd72 100755 --- a/screwdriver/delete-old-artifactory-artifacts.sh +++ b/screwdriver/delete-old-artifactory-artifacts.sh @@ -12,13 +12,13 @@ if [[ ! -f /etc/yum.repos.d/vespa.repo ]]; then cat << EOF > /etc/yum.repos.d/vespa.repo [vespa-release] name=Vespa releases -baseurl=$ARTIFACTORY_URL/vespa/centos/7/release/\$basearch +baseurl=$ARTIFACTORY_URL/vespa/centos/8/release/\$basearch gpgcheck=0 enabled=1 EOF fi -VERSIONS_TO_DELETE=$(yum list --quiet --showduplicates --disablerepo='*' --enablerepo=vespa-release vespa | awk '/[0-9].*\.[0-9].*\.[0-9].*/{print $2}' | sort -V | head -n -200) +VERSIONS_TO_DELETE=$(dnf list --quiet --showduplicates --disablerepo='*' --enablerepo=vespa-release vespa | awk '/[0-9].*\.[0-9].*\.[0-9].*/{print $2}' | sort -V | head -n -200) RPMS_TO_DELETE=$(mktemp) trap "rm -f $RPMS_TO_DELETE" EXIT diff --git a/screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh b/screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh index dd0c953b424..3bcb4e101a5 100755 --- a/screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh +++ b/screwdriver/publish-unpublished-rpms-to-jfrog-cloud.sh @@ -13,7 +13,7 @@ if [[ ! -f /etc/yum.repos.d/group_vespa-vespa-epel-7.repo ]]; then cat << 'EOF' > /etc/yum.repos.d/group_vespa-vespa-epel-7.repo [copr:copr.fedorainfracloud.org:group_vespa:vespa] name=Copr repo for vespa owned by @vespa -baseurl=https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/epel-7-$basearch/ +baseurl=https://download.copr.fedorainfracloud.org/results/@vespa/vespa/centos-stream-8-$basearch/ type=rpm-md gpgcheck=1 gpgkey=https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/pubkey.gpg @@ -27,7 +27,7 @@ if [[ ! -f /etc/yum.repos.d/vespa.repo ]]; then cat << 'EOF' > /etc/yum.repos.d/vespa.repo [vespa-release] name=Vespa releases -baseurl=https://artifactory.yahooinc.com/artifactory/vespa/centos/7/release/$basearch +baseurl=https://artifactory.yahooinc.com/artifactory/vespa/centos/8/release/$basearch gpgcheck=0 enabled=1 EOF @@ -36,16 +36,16 @@ fi readonly COPR_PACKAGES=$(mktemp) trap "rm -f $COPR_PACKAGES" EXIT -yum list -q --disablerepo='*' --enablerepo=copr:copr.fedorainfracloud.org:group_vespa:vespa --showduplicates 'vespa*' | grep "Available Packages" -A 100000 | tail -n +2 | sed "s/\.x86_64\ */-/"| awk '{print $1}' > $COPR_PACKAGES +dnf list -q --disablerepo='*' --enablerepo=copr:copr.fedorainfracloud.org:group_vespa:vespa --showduplicates 'vespa*' | grep "Available Packages" -A 100000 | tail -n +2 | sed "s/\.x86_64\ */-/"| awk '{print $1}' > $COPR_PACKAGES echo "Packages on Copr:" cat $COPR_PACKAGES echo for pv in $(cat $COPR_PACKAGES); do - if ! yum list -q --disablerepo='*' --enablerepo=vespa-release $pv &> /dev/null; then + if ! dnf list -q --disablerepo='*' --enablerepo=vespa-release $pv &> /dev/null; then echo "$pv not found on JFrog Clould. Downloading..." - yumdownloader -q $pv + dnf download -q $pv echo "$pv downloaded." fi done diff --git a/screwdriver/release-container-image.sh b/screwdriver/release-container-image.sh index 87f06762169..30287bc5b52 100755 --- a/screwdriver/release-container-image.sh +++ b/screwdriver/release-container-image.sh @@ -26,7 +26,7 @@ cd $BUILD_DIR git clone --depth 1 https://github.com/vespa-engine/docker-image cd docker-image -docker build --build-arg VESPA_VERSION=$VESPA_VERSION \ +docker build --build-arg VESPA_VERSION=$VESPA_VERSION --file Dockerfile.stream8 \ --tag docker.io/vespaengine/vespa:$VESPA_VERSION --tag docker.io/vespaengine/vespa:latest \ --tag ghcr.io/vespa-engine/vespa:$VESPA_VERSION --tag ghcr.io/vespa-engine/vespa:latest . diff --git a/screwdriver/upload-rpm-to-artifactory.sh b/screwdriver/upload-rpm-to-artifactory.sh index 9482bc2dc6a..0c679debb48 100755 --- a/screwdriver/upload-rpm-to-artifactory.sh +++ b/screwdriver/upload-rpm-to-artifactory.sh @@ -5,7 +5,7 @@ set -euo pipefail RPM=$1 OS_DISTRO=centos -RELEASEVER=7 +RELEASEVER=8 MATURITY=release BASEARCH=x86_64 diff --git a/searchcore/pom.xml b/searchcore/pom.xml index 27b0e55c781..46b576921cf 100644 --- a/searchcore/pom.xml +++ b/searchcore/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>searchcore</artifactId> <packaging>jar</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>${project.artifactId}</name> <dependencies> <dependency> diff --git a/searchcore/src/tests/proton/common/cachedselect_test.cpp b/searchcore/src/tests/proton/common/cachedselect_test.cpp index 2fd798cd8fd..64c95c36350 100644 --- a/searchcore/src/tests/proton/common/cachedselect_test.cpp +++ b/searchcore/src/tests/proton/common/cachedselect_test.cpp @@ -190,7 +190,7 @@ public: MyIntAv(const string &name) : SvIntAttr(name, Config(BasicType::INT32, CollectionType::SINGLE, - true, false)), + true)), _gets(0) { } diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp index 4350ebc0ee2..b8579d66c5f 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp @@ -40,9 +40,6 @@ assertDefaultAttribute(const AttributesConfig::Attribute &attribute, if (!EXPECT_FALSE(attribute.fastsearch)) { return false; } - if (!EXPECT_FALSE(attribute.huge)) { - return false; - } if (!EXPECT_FALSE(attribute.paged)) { return false; } @@ -66,9 +63,6 @@ assertFastSearchAttribute(const AttributesConfig::Attribute &attribute, if (!EXPECT_TRUE(attribute.fastsearch)) { return false; } - if (!EXPECT_FALSE(attribute.huge)) { - return false; - } if (!EXPECT_FALSE(attribute.paged)) { return false; } @@ -92,9 +86,6 @@ assertFastSearchAndMoreAttribute(const AttributesConfig::Attribute &attribute, if (!EXPECT_TRUE(attribute.fastsearch)) { return false; } - if (!EXPECT_TRUE(attribute.huge)) { - return false; - } if (!EXPECT_TRUE(attribute.paged)) { return false; } @@ -241,7 +232,6 @@ setupFastSearchAndMoreAttribute(const vespalib::string name) AttributesConfig::Attribute attribute; attribute.name = name; attribute.fastsearch = true; - attribute.huge = true; attribute.paged = true; attribute.enablebitvectors = true; attribute.enableonlybitvector = true; diff --git a/searchcore/src/tests/proton/matching/matching_stats_test.cpp b/searchcore/src/tests/proton/matching/matching_stats_test.cpp index ce623aa877e..b8bc4eaa7e6 100644 --- a/searchcore/src/tests/proton/matching/matching_stats_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_stats_test.cpp @@ -47,51 +47,43 @@ TEST("requireThatAverageTimesAreRecorded") { EXPECT_APPROX(0.0, stats.matchTimeAvg(), 0.00001); EXPECT_APPROX(0.0, stats.groupingTimeAvg(), 0.00001); EXPECT_APPROX(0.0, stats.rerankTimeAvg(), 0.00001); - EXPECT_APPROX(0.0, stats.queryCollateralTimeAvg(), 0.00001); EXPECT_APPROX(0.0, stats.querySetupTimeAvg(), 0.00001); EXPECT_APPROX(0.0, stats.queryLatencyAvg(), 0.00001); EXPECT_EQUAL(0u, stats.matchTimeCount()); EXPECT_EQUAL(0u, stats.groupingTimeCount()); EXPECT_EQUAL(0u, stats.rerankTimeCount()); - EXPECT_EQUAL(0u, stats.queryCollateralTimeCount()); EXPECT_EQUAL(0u, stats.querySetupTimeCount()); EXPECT_EQUAL(0u, stats.queryLatencyCount()); - stats.matchTime(0.01).groupingTime(0.1).rerankTime(0.5).queryCollateralTime(2.0).querySetupTime(2.0).queryLatency(1.0); + stats.matchTime(0.01).groupingTime(0.1).rerankTime(0.5).querySetupTime(2.0).queryLatency(1.0); EXPECT_APPROX(0.01, stats.matchTimeAvg(), 0.00001); EXPECT_APPROX(0.1, stats.groupingTimeAvg(), 0.00001); EXPECT_APPROX(0.5, stats.rerankTimeAvg(), 0.00001); - EXPECT_APPROX(2.0, stats.queryCollateralTimeAvg(), 0.00001); EXPECT_APPROX(2.0, stats.querySetupTimeAvg(), 0.00001); EXPECT_APPROX(1.0, stats.queryLatencyAvg(), 0.00001); - stats.add(MatchingStats().matchTime(0.03).groupingTime(0.3).rerankTime(1.5).queryCollateralTime(6.0).querySetupTime(6.0).queryLatency(3.0)); + stats.add(MatchingStats().matchTime(0.03).groupingTime(0.3).rerankTime(1.5).querySetupTime(6.0).queryLatency(3.0)); EXPECT_APPROX(0.02, stats.matchTimeAvg(), 0.00001); EXPECT_APPROX(0.2, stats.groupingTimeAvg(), 0.00001); EXPECT_APPROX(1.0, stats.rerankTimeAvg(), 0.00001); - EXPECT_APPROX(4.0, stats.queryCollateralTimeAvg(), 0.00001); EXPECT_APPROX(4.0, stats.querySetupTimeAvg(), 0.00001); EXPECT_APPROX(2.0, stats.queryLatencyAvg(), 0.00001); stats.add(MatchingStats().matchTime(0.05) .groupingTime(0.5) .rerankTime(2.5) - .queryCollateralTime(10.0) .querySetupTime(10.0) .queryLatency(5.0)); stats.add(MatchingStats().matchTime(0.05).matchTime(0.03) .groupingTime(0.5).groupingTime(0.3) .rerankTime(2.5).rerankTime(1.5) - .queryCollateralTime(10.0).queryCollateralTime(6.0) .querySetupTime(10.0).querySetupTime(6.0) .queryLatency(5.0).queryLatency(3.0)); EXPECT_APPROX(0.03, stats.matchTimeAvg(), 0.00001); EXPECT_APPROX(0.3, stats.groupingTimeAvg(), 0.00001); EXPECT_APPROX(1.5, stats.rerankTimeAvg(), 0.00001); - EXPECT_APPROX(6.0, stats.queryCollateralTimeAvg(), 0.00001); EXPECT_APPROX(6.0, stats.querySetupTimeAvg(), 0.00001); EXPECT_APPROX(3.0, stats.queryLatencyAvg(), 0.00001); EXPECT_EQUAL(4u, stats.matchTimeCount()); EXPECT_EQUAL(4u, stats.groupingTimeCount()); EXPECT_EQUAL(4u, stats.rerankTimeCount()); - EXPECT_EQUAL(4u, stats.queryCollateralTimeCount()); EXPECT_EQUAL(4u, stats.querySetupTimeCount()); EXPECT_EQUAL(4u, stats.queryLatencyCount()); } @@ -101,63 +93,53 @@ TEST("requireThatMinMaxTimesAreRecorded") { EXPECT_APPROX(0.0, stats.matchTimeMin(), 0.00001); EXPECT_APPROX(0.0, stats.groupingTimeMin(), 0.00001); EXPECT_APPROX(0.0, stats.rerankTimeMin(), 0.00001); - EXPECT_APPROX(0.0, stats.queryCollateralTimeMin(), 0.00001); EXPECT_APPROX(0.0, stats.querySetupTimeMin(), 0.00001); EXPECT_APPROX(0.0, stats.queryLatencyMin(), 0.00001); EXPECT_APPROX(0.0, stats.matchTimeMax(), 0.00001); EXPECT_APPROX(0.0, stats.groupingTimeMax(), 0.00001); EXPECT_APPROX(0.0, stats.rerankTimeMax(), 0.00001); - EXPECT_APPROX(0.0, stats.queryCollateralTimeMax(), 0.00001); EXPECT_APPROX(0.0, stats.querySetupTimeMax(), 0.00001); EXPECT_APPROX(0.0, stats.queryLatencyMax(), 0.00001); - stats.matchTime(0.01).groupingTime(0.1).rerankTime(0.5).queryCollateralTime(2.0).querySetupTime(2.0).queryLatency(1.0); + stats.matchTime(0.01).groupingTime(0.1).rerankTime(0.5).querySetupTime(2.0).queryLatency(1.0); EXPECT_APPROX(0.01, stats.matchTimeMin(), 0.00001); EXPECT_APPROX(0.1, stats.groupingTimeMin(), 0.00001); EXPECT_APPROX(0.5, stats.rerankTimeMin(), 0.00001); - EXPECT_APPROX(2.0, stats.queryCollateralTimeMin(), 0.00001); EXPECT_APPROX(2.0, stats.querySetupTimeMin(), 0.00001); EXPECT_APPROX(1.0, stats.queryLatencyMin(), 0.00001); EXPECT_APPROX(0.01, stats.matchTimeMax(), 0.00001); EXPECT_APPROX(0.1, stats.groupingTimeMax(), 0.00001); EXPECT_APPROX(0.5, stats.rerankTimeMax(), 0.00001); - EXPECT_APPROX(2.0, stats.queryCollateralTimeMax(), 0.00001); EXPECT_APPROX(2.0, stats.querySetupTimeMax(), 0.00001); EXPECT_APPROX(1.0, stats.queryLatencyMax(), 0.00001); - stats.add(MatchingStats().matchTime(0.03).groupingTime(0.3).rerankTime(1.5).queryCollateralTime(6.0).querySetupTime(6.0).queryLatency(3.0)); + stats.add(MatchingStats().matchTime(0.03).groupingTime(0.3).rerankTime(1.5).querySetupTime(6.0).queryLatency(3.0)); EXPECT_APPROX(0.01, stats.matchTimeMin(), 0.00001); EXPECT_APPROX(0.1, stats.groupingTimeMin(), 0.00001); EXPECT_APPROX(0.5, stats.rerankTimeMin(), 0.00001); - EXPECT_APPROX(2.0, stats.queryCollateralTimeMin(), 0.00001); EXPECT_APPROX(2.0, stats.querySetupTimeMin(), 0.00001); EXPECT_APPROX(1.0, stats.queryLatencyMin(), 0.00001); EXPECT_APPROX(0.03, stats.matchTimeMax(), 0.00001); EXPECT_APPROX(0.3, stats.groupingTimeMax(), 0.00001); EXPECT_APPROX(1.5, stats.rerankTimeMax(), 0.00001); - EXPECT_APPROX(6.0, stats.queryCollateralTimeMax(), 0.00001); EXPECT_APPROX(6.0, stats.querySetupTimeMax(), 0.00001); EXPECT_APPROX(3.0, stats.queryLatencyMax(), 0.00001); stats.add(MatchingStats().matchTime(0.05) .groupingTime(0.5) .rerankTime(2.5) - .queryCollateralTime(10.0) .querySetupTime(10.0) .queryLatency(5.0)); stats.add(MatchingStats().matchTime(0.05).matchTime(0.03) .groupingTime(0.5).groupingTime(0.3) .rerankTime(2.5).rerankTime(1.5) - .queryCollateralTime(10.0).queryCollateralTime(6.0) .querySetupTime(10.0).querySetupTime(6.0) .queryLatency(5.0).queryLatency(3.0)); EXPECT_APPROX(0.01, stats.matchTimeMin(), 0.00001); EXPECT_APPROX(0.1, stats.groupingTimeMin(), 0.00001); EXPECT_APPROX(0.5, stats.rerankTimeMin(), 0.00001); - EXPECT_APPROX(2.0, stats.queryCollateralTimeMin(), 0.00001); EXPECT_APPROX(2.0, stats.querySetupTimeMin(), 0.00001); EXPECT_APPROX(1.0, stats.queryLatencyMin(), 0.00001); EXPECT_APPROX(0.05, stats.matchTimeMax(), 0.00001); EXPECT_APPROX(0.5, stats.groupingTimeMax(), 0.00001); EXPECT_APPROX(2.5, stats.rerankTimeMax(), 0.00001); - EXPECT_APPROX(10.0, stats.queryCollateralTimeMax(), 0.00001); EXPECT_APPROX(10.0, stats.querySetupTimeMax(), 0.00001); EXPECT_APPROX(5.0, stats.queryLatencyMax(), 0.00001); } diff --git a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp index 802d175b5af..6f4420695c4 100644 --- a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp +++ b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp @@ -270,6 +270,7 @@ struct SimpleSetup : Setup { index("list", DataType::STRING, CollectionType::ARRAY); index("keywords", DataType::STRING, CollectionType::WEIGHTEDSET); attribute("date", DataType::INT32, CollectionType::SINGLE); + attribute("pos_zcurve", DataType::INT64, CollectionType::SINGLE); attribute("imported_attr", DataType::INT32, CollectionType::SINGLE, true); constants["my_tensor"] = "tensor(x{},y{})"; } @@ -493,6 +494,16 @@ TEST_F("require that query tensor default value expression does not need paramet //----------------------------------------------------------------------------- +TEST_F("require that zcurve distance can be set up", SimpleSetup()) { + f.verify_valid({"distance(pos)"}); +} + +TEST_F("require that zcurve distance must be backed by an attribute", SimpleSetup()) { + f.verify_invalid({"distance(unknown)"}); +} + +//----------------------------------------------------------------------------- + TEST_F("cleanup files", Setup()) { ASSERT_TRUE(vespalib::Process::run(fmt("rm -rf %s", gen_dir.c_str()))); } diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index bb664ea1743..ffd9db2f58b 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -383,17 +383,6 @@ lidspacecompaction.allowedlidbloat int default=1 ## The lid bloat factor must be >= allowedlidbloatfactor before considering compaction. lidspacecompaction.allowedlidbloatfactor double default=0.01 -## DEPRECATED (no longer used): Remove on Vespa 8 -## The delay (in seconds) for when the last remove batch operation would be considered to block lid space compaction. -## -## When considering compaction, if the document meta store has received a remove batch operation in the last delay seconds, -## the lid space compaction job is blocked. It is considered again at the next regular interval (see above). -## -## Remove batch operations are used when deleting buckets on a content node. -## This functionality ensures that during massive deleting of buckets (e.g. as part of redistribution of data to a new node), -## lid space compaction do not interfere, but instead is applied after deleting of buckets is complete. -lidspacecompaction.removebatchblockdelay double default=2.0 - ## The rate (ops / second) of remove batch operations for when to block lid space compaction. ## ## When considering compaction, if the current observed rate of remove batch operations diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp index e21230a1d58..e5a087e1bf2 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp @@ -28,7 +28,6 @@ AttributesConfigScout::adjust(AttributesConfig::Attribute &attr, attr.enablebitvectors = liveAttr.enablebitvectors; attr.enableonlybitvector = liveAttr.enableonlybitvector; attr.fastsearch = liveAttr.fastsearch; - attr.huge = liveAttr.huge; attr.paged = liveAttr.paged; // Note: Predicate attributes only handle changes for the dense-posting-list-threshold config. attr.densepostinglistthreshold = liveAttr.densepostinglistthreshold; diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index 381915f53d4..88d5281535a 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -290,7 +290,6 @@ Matcher::match(const SearchRequest &request, vespalib::ThreadBundle &threadBundl request.ranking.c_str()); } double querySetupTime = vespalib::to_s(total_matching_time.elapsed()) - my_stats.queryLatencyAvg(); - my_stats.queryCollateralTime(querySetupTime); // TODO: Remove in Vespa 8 my_stats.querySetupTime(querySetupTime); { vespalib::duration duration = request.getTimeUsed(); diff --git a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp index bbb9dba9a30..86fb3cf8107 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp @@ -29,7 +29,6 @@ MatchingStats::MatchingStats(double prev_soft_doom_factor) _softDoomed(0), _doomOvertime(), _softDoomFactor(prev_soft_doom_factor), - _queryCollateralTime(), // TODO: Remove in Vespa 8 _querySetupTime(), _queryLatency(), _matchTime(), @@ -70,7 +69,6 @@ MatchingStats::add(const MatchingStats &rhs) _softDoomed += rhs.softDoomed(); _doomOvertime.add(rhs._doomOvertime); - _queryCollateralTime.add(rhs._queryCollateralTime); // TODO: Remove in Vespa 8 _querySetupTime.add(rhs._querySetupTime); _queryLatency.add(rhs._queryLatency); _matchTime.add(rhs._matchTime); diff --git a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h index eafa14870fa..4139bfbaf66 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h +++ b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h @@ -127,7 +127,6 @@ private: Avg _doomOvertime; using SoftDoomFactor = vespalib::datastore::AtomicValueWrapper<double>; SoftDoomFactor _softDoomFactor; - Avg _queryCollateralTime; // TODO: Remove in Vespa 8 Avg _querySetupTime; Avg _queryLatency; Avg _matchTime; @@ -171,13 +170,6 @@ public: double softDoomFactor() const { return _softDoomFactor.load_relaxed(); } MatchingStats &updatesoftDoomFactor(vespalib::duration hardLimit, vespalib::duration softLimit, vespalib::duration duration); - // TODO: Remove in Vespa 8 - MatchingStats &queryCollateralTime(double time_s) { _queryCollateralTime.set(time_s); return *this; } - double queryCollateralTimeAvg() const { return _queryCollateralTime.avg(); } - size_t queryCollateralTimeCount() const { return _queryCollateralTime.count(); } - double queryCollateralTimeMin() const { return _queryCollateralTime.min(); } - double queryCollateralTimeMax() const { return _queryCollateralTime.max(); } - MatchingStats &querySetupTime(double time_s) { _querySetupTime.set(time_s); return *this; } double querySetupTimeAvg() const { return _querySetupTime.avg(); } size_t querySetupTimeCount() const { return _querySetupTime.count(); } diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp index e895a03b190..2535b518335 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp @@ -118,8 +118,6 @@ DocumentDBTaggedMetrics::MatchingMetrics::update(const MatchingStats &stats) docsReRanked.inc(stats.docsReRanked()); softDoomedQueries.inc(stats.softDoomed()); queries.inc(stats.queries()); - queryCollateralTime.addValueBatch(stats.queryCollateralTimeAvg(), stats.queryCollateralTimeCount(), - stats.queryCollateralTimeMin(), stats.queryCollateralTimeMax()); querySetupTime.addValueBatch(stats.querySetupTimeAvg(), stats.querySetupTimeCount(), stats.querySetupTimeMin(), stats.querySetupTimeMax()); queryLatency.addValueBatch(stats.queryLatencyAvg(), stats.queryLatencyCount(), @@ -133,7 +131,6 @@ DocumentDBTaggedMetrics::MatchingMetrics::MatchingMetrics(MetricSet *parent) docsReRanked("docs_reranked", {}, "Number of documents re-ranked (second phase)", this), queries("queries", {}, "Number of queries executed", this), softDoomedQueries("soft_doomed_queries", {}, "Number of queries hitting the soft timeout", this), - queryCollateralTime("query_collateral_time", {}, "Average time (sec) spent setting up and tearing down queries", this), querySetupTime("query_setup_time", {}, "Average time (sec) spent setting up and tearing down queries", this), queryLatency("query_latency", {}, "Total average latency (sec) when matching and ranking a query", this) { @@ -155,7 +152,6 @@ DocumentDBTaggedMetrics::MatchingMetrics::RankProfileMetrics::RankProfileMetrics matchTime("match_time", {}, "Average time (sec) for matching a query (1st phase)", this), groupingTime("grouping_time", {}, "Average time (sec) spent on grouping", this), rerankTime("rerank_time", {}, "Average time (sec) spent on 2nd phase ranking", this), - queryCollateralTime("query_collateral_time", {}, "Average time (sec) spent setting up and tearing down queries", this), querySetupTime("query_setup_time", {}, "Average time (sec) spent setting up and tearing down queries", this), queryLatency("query_latency", {}, "Total average latency (sec) when matching and ranking a query", this) { @@ -208,8 +204,6 @@ DocumentDBTaggedMetrics::MatchingMetrics::RankProfileMetrics::update(const metri stats.groupingTimeMin(), stats.groupingTimeMax()); rerankTime.addValueBatch(stats.rerankTimeAvg(), stats.rerankTimeCount(), stats.rerankTimeMin(), stats.rerankTimeMax()); - queryCollateralTime.addValueBatch(stats.queryCollateralTimeAvg(), stats.queryCollateralTimeCount(), - stats.queryCollateralTimeMin(), stats.queryCollateralTimeMax()); querySetupTime.addValueBatch(stats.querySetupTimeAvg(), stats.querySetupTimeCount(), stats.querySetupTimeMin(), stats.querySetupTimeMax()); queryLatency.addValueBatch(stats.queryLatencyAvg(), stats.queryLatencyCount(), diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h index c9d321d6752..7d3a50647b9 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h @@ -117,7 +117,6 @@ struct DocumentDBTaggedMetrics : metrics::MetricSet metrics::LongCountMetric docsReRanked; metrics::LongCountMetric queries; metrics::LongCountMetric softDoomedQueries; - metrics::DoubleAverageMetric queryCollateralTime; metrics::DoubleAverageMetric querySetupTime; metrics::DoubleAverageMetric queryLatency; @@ -147,7 +146,6 @@ struct DocumentDBTaggedMetrics : metrics::MetricSet metrics::DoubleAverageMetric matchTime; metrics::DoubleAverageMetric groupingTime; metrics::DoubleAverageMetric rerankTime; - metrics::DoubleAverageMetric queryCollateralTime; metrics::DoubleAverageMetric querySetupTime; metrics::DoubleAverageMetric queryLatency; DocIdPartitions partitions; diff --git a/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.cpp index 406432ef697..7c4dcf434a3 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.cpp @@ -7,7 +7,6 @@ namespace proton { void ExecutorMetrics::update(const vespalib::ExecutorStats &stats) { - maxPending.set(stats.queueSize.max()); accepted.inc(stats.acceptedTasks); rejected.inc(stats.rejectedTasks); wakeupCount.inc(stats.wakeupCount); @@ -18,7 +17,6 @@ ExecutorMetrics::update(const vespalib::ExecutorStats &stats) ExecutorMetrics::ExecutorMetrics(const std::string &name, metrics::MetricSet *parent) : metrics::MetricSet(name, {}, "Instance specific thread executor metrics", parent), - maxPending("maxpending", {}, "Maximum number of pending (active + queued) tasks", this), accepted("accepted", {}, "Number of accepted tasks", this), rejected("rejected", {}, "Number of rejected tasks", this), wakeupCount("wakeups", {}, "Number of times a worker thread has been woken up", this), diff --git a/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.h index 31d959a399f..af3a27da240 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.h @@ -11,7 +11,6 @@ namespace proton { struct ExecutorMetrics : metrics::MetricSet { - metrics::LongValueMetric maxPending; // TODO Remove on Vespa 8 or sooner if possible. metrics::LongCountMetric accepted; metrics::LongCountMetric rejected; metrics::LongCountMetric wakeupCount; diff --git a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp index d026ef549d4..0ad248060fd 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp @@ -36,11 +36,7 @@ ResourceUsageMetrics::DetailedResourceMetrics::~DetailedResourceMetrics() = defa ResourceUsageMetrics::ResourceUsageMetrics(metrics::MetricSet *parent) : MetricSet("resource_usage", {}, "Usage metrics for various resources in this content node", parent), disk("disk", {}, "The relative amount of disk used by this content node (transient usage not included, value in the range [0, 1]). Same value as reported to the cluster controller", this), - diskUtilization("disk_utilization", {}, "The relative amount of disk used compared to the disk resource limit", this), memory("memory", {}, "The relative amount of memory used by this content node (transient usage not included, value in the range [0, 1]). Same value as reported to the cluster controller", this), - memoryUtilization("memory_utilization", {}, "The relative amount of memory used compared to the memory resource limit", this), - transient_memory("transient_memory", {}, "The relative amount of transient memory needed for loading attributes. Max value among all attributes (value in the range [0, 1])", this), - transient_disk("transient_disk", {}, "The relative amount of transient disk needed for running disk index fusion. Max value among all disk indexes (value in the range [0, 1])", this), disk_usage("disk", this), memory_usage("memory", this), memoryMappings("memory_mappings", {}, "The number of mapped memory areas", this), diff --git a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h index 97cad935dba..6f0f16d822e 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h @@ -32,15 +32,8 @@ struct ResourceUsageMetrics : metrics::MetricSet ~DetailedResourceMetrics(); }; - // TODO Vespa 8: Remove diskUtilization, memoryUtilization, transient_memory, transient_disk. - // These are now included in disk_usage and memory_usage. - metrics::DoubleValueMetric disk; - metrics::DoubleValueMetric diskUtilization; metrics::DoubleValueMetric memory; - metrics::DoubleValueMetric memoryUtilization; - metrics::DoubleValueMetric transient_memory; - metrics::DoubleValueMetric transient_disk; DetailedResourceMetrics disk_usage; DetailedResourceMetrics memory_usage; metrics::LongValueMetric memoryMappings; diff --git a/searchcore/src/vespa/searchcore/proton/server/matchers.cpp b/searchcore/src/vespa/searchcore/proton/server/matchers.cpp index e87f9928ce4..809f2e0a744 100644 --- a/searchcore/src/vespa/searchcore/proton/server/matchers.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/matchers.cpp @@ -59,8 +59,7 @@ Matchers::lookup(const vespalib::string &name) const Map::const_iterator found(_rpmap.find(name)); if (found == _rpmap.end()) { if (_default) { - // TODO Report as issue on Vespa 8 - // vespalib::Issue::report(fmt("Failed to find rank-profile '%s'. Falling back to 'default'", name.c_str())); + vespalib::Issue::report(fmt("Failed to find rank-profile '%s'. Falling back to 'default'", name.c_str())); return _default; } else { vespalib::Issue::report(fmt("Failed to find rank-profile '%s'. Most likely a configuration issue.", name.c_str())); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 871a72de3ca..ff5924fa6bc 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -760,15 +760,11 @@ Proton::updateMetrics(const metrics::MetricLockGuard &) const DiskMemUsageFilter &usageFilter = _diskMemUsageSampler->writeFilter(); auto dm_metrics = usageFilter.get_metrics(); metrics.resourceUsage.disk.set(dm_metrics.non_transient_disk_usage()); - metrics.resourceUsage.diskUtilization.set(dm_metrics.total_disk_utilization()); - metrics.resourceUsage.transient_disk.set(dm_metrics.transient_disk_usage()); metrics.resourceUsage.disk_usage.total.set(dm_metrics.total_disk_usage()); metrics.resourceUsage.disk_usage.total_util.set(dm_metrics.total_disk_utilization()); metrics.resourceUsage.disk_usage.transient.set(dm_metrics.transient_disk_usage()); metrics.resourceUsage.memory.set(dm_metrics.non_transient_memory_usage()); - metrics.resourceUsage.memoryUtilization.set(dm_metrics.total_memory_utilization()); - metrics.resourceUsage.transient_memory.set(dm_metrics.transient_memory_usage()); metrics.resourceUsage.memory_usage.total.set(dm_metrics.total_memory_usage()); metrics.resourceUsage.memory_usage.total_util.set(dm_metrics.total_memory_utilization()); metrics.resourceUsage.memory_usage.transient.set(dm_metrics.transient_memory_usage()); diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index 3081c88ec99..1896337b419 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -334,7 +334,6 @@ "public boolean equals(java.lang.Object)", "public java.lang.String toString()", "public java.util.Map getRankProperties(com.yahoo.searchlib.rankingexpression.rule.SerializationContext)", - "public java.util.Map getRankProperties(java.util.List)", "public static java.lang.String propertyName(java.lang.String)", "public static java.lang.String propertyExpressionName(java.lang.String)", "public static java.lang.String extractScriptName(java.lang.String)", @@ -1325,10 +1324,8 @@ ], "methods": [ "public void <init>(com.yahoo.searchlib.rankingexpression.evaluation.Value)", - "public void <init>(com.yahoo.searchlib.rankingexpression.evaluation.Value, java.lang.String)", "public com.yahoo.searchlib.rankingexpression.evaluation.Value getValue()", "public java.lang.StringBuilder toString(java.lang.StringBuilder, com.yahoo.searchlib.rankingexpression.rule.SerializationContext, java.util.Deque, com.yahoo.searchlib.rankingexpression.rule.CompositeNode)", - "public java.lang.String sourceString()", "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public com.yahoo.searchlib.rankingexpression.evaluation.Value evaluate(com.yahoo.searchlib.rankingexpression.evaluation.Context)", "public int hashCode()" @@ -1460,9 +1457,7 @@ "public void <init>(java.util.Collection, java.util.Map)", "public void <init>(java.util.Map)", "public void <init>(java.util.Map, java.util.Map)", - "protected void <init>(com.google.common.collect.ImmutableMap, java.util.Map)", "public com.yahoo.searchlib.rankingexpression.ExpressionFunction getFunction(java.lang.String)", - "protected com.google.common.collect.ImmutableMap functions()", "protected java.util.Map getFunctions()", "public java.lang.String getBinding(java.lang.String)", "public com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext withBindings(java.util.Map)", @@ -1627,7 +1622,6 @@ "public void <init>(java.util.Collection, java.util.Map, com.yahoo.tensor.evaluation.TypeContext)", "public void <init>(java.util.Map, java.util.Map, java.util.Map)", "public void <init>(java.util.Map, java.util.Map, java.util.Optional, java.util.Map)", - "public void <init>(com.google.common.collect.ImmutableMap, java.util.Map, java.util.Map)", "public java.util.Optional typeContext()", "public void addFunctionSerialization(java.lang.String, java.lang.String)", "public void addArgumentTypeSerialization(java.lang.String, java.lang.String, com.yahoo.tensor.TensorType)", diff --git a/searchlib/pom.xml b/searchlib/pom.xml index d1a167b14c8..e1e61c954b9 100644 --- a/searchlib/pom.xml +++ b/searchlib/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>searchlib</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>searchlib</name> <description>Search library functions.</description> <dependencies> @@ -68,6 +68,10 @@ <artifactId>protobuf-java</artifactId> <scope>provided</scope> </dependency> + <dependency> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> + </dependency> </dependencies> <build> <plugins> diff --git a/searchlib/src/apps/vespa-attribute-inspect/vespa-attribute-inspect.cpp b/searchlib/src/apps/vespa-attribute-inspect/vespa-attribute-inspect.cpp index d9d6bb56152..5833f9cc904 100644 --- a/searchlib/src/apps/vespa-attribute-inspect/vespa-attribute-inspect.cpp +++ b/searchlib/src/apps/vespa-attribute-inspect/vespa-attribute-inspect.cpp @@ -112,11 +112,10 @@ LoadAttribute::main(int argc, char **argv) bool doApplyUpdate = false; bool doSave = false; bool doFastSearch = false; - bool doHuge = false; int opt; bool optError = false; - while ((opt = getopt(argc, argv, "pasf:h")) != -1) { + while ((opt = getopt(argc, argv, "pasf:")) != -1) { switch (opt) { case 'p': doPrintContent = true; @@ -124,9 +123,6 @@ LoadAttribute::main(int argc, char **argv) case 'a': doApplyUpdate = true; break; - case 'h': - doHuge = true; - break; case 'f': if (strcmp(optarg, "search") == 0) { doFastSearch = true; @@ -162,7 +158,6 @@ LoadAttribute::main(int argc, char **argv) attribute::CollectionType ct(fh.getTag("collectiontype").asString()); attribute::Config c(bt, ct); c.setFastSearch(doFastSearch); - c.setHuge(doHuge); AttributePtr ptr = AttributeFactory::createAttribute(fileName, c); vespalib::Timer timer; load(ptr); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java index 780d738eb2b..6d6cbe13b5b 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java @@ -264,12 +264,6 @@ public class RankingExpression implements Serializable { return serializedExpressions; } - @Deprecated - @SuppressWarnings("removal") - public Map<String, String> getRankProperties(List<ExpressionFunction> functions) { - return getRankProperties(new SerializationContext(functions)); - } - /** * Returns the rank-property name for a given expression name. * diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ConstantNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ConstantNode.java index a305c9c76af..453a7434469 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ConstantNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ConstantNode.java @@ -2,7 +2,6 @@ package com.yahoo.searchlib.rankingexpression.rule; import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.searchlib.rankingexpression.evaluation.BooleanValue; import com.yahoo.searchlib.rankingexpression.evaluation.Context; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.tensor.TensorType; @@ -20,20 +19,14 @@ public final class ConstantNode extends ExpressionNode { private final Value value; - public ConstantNode(Value value) { - value.freeze(); - this.value = value; - } - /** * Creates a constant value * * @param value the value. Ownership of this value is transferred to this. - * @param sourceImage the source string image producing this value */ - @Deprecated - public ConstantNode(Value value, String sourceImage) { - this(value); + public ConstantNode(Value value) { + value.freeze(); + this.value = value; } public Value getValue() { return value; } @@ -43,12 +36,6 @@ public final class ConstantNode extends ExpressionNode { return string.append(value.toString()); } - /** Returns the string which created this, or the value.toString() if not known */ - @Deprecated - public String sourceString() { - return value.toString(); - } - @Override public TensorType type(TypeContext<Reference> context) { return value.type(); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java index 287bc2655f5..96f0fa033d6 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.rankingexpression.rule; -import com.google.common.collect.ImmutableMap; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import java.util.Collection; @@ -48,12 +47,6 @@ public class FunctionReferenceContext { this.bindings.putAll(bindings); } - /** @deprecated Use {@link #FunctionReferenceContext(Map, Map)} instead */ - @Deprecated(forRemoval = true, since = "7") - protected FunctionReferenceContext(ImmutableMap<String, ExpressionFunction> functions, Map<String, String> bindings) { - this((Map<String, ExpressionFunction>)functions, bindings); - } - private static Map<String, ExpressionFunction> toMap(Collection<ExpressionFunction> list) { Map<String, ExpressionFunction> mapBuilder = new HashMap<>(); for (ExpressionFunction function : list) @@ -64,10 +57,6 @@ public class FunctionReferenceContext { /** Returns a function or null if it isn't defined in this context */ public ExpressionFunction getFunction(String name) { return functions.get(name); } - /** @deprecated Use {@link #getFunctions()} instead */ - @Deprecated(forRemoval = true, since = "7") - protected ImmutableMap<String, ExpressionFunction> functions() { return ImmutableMap.copyOf(functions); } - protected Map<String, ExpressionFunction> getFunctions() { return functions; } /** Returns the resolution of an identifier, or null if it isn't defined in this context */ diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java index d0a8e812091..5e5f7be7001 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.rankingexpression.rule; -import com.google.common.collect.ImmutableMap; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; @@ -90,13 +89,6 @@ public class SerializationContext extends FunctionReferenceContext { this.serializedFunctions = serializedFunctions; } - /** @deprecated Use {@link #SerializationContext(Map, Map, Optional, Map) instead}*/ - @Deprecated(forRemoval = true, since = "7") - public SerializationContext(ImmutableMap<String,ExpressionFunction> functions, Map<String, String> bindings, - Map<String, String> serializedFunctions) { - this((Map<String, ExpressionFunction>)functions, bindings, serializedFunctions); - } - /** Returns the type context of this, if it is able to resolve types. */ public Optional<TypeContext<Reference>> typeContext() { return typeContext; } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java index 7b68ad7e2af..1b72213d5ff 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java @@ -2,7 +2,6 @@ package com.yahoo.searchlib.rankingexpression.rule; import com.yahoo.api.annotations.Beta; -import com.google.common.collect.ImmutableMap; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.Context; @@ -342,13 +341,6 @@ public class TensorFunctionNode extends CompositeNode { return wrappedSerializationContext.typeContext(); } - /** @deprecated Use {@link #getFunctions()} instead */ - @SuppressWarnings("removal") - @Deprecated(forRemoval = true, since = "7") - protected ImmutableMap<String, ExpressionFunction> functions() { - return ImmutableMap.copyOf(wrappedSerializationContext.getFunctions()); - } - @Override protected Map<String, ExpressionFunction> getFunctions() { return wrappedSerializationContext.getFunctions(); } diff --git a/searchlib/src/tests/aggregator/attr_test.cpp b/searchlib/src/tests/aggregator/attr_test.cpp index 15a0f992701..b082e89ce19 100644 --- a/searchlib/src/tests/aggregator/attr_test.cpp +++ b/searchlib/src/tests/aggregator/attr_test.cpp @@ -124,12 +124,12 @@ TEST_F("testArrayAt", AttributeFixture()) { ExpressionTree et(MU<ArrayAtLookup>(*f1.guard, MU<ConstantNode>(MU<Int64ResultNode>(i)))); ExpressionTree::Configure treeConf; et.select(treeConf, treeConf); - EXPECT_TRUE(et.getResult().getClass().inherits(FloatResultNode::classId)); + EXPECT_TRUE(et.getResult()->getClass().inherits(FloatResultNode::classId)); EXPECT_TRUE(et.execute(0, HitRank(0.0))); - EXPECT_EQUAL(et.getResult().getFloat(), f1.doc0attr[i]); + EXPECT_EQUAL(et.getResult()->getFloat(), f1.doc0attr[i]); EXPECT_TRUE(et.execute(1, HitRank(0.0))); - EXPECT_EQUAL(et.getResult().getFloat(), f1.doc1attr[i]); + EXPECT_EQUAL(et.getResult()->getFloat(), f1.doc1attr[i]); } } @@ -142,12 +142,12 @@ TEST_F("testArrayAtInt", IntAttrFixture()) { ExpressionTree et(std::move(x)); ExpressionTree::Configure treeConf; et.select(treeConf, treeConf); - EXPECT_TRUE(et.getResult().getClass().inherits(IntegerResultNode::classId)); + EXPECT_TRUE(et.getResult()->getClass().inherits(IntegerResultNode::classId)); EXPECT_TRUE(et.execute(0, HitRank(0.0))); - EXPECT_EQUAL(et.getResult().getInteger(), f1.doc0attr[i]); + EXPECT_EQUAL(et.getResult()->getInteger(), f1.doc0attr[i]); EXPECT_TRUE(et.execute(1, HitRank(0.0))); - EXPECT_EQUAL(et.getResult().getInteger(), f1.doc1attr[i]); + EXPECT_EQUAL(et.getResult()->getInteger(), f1.doc1attr[i]); } } @@ -156,16 +156,16 @@ TEST_F("testArrayAtString", StringAttrFixture()) { ExpressionTree et(MU<ArrayAtLookup>(*f1.guard, MU<ConstantNode>(MU<Int64ResultNode>(1)))); ExpressionTree::Configure treeConf; et.select(treeConf, treeConf); - EXPECT_TRUE(et.getResult().getClass().inherits(StringResultNode::classId)); + EXPECT_TRUE(et.getResult()->getClass().inherits(StringResultNode::classId)); char mem[64]; ResultNode::BufferRef buf(&mem, sizeof(mem)); EXPECT_TRUE(et.execute(0, HitRank(0.0))); - EXPECT_EQUAL(et.getResult().getString(buf).c_str(), std::string("333")); + EXPECT_EQUAL(et.getResult()->getString(buf).c_str(), std::string("333")); EXPECT_TRUE(et.execute(1, HitRank(0.0))); - EXPECT_EQUAL(et.getResult().getString(buf).c_str(), std::string("4444")); + EXPECT_EQUAL(et.getResult()->getString(buf).c_str(), std::string("4444")); } struct ArrayAtExpressionFixture : @@ -184,21 +184,21 @@ struct ArrayAtExpressionFixture : TEST_F("testArrayAtBelowRange", ArrayAtExpressionFixture(-1)) { - EXPECT_TRUE(f1.et.getResult().getClass().inherits(FloatResultNode::classId)); + EXPECT_TRUE(f1.et.getResult()->getClass().inherits(FloatResultNode::classId)); EXPECT_TRUE(f1.et.execute(0, HitRank(0.0))); - EXPECT_EQUAL(f1.et.getResult().getFloat(), f1.doc0attr[0]); + EXPECT_EQUAL(f1.et.getResult()->getFloat(), f1.doc0attr[0]); EXPECT_TRUE(f1.et.execute(1, HitRank(0.0))); - EXPECT_EQUAL(f1.et.getResult().getFloat(), f1.doc1attr[0]); + EXPECT_EQUAL(f1.et.getResult()->getFloat(), f1.doc1attr[0]); } TEST_F("testArrayAtAboveRange", ArrayAtExpressionFixture(17)) { - EXPECT_TRUE(f1.et.getResult().getClass().inherits(FloatResultNode::classId)); + EXPECT_TRUE(f1.et.getResult()->getClass().inherits(FloatResultNode::classId)); EXPECT_TRUE(f1.et.execute(0, HitRank(0.0))); - EXPECT_EQUAL(f1.et.getResult().getFloat(), f1.doc0attr[10]); + EXPECT_EQUAL(f1.et.getResult()->getFloat(), f1.doc0attr[10]); EXPECT_TRUE(f1.et.execute(1, HitRank(0.0))); - EXPECT_EQUAL(f1.et.getResult().getFloat(), f1.doc1attr[10]); + EXPECT_EQUAL(f1.et.getResult()->getFloat(), f1.doc1attr[10]); } TEST_F("testInterpolatedLookup", AttributeFixture()) { @@ -206,13 +206,13 @@ TEST_F("testInterpolatedLookup", AttributeFixture()) { ExpressionTree::Configure treeConf; et.select(treeConf, treeConf); - EXPECT_TRUE(et.getResult().getClass().inherits(FloatResultNode::classId)); + EXPECT_TRUE(et.getResult()->getClass().inherits(FloatResultNode::classId)); EXPECT_TRUE(et.execute(0, HitRank(0.0))); - EXPECT_EQUAL(et.getResult().getFloat(), 2.0); + EXPECT_EQUAL(et.getResult()->getFloat(), 2.0); EXPECT_TRUE(et.execute(1, HitRank(0.0))); - EXPECT_EQUAL(et.getResult().getFloat(), 2.053082175617388); + EXPECT_EQUAL(et.getResult()->getFloat(), 2.053082175617388); } TEST_F("testWithRelevance", AttributeFixture()) { @@ -220,7 +220,7 @@ TEST_F("testWithRelevance", AttributeFixture()) { ExpressionTree::Configure treeConf; et.select(treeConf, treeConf); - EXPECT_TRUE(et.getResult().getClass().inherits(FloatResultNode::classId)); + EXPECT_TRUE(et.getResult()->getClass().inherits(FloatResultNode::classId)); // docid 0 double expect0[] = { 0.0, 0.0, 0.0, @@ -240,30 +240,30 @@ TEST_F("testWithRelevance", AttributeFixture()) { r *= 0.1; TEST_STATE(vespalib::make_string("i=%d", i).c_str()); EXPECT_TRUE(et.execute(0, HitRank(r))); - EXPECT_EQUAL(expect0[i], et.getResult().getFloat()); + EXPECT_EQUAL(expect0[i], et.getResult()->getFloat()); } EXPECT_TRUE(et.execute(0, HitRank(f1.doc0attr[2]))); - EXPECT_EQUAL(et.getResult().getFloat(), 2.0); + EXPECT_EQUAL(et.getResult()->getFloat(), 2.0); // docid 1 EXPECT_TRUE(et.execute(1, HitRank(f1.doc1attr[0] - 0.001))); - EXPECT_EQUAL(et.getResult().getFloat(), 0.0); + EXPECT_EQUAL(et.getResult()->getFloat(), 0.0); EXPECT_TRUE(et.execute(1, HitRank(f1.doc1attr[0]))); - EXPECT_EQUAL(et.getResult().getFloat(), 0.0); + EXPECT_EQUAL(et.getResult()->getFloat(), 0.0); EXPECT_TRUE(et.execute(1, HitRank(f1.doc1attr[2]))); - EXPECT_EQUAL(et.getResult().getFloat(), 2.0); + EXPECT_EQUAL(et.getResult()->getFloat(), 2.0); EXPECT_TRUE(et.execute(1, HitRank(f1.doc1attr[4]))); - EXPECT_EQUAL(et.getResult().getFloat(), 4.0); + EXPECT_EQUAL(et.getResult()->getFloat(), 4.0); EXPECT_TRUE(et.execute(1, HitRank(f1.doc1attr[10]))); - EXPECT_EQUAL(et.getResult().getFloat(), 10.0); + EXPECT_EQUAL(et.getResult()->getFloat(), 10.0); EXPECT_TRUE(et.execute(1, HitRank(f1.doc1attr[10] + 0.01))); - EXPECT_EQUAL(et.getResult().getFloat(), 10.0); + EXPECT_EQUAL(et.getResult()->getFloat(), 10.0); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/aggregator/perdocexpr.cpp b/searchlib/src/tests/aggregator/perdocexpr.cpp index 7bb10433f31..f56d61eabec 100644 --- a/searchlib/src/tests/aggregator/perdocexpr.cpp +++ b/searchlib/src/tests/aggregator/perdocexpr.cpp @@ -45,12 +45,12 @@ void testMin(const ResultNode & a, const ResultNode & b) { MinFunctionNode func; func.appendArg(MU<ConstantNode>(ResultNode::UP(a.clone()))) .appendArg(MU<ConstantNode>(ResultNode::UP(b.clone()))).prepare(false).execute(); - ASSERT_TRUE(func.getResult().cmp(a) == 0); + ASSERT_TRUE(func.getResult()->cmp(a) == 0); MinFunctionNode funcR; funcR.appendArg(MU<ConstantNode>(ResultNode::UP(b.clone()))) .appendArg(MU<ConstantNode>(ResultNode::UP(a.clone()))).prepare(false).execute(); - ASSERT_TRUE(funcR.getResult().cmp(a) == 0); + ASSERT_TRUE(funcR.getResult()->cmp(a) == 0); } ExpressionNode::UP @@ -87,13 +87,13 @@ void testMax(const ResultNode & a, const ResultNode & b) { func.appendArg(MU<ConstantNode>(ResultNode::UP(a.clone()))) .appendArg(MU<ConstantNode>(ResultNode::UP(b.clone()))).prepare(false) .execute(); - ASSERT_TRUE(func.getResult().cmp(b) == 0); + ASSERT_TRUE(func.getResult()->cmp(b) == 0); MaxFunctionNode funcR; funcR.appendArg(MU<ConstantNode>(ResultNode::UP(a.clone()))) .appendArg(MU<ConstantNode>(ResultNode::UP(b.clone()))).prepare(false) .execute(); - ASSERT_TRUE(funcR.getResult().cmp(b) == 0); + ASSERT_TRUE(funcR.getResult()->cmp(b) == 0); } TEST("testMax") { @@ -255,9 +255,9 @@ void testAdd(const ResultNode &a, const ResultNode &b, const ResultNode &c) { AddFunctionNode func; func.appendArg(MU<ConstantNode>(ResultNode::UP(a.clone()))) .appendArg(MU<ConstantNode>(ResultNode::UP(b.clone()))).prepare(false).execute(); - EXPECT_EQUAL(func.getResult().asString(), c.asString()); - EXPECT_EQUAL(func.getResult().cmp(c), 0); - EXPECT_EQUAL(c.cmp(func.getResult()), 0); + EXPECT_EQUAL(func.getResult()->asString(), c.asString()); + EXPECT_EQUAL(func.getResult()->cmp(c), 0); + EXPECT_EQUAL(c.cmp(*func.getResult()), 0); } TEST("testAdd") { @@ -271,10 +271,10 @@ void testDivide(const ResultNode &a, const ResultNode &b, const ResultNode &c) { DivideFunctionNode func; func.appendArg(MU<ConstantNode>(ResultNode::UP(a.clone()))) .appendArg(MU<ConstantNode>(ResultNode::UP(b.clone()))).prepare(false).execute(); - EXPECT_EQUAL(func.getResult().asString(), c.asString()); - EXPECT_EQUAL(func.getResult().getFloat(), c.getFloat()); - EXPECT_EQUAL(func.getResult().cmp(c), 0); - EXPECT_EQUAL(c.cmp(func.getResult()), 0); + EXPECT_EQUAL(func.getResult()->asString(), c.asString()); + EXPECT_EQUAL(func.getResult()->getFloat(), c.getFloat()); + EXPECT_EQUAL(func.getResult()->cmp(c), 0); + EXPECT_EQUAL(c.cmp(*func.getResult()), 0); } TEST("testDivide") { @@ -287,10 +287,10 @@ void testModulo(const ResultNode &a, const ResultNode &b, const ResultNode &c) { ModuloFunctionNode func; func.appendArg(MU<ConstantNode>(ResultNode::UP(a.clone()))) .appendArg(MU<ConstantNode>(ResultNode::UP(b.clone()))).prepare(false).execute(); - EXPECT_EQUAL(func.getResult().asString(), c.asString()); - EXPECT_EQUAL(func.getResult().getFloat(), c.getFloat()); - EXPECT_EQUAL(func.getResult().cmp(c), 0); - EXPECT_EQUAL(c.cmp(func.getResult()), 0); + EXPECT_EQUAL(func.getResult()->asString(), c.asString()); + EXPECT_EQUAL(func.getResult()->getFloat(), c.getFloat()); + EXPECT_EQUAL(func.getResult()->cmp(c), 0); + EXPECT_EQUAL(c.cmp(*func.getResult()), 0); } TEST("testModulo") { @@ -312,9 +312,9 @@ TEST("testModulo") { void testNegate(const ResultNode & a, const ResultNode & b) { NegateFunctionNode func; func.appendArg(MU<ConstantNode>(ResultNode::UP(a.clone()))).prepare(false).execute(); - EXPECT_EQUAL(func.getResult().asString(), b.asString()); - EXPECT_EQUAL(func.getResult().cmp(b), 0); - EXPECT_EQUAL(b.cmp(func.getResult()), 0); + EXPECT_EQUAL(func.getResult()->asString(), b.asString()); + EXPECT_EQUAL(func.getResult()->cmp(b), 0); + EXPECT_EQUAL(b.cmp(*func.getResult()), 0); } TEST("testNegate") { @@ -664,7 +664,7 @@ TEST("testMailChecksumExpression") { cfn.prepare(false); cfn.execute(); - ConstBufferRef ref = static_cast<const RawResultNode &>(cfn.getResult()).get(); + ConstBufferRef ref = static_cast<const RawResultNode &>(*cfn.getResult()).get(); std::string cmp = getVespaChecksumV2(ymumid, folder, flags); @@ -682,7 +682,7 @@ TEST("testMailChecksumExpression") { node.execute(); ConstBufferRef ref2 = - static_cast<const RawResultNode &>(node.getResult()).get(); + static_cast<const RawResultNode &>(*node.getResult()).get(); for (uint32_t i = 0; i < ref2.size(); i++) { std::cerr << i << ": " << (int)ref2.c_str()[i] << "\n"; @@ -700,7 +700,7 @@ TEST("testDebugFunction") { vespalib::Timer timer; n.execute(); EXPECT_TRUE(timer.elapsed() > 1s); - EXPECT_EQUAL(static_cast<const Int64ResultNode &>(n.getResult()).get(), 7); + EXPECT_EQUAL(static_cast<const Int64ResultNode &>(*n.getResult()).get(), 7); } { std::unique_ptr<AddFunctionNode> add = MU<AddFunctionNode>(); @@ -712,7 +712,7 @@ TEST("testDebugFunction") { vespalib::Timer timer; n.execute(); EXPECT_TRUE(timer.elapsed() > 1s); - EXPECT_EQUAL(static_cast<const Int64ResultNode &>(n.getResult()).get(), 7); + EXPECT_EQUAL(static_cast<const Int64ResultNode &>(*n.getResult()).get(), 7); } } @@ -732,40 +732,40 @@ TEST("testDivExpressions") { StrLenFunctionNode e(MU<ConstantNode>(MU<Int64ResultNode>(238686))); e.prepare(false); e.execute(); - EXPECT_EQUAL(static_cast<const Int64ResultNode &>(e.getResult()).get(), 6); + EXPECT_EQUAL(static_cast<const Int64ResultNode &>(*e.getResult()).get(), 6); } { NormalizeSubjectFunctionNode e(MU<ConstantNode>(MU<StringResultNode>("Re: Your mail"))); e.prepare(false); e.execute(); - EXPECT_EQUAL(static_cast<const StringResultNode &>(e.getResult()).get(), "Your mail"); + EXPECT_EQUAL(static_cast<const StringResultNode &>(*e.getResult()).get(), "Your mail"); } { NormalizeSubjectFunctionNode e(MU<ConstantNode>(MU<StringResultNode>("Your mail"))); e.prepare(false); e.execute(); - EXPECT_EQUAL(static_cast<const StringResultNode &>(e.getResult()).get(), "Your mail"); + EXPECT_EQUAL(static_cast<const StringResultNode &>(*e.getResult()).get(), "Your mail"); } { StrCatFunctionNode e(MU<ConstantNode>(MU<Int64ResultNode>(238686))); e.appendArg(MU<ConstantNode>(MU<StringResultNode>("ARG 2"))); e.prepare(false); e.execute(); - EXPECT_EQUAL(static_cast<const StringResultNode &>(e.getResult()).get(), "238686ARG 2"); + EXPECT_EQUAL(static_cast<const StringResultNode &>(*e.getResult()).get(), "238686ARG 2"); } { ToStringFunctionNode e(MU<ConstantNode>(MU<Int64ResultNode>(238686))); e.prepare(false); e.execute(); - EXPECT_EQUAL(strcmp(static_cast<const StringResultNode &>(e.getResult()).get().c_str(), "238686"), 0); + EXPECT_EQUAL(strcmp(static_cast<const StringResultNode &>(*e.getResult()).get().c_str(), "238686"), 0); } { ToRawFunctionNode e(MU<ConstantNode>(MU<Int64ResultNode>(238686))); e.prepare(false); e.execute(); - auto raw_result = static_cast<const RawResultNode &>(e.getResult()).get(); + auto raw_result = static_cast<const RawResultNode &>(*e.getResult()).get(); EXPECT_EQUAL(6u, raw_result.size()); EXPECT_EQUAL(strncmp(raw_result.c_str(), "238686", 6u), 0); } @@ -774,21 +774,21 @@ TEST("testDivExpressions") { CatFunctionNode e(MU<ConstantNode>(MU<Int64ResultNode>(238686))); e.prepare(false); e.execute(); - EXPECT_EQUAL(static_cast<const RawResultNode &>(e.getResult()).get().size(), 8u); + EXPECT_EQUAL(static_cast<const RawResultNode &>(*e.getResult()).get().size(), 8u); } { CatFunctionNode e(MU<ConstantNode>(MU<Int32ResultNode>(23886))); e.prepare(false); e.execute(); - EXPECT_EQUAL(static_cast<const RawResultNode &>(e.getResult()).get().size(), 4u); + EXPECT_EQUAL(static_cast<const RawResultNode &>(*e.getResult()).get().size(), 4u); } { const uint8_t buf[4] = { 0, 0, 0, 7 }; MD5BitFunctionNode e(MU<ConstantNode>(MU<RawResultNode>(buf, sizeof(buf))), 16*8); e.prepare(false); e.execute(); - ASSERT_TRUE(e.getResult().getClass().inherits(RawResultNode::classId)); - const RawResultNode &r(static_cast<const RawResultNode &>(e.getResult())); + ASSERT_TRUE(e.getResult()->getClass().inherits(RawResultNode::classId)); + const RawResultNode &r(static_cast<const RawResultNode &>(*e.getResult())); EXPECT_EQUAL(r.get().size(), 16u); } { @@ -796,24 +796,24 @@ TEST("testDivExpressions") { MD5BitFunctionNode e(MU<ConstantNode>(MU<RawResultNode>(buf, sizeof(buf))), 2*8); e.prepare(false); e.execute(); - EXPECT_EQUAL(static_cast<const RawResultNode &>(e.getResult()).get().size(), 2u); + EXPECT_EQUAL(static_cast<const RawResultNode &>(*e.getResult()).get().size(), 2u); } { const uint8_t buf[4] = { 0, 0, 0, 7 }; XorBitFunctionNode e(MU<ConstantNode>(MU<RawResultNode>(buf, sizeof(buf))), 1*8); e.prepare(false); e.execute(); - EXPECT_EQUAL(static_cast<const RawResultNode &>(e.getResult()).get().size(), 1u); - EXPECT_EQUAL(static_cast<const RawResultNode &>(e.getResult()).get().c_str()[0], 0x7); + EXPECT_EQUAL(static_cast<const RawResultNode &>(*e.getResult()).get().size(), 1u); + EXPECT_EQUAL(static_cast<const RawResultNode &>(*e.getResult()).get().c_str()[0], 0x7); } { const uint8_t buf[4] = { 6, 0, 7, 7 }; XorBitFunctionNode e(MU<ConstantNode>(MU<RawResultNode>(buf, sizeof(buf))), 2*8); e.prepare(false); e.execute(); - EXPECT_EQUAL(static_cast<const RawResultNode &>(e.getResult()).get().size(), 2u); - EXPECT_EQUAL((int)static_cast<const RawResultNode &>(e.getResult()).get().c_str()[0], 0x1); - EXPECT_EQUAL((int)static_cast<const RawResultNode &>(e.getResult()).get().c_str()[1], 0x7); + EXPECT_EQUAL(static_cast<const RawResultNode &>(*e.getResult()).get().size(), 2u); + EXPECT_EQUAL((int)static_cast<const RawResultNode &>(*e.getResult()).get().c_str()[0], 0x1); + EXPECT_EQUAL((int)static_cast<const RawResultNode &>(*e.getResult()).get().c_str()[1], 0x7); } { const uint8_t wantedBuf[14] = @@ -831,8 +831,8 @@ TEST("testDivExpressions") { MD5BitFunctionNode e(MU<ConstantNode>(MU<RawResultNode>(wantedBuf, sizeof(wantedBuf))), 16*8); e.prepare(false); e.execute(); - ASSERT_TRUE(e.getResult().getClass().inherits(RawResultNode::classId)); - const RawResultNode &r(static_cast<const RawResultNode &>(e.getResult())); + ASSERT_TRUE(e.getResult()->getClass().inherits(RawResultNode::classId)); + const RawResultNode &r(static_cast<const RawResultNode &>(*e.getResult())); EXPECT_EQUAL(r.get().size(), 16u); uint8_t md5[16]; fastc_md5sum(currentBuf, sizeof(currentBuf), md5); @@ -849,7 +849,7 @@ TEST("testDivExpressions") { MD5BitFunctionNode finalCheck(std::move(cat), 32); finalCheck.prepare(false); finalCheck.execute(); - const RawResultNode &rr(static_cast<const RawResultNode &>(finalCheck.getResult())); + const RawResultNode &rr(static_cast<const RawResultNode &>(*finalCheck.getResult())); EXPECT_EQUAL(rr.get().size(), 4u); fastc_md5sum(wantedBuf, sizeof(wantedBuf), md5); EXPECT_TRUE(memcmp(md5facit, md5, sizeof(md5)) == 0); @@ -859,37 +859,37 @@ TEST("testDivExpressions") { CatFunctionNode e(MU<ConstantNode>(MU<Int16ResultNode>(23886))); e.prepare(false); e.execute(); - EXPECT_EQUAL(static_cast<const RawResultNode &>(e.getResult()).get().size(), 2u); + EXPECT_EQUAL(static_cast<const RawResultNode &>(*e.getResult()).get().size(), 2u); } { CatFunctionNode e(MU<ConstantNode>(createIntRV<Int8ResultNodeVector>({86, 14}))); e.prepare(false); e.execute(); - EXPECT_EQUAL(static_cast<const RawResultNode &>(e.getResult()).get().size(), 1*2u); + EXPECT_EQUAL(static_cast<const RawResultNode &>(*e.getResult()).get().size(), 1*2u); } { CatFunctionNode e(MU<ConstantNode>(createIntRV<Int32ResultNodeVector>({238686,2133214}))); e.prepare(false); e.execute(); - EXPECT_EQUAL(static_cast<const RawResultNode &>(e.getResult()).get().size(), 4*2u); + EXPECT_EQUAL(static_cast<const RawResultNode &>(*e.getResult()).get().size(), 4*2u); } { NumElemFunctionNode e(MU<ConstantNode>(MU<Int64ResultNode>(238686))); e.prepare(false); e.execute(); - EXPECT_EQUAL(e.getResult().getInteger(), 1); + EXPECT_EQUAL(e.getResult()->getInteger(), 1); } { NumElemFunctionNode e(MU<ConstantNode>(createIntRV<Int32ResultNodeVector>({238686,2133214}))); e.prepare(false); e.execute(); - EXPECT_EQUAL(e.getResult().getInteger(), 2); + EXPECT_EQUAL(e.getResult()->getInteger(), 2); } { NumElemFunctionNode e(MU<ConstantNode>(createIntRV<Int32ResultNodeVector>({238686,2133214}))); e.prepare(false); e.execute(); - EXPECT_EQUAL(e.getResult().getInteger(), 2); + EXPECT_EQUAL(e.getResult()->getInteger(), 2); } } @@ -900,10 +900,10 @@ bool test1MultivalueExpression(const MultiArgFunctionNode &exprConst, Expression expr.appendArg(std::move(mv)); expr.prepare(false); bool ok = EXPECT_TRUE(expr.execute()) && - EXPECT_EQUAL(0, expr.getResult().cmp(expected)); + EXPECT_EQUAL(0, expr.getResult()->cmp(expected)); if (!ok) { std::cerr << "Expected:" << expected.asString() << std::endl - << "Got: " << expr.getResult().asString() << std::endl; + << "Got: " << expr.getResult()->asString() << std::endl; } return ok; } @@ -1030,36 +1030,36 @@ TEST("testArithmeticNodes") { ExpressionTree::Configure treeConf; et.select(treeConf, treeConf); - EXPECT_TRUE(et.getResult().getClass().inherits(IntegerResultNode::classId)); + EXPECT_TRUE(et.getResult()->getClass().inherits(IntegerResultNode::classId)); EXPECT_TRUE(et.ExpressionNode::execute()); - EXPECT_EQUAL(et.getResult().getInteger(), 3); + EXPECT_EQUAL(et.getResult()->getInteger(), 3); EXPECT_TRUE(et.ExpressionNode::execute()); - EXPECT_EQUAL(et.getResult().getInteger(), 3); + EXPECT_EQUAL(et.getResult()->getInteger(), 3); AddFunctionNode add2; add2.appendArg(createScalarInt(I1)); add2.appendArg(createScalarFloat(F2)); add2.prepare(false); - EXPECT_TRUE(add2.getResult().getClass().inherits(FloatResultNode::classId)); + EXPECT_TRUE(add2.getResult()->getClass().inherits(FloatResultNode::classId)); AddFunctionNode add3; add3.appendArg(createScalarInt(I1)); add3.appendArg(createScalarString(S2)); add3.prepare(false); - EXPECT_TRUE(add3.getResult().getClass().inherits(IntegerResultNode::classId)); + EXPECT_TRUE(add3.getResult()->getClass().inherits(IntegerResultNode::classId)); AddFunctionNode add4; add4.appendArg(createScalarInt(I1)); add4.appendArg(createScalarRaw(S2)); add4.prepare(false); - EXPECT_TRUE(add4.getResult().getClass().inherits(IntegerResultNode::classId)); + EXPECT_TRUE(add4.getResult()->getClass().inherits(IntegerResultNode::classId)); AddFunctionNode add5; add5.appendArg(createScalarInt(I1)); add5.appendArg(MU<AttributeNode>(*attr1)); add5.prepare(false); - EXPECT_TRUE(add5.getResult().getClass().inherits(IntegerResultNode::classId)); + EXPECT_TRUE(add5.getResult()->getClass().inherits(IntegerResultNode::classId)); AddFunctionNode add6; add6.appendArg(createScalarFloat(F1)); add6.appendArg(MU<AttributeNode>(*attr1)); add6.prepare(false); - EXPECT_TRUE(add6.getResult().getClass().inherits(FloatResultNode::classId)); + EXPECT_TRUE(add6.getResult()->getClass().inherits(FloatResultNode::classId)); } void testArith(MultiArgFunctionNode &op, ExpressionNode::UP arg1, ExpressionNode::UP arg2, @@ -1069,9 +1069,9 @@ void testArith(MultiArgFunctionNode &op, ExpressionNode::UP arg1, ExpressionNode op.appendArg(std::move(arg2)); op.prepare(false); op.execute(); - EXPECT_EQUAL(intResult, op.getResult().getInteger()); - ASSERT_TRUE(intResult == op.getResult().getInteger()); - EXPECT_EQUAL(floatResult, op.getResult().getFloat()); + EXPECT_EQUAL(intResult, op.getResult()->getInteger()); + ASSERT_TRUE(intResult == op.getResult()->getInteger()); + EXPECT_EQUAL(floatResult, op.getResult()->getFloat()); } void testAdd(ExpressionNode::UP arg1, ExpressionNode::UP arg2, @@ -1120,89 +1120,89 @@ void testArithmeticArguments(NumericFunctionNode &function, } function.appendArg(createScalarInt(arg1[0])).appendArg(createScalarInt(arg2[0])); function.prepare(false); - EXPECT_TRUE(function.getResult().getClass().equal(Int64ResultNode::classId)); + EXPECT_TRUE(function.getResult()->getClass().equal(Int64ResultNode::classId)); EXPECT_TRUE(function.execute()); - EXPECT_EQUAL(function.getResult().getInteger(), static_cast<int64_t>(result[0])); + EXPECT_EQUAL(function.getResult()->getInteger(), static_cast<int64_t>(result[0])); function.reset(); function.appendArg(createScalarInt(arg1[0])).appendArg(createScalarFloat(arg2[0])); function.prepare(false); - EXPECT_TRUE(function.getResult().getClass().equal(FloatResultNode::classId)); + EXPECT_TRUE(function.getResult()->getClass().equal(FloatResultNode::classId)); EXPECT_TRUE(function.execute()); - EXPECT_EQUAL(function.getResult().getFloat(), result[0]); + EXPECT_EQUAL(function.getResult()->getFloat(), result[0]); function.reset(); function.appendArg(createScalarFloat(arg1[0])).appendArg(createScalarInt(arg2[0])); function.prepare(false); - EXPECT_TRUE(function.getResult().getClass().equal(FloatResultNode::classId)); + EXPECT_TRUE(function.getResult()->getClass().equal(FloatResultNode::classId)); EXPECT_TRUE(function.execute()); - EXPECT_EQUAL(function.getResult().getFloat(), result[0]); + EXPECT_EQUAL(function.getResult()->getFloat(), result[0]); function.reset(); function.appendArg(createScalarFloat(arg1[0])).appendArg(createScalarFloat(arg2[0])); function.prepare(false); - EXPECT_TRUE(function.getResult().getClass().equal(FloatResultNode::classId)); + EXPECT_TRUE(function.getResult()->getClass().equal(FloatResultNode::classId)); EXPECT_TRUE(function.execute()); - EXPECT_EQUAL(function.getResult().getFloat(), result[0]); + EXPECT_EQUAL(function.getResult()->getFloat(), result[0]); function.reset(); function.appendArg(createVectorInt(arg1)); function.prepare(false); - EXPECT_TRUE(function.getResult().getClass().equal(Int64ResultNode::classId)); + EXPECT_TRUE(function.getResult()->getClass().equal(Int64ResultNode::classId)); EXPECT_TRUE(function.execute()); - EXPECT_EQUAL(function.getResult().getInteger(), static_cast<int64_t>(flattenResult)); + EXPECT_EQUAL(function.getResult()->getInteger(), static_cast<int64_t>(flattenResult)); function.reset(); function.appendArg(createVectorFloat(arg1)); function.prepare(false); - EXPECT_TRUE(function.getResult().getClass().equal(FloatResultNode::classId)); + EXPECT_TRUE(function.getResult()->getClass().equal(FloatResultNode::classId)); EXPECT_TRUE(function.execute()); - EXPECT_EQUAL(function.getResult().getFloat(), flattenResult); + EXPECT_EQUAL(function.getResult()->getFloat(), flattenResult); function.reset(); function.appendArg(createVectorInt(arg1)).appendArg(createVectorInt(arg2)); function.prepare(false); - EXPECT_TRUE(function.getResult().getClass().equal(IntegerResultNodeVector::classId)); + EXPECT_TRUE(function.getResult()->getClass().equal(IntegerResultNodeVector::classId)); EXPECT_TRUE(function.execute()); - EXPECT_TRUE(function.getResult().getClass().equal(IntegerResultNodeVector::classId)); - EXPECT_EQUAL(static_cast<const IntegerResultNodeVector &>(function.getResult()).size(), 7u); - EXPECT_EQUAL(0, function.getResult().cmp(ir)); + EXPECT_TRUE(function.getResult()->getClass().equal(IntegerResultNodeVector::classId)); + EXPECT_EQUAL(static_cast<const IntegerResultNodeVector &>(*function.getResult()).size(), 7u); + EXPECT_EQUAL(0, function.getResult()->cmp(ir)); function.reset(); function.appendArg(createVectorFloat(arg1)).appendArg(createVectorFloat(arg2)); function.prepare(false); - EXPECT_TRUE(function.getResult().getClass().equal(FloatResultNodeVector::classId)); + EXPECT_TRUE(function.getResult()->getClass().equal(FloatResultNodeVector::classId)); EXPECT_TRUE(function.execute()); - EXPECT_TRUE(function.getResult().getClass().equal(FloatResultNodeVector::classId)); - EXPECT_EQUAL(static_cast<const FloatResultNodeVector &>(function.getResult()).size(), 7u); - EXPECT_EQUAL(0, function.getResult().cmp(fr)); + EXPECT_TRUE(function.getResult()->getClass().equal(FloatResultNodeVector::classId)); + EXPECT_EQUAL(static_cast<const FloatResultNodeVector &>(*function.getResult()).size(), 7u); + EXPECT_EQUAL(0, function.getResult()->cmp(fr)); function.reset(); function.appendArg(createVectorInt(arg1)).appendArg(createVectorFloat(arg2)); function.prepare(false); - EXPECT_TRUE(function.getResult().getClass().equal(FloatResultNodeVector::classId)); + EXPECT_TRUE(function.getResult()->getClass().equal(FloatResultNodeVector::classId)); EXPECT_TRUE(function.execute()); - EXPECT_TRUE(function.getResult().getClass().equal(FloatResultNodeVector::classId)); - EXPECT_EQUAL(static_cast<const FloatResultNodeVector &>(function.getResult()).size(), 7u); - EXPECT_EQUAL(0, function.getResult().cmp(fr)); + EXPECT_TRUE(function.getResult()->getClass().equal(FloatResultNodeVector::classId)); + EXPECT_EQUAL(static_cast<const FloatResultNodeVector &>(*function.getResult()).size(), 7u); + EXPECT_EQUAL(0, function.getResult()->cmp(fr)); function.reset(); function.appendArg(createVectorFloat(arg1)).appendArg(createVectorInt(arg2)); function.prepare(false); - EXPECT_TRUE(function.getResult().getClass().equal(FloatResultNodeVector::classId)); + EXPECT_TRUE(function.getResult()->getClass().equal(FloatResultNodeVector::classId)); EXPECT_TRUE(function.execute()); - EXPECT_TRUE(function.getResult().getClass().equal(FloatResultNodeVector::classId)); - EXPECT_EQUAL(static_cast<const FloatResultNodeVector &>(function.getResult()).size(), 7u); - EXPECT_EQUAL(0, function.getResult().cmp(fr)); + EXPECT_TRUE(function.getResult()->getClass().equal(FloatResultNodeVector::classId)); + EXPECT_EQUAL(static_cast<const FloatResultNodeVector &>(*function.getResult()).size(), 7u); + EXPECT_EQUAL(0, function.getResult()->cmp(fr)); } TEST("testArithmeticOperations") { @@ -1628,59 +1628,59 @@ AttributeGuard createBoolAttribute() { TEST("testIntegerTypes") { EXPECT_EQUAL(AttributeNode(*createBoolAttribute()).prepare(false) - .getResult().getClass().id(), + .getResult()->getClass().id(), uint32_t(BoolResultNode::classId)); EXPECT_EQUAL(AttributeNode(*createBoolAttribute()) - .prepare(true).getResult().getClass().id(), + .prepare(true).getResult()->getClass().id(), uint32_t(BoolResultNode::classId)); EXPECT_EQUAL(AttributeNode(*createInt8Attribute()).prepare(false) - .getResult().getClass().id(), + .getResult()->getClass().id(), uint32_t(Int64ResultNode::classId)); EXPECT_EQUAL(AttributeNode(*createInt8Attribute()) - .prepare(true).getResult().getClass().id(), + .prepare(true).getResult()->getClass().id(), uint32_t(Int8ResultNode::classId)); EXPECT_EQUAL(AttributeNode(*createInt16Attribute()) - .prepare(false).getResult().getClass().id(), + .prepare(false).getResult()->getClass().id(), uint32_t(Int64ResultNode::classId)); EXPECT_EQUAL(AttributeNode(*createInt16Attribute()) - .prepare(true).getResult().getClass().id(), + .prepare(true).getResult()->getClass().id(), uint32_t(Int16ResultNode::classId)); EXPECT_EQUAL(AttributeNode(*createInt32Attribute()) - .prepare(false).getResult().getClass().id(), + .prepare(false).getResult()->getClass().id(), uint32_t(Int64ResultNode::classId)); EXPECT_EQUAL(AttributeNode(*createInt32Attribute()) - .prepare(true).getResult().getClass().id(), + .prepare(true).getResult()->getClass().id(), uint32_t(Int32ResultNode::classId)); EXPECT_EQUAL(AttributeNode(*createInt64Attribute()) - .prepare(false).getResult().getClass().id(), + .prepare(false).getResult()->getClass().id(), uint32_t(Int64ResultNode::classId)); EXPECT_EQUAL(AttributeNode(*createInt64Attribute()) - .prepare(true).getResult().getClass().id(), + .prepare(true).getResult()->getClass().id(), uint32_t(Int64ResultNode::classId)); EXPECT_EQUAL(AttributeNode(*AttributeGuard(AttributeVector::SP(new MultiInt8ExtAttribute("test")))) - .prepare(false).getResult().getClass().id(), + .prepare(false).getResult()->getClass().id(), uint32_t(Int64ResultNodeVector::classId)); EXPECT_EQUAL(AttributeNode(*AttributeGuard(AttributeVector::SP(new MultiInt8ExtAttribute("test")))) - .prepare(true).getResult().getClass().id(), + .prepare(true).getResult()->getClass().id(), uint32_t(Int8ResultNodeVector::classId)); EXPECT_EQUAL(AttributeNode(*AttributeGuard(AttributeVector::SP(new MultiInt16ExtAttribute("test")))) - .prepare(false).getResult().getClass().id(), + .prepare(false).getResult()->getClass().id(), uint32_t(Int64ResultNodeVector::classId)); EXPECT_EQUAL(AttributeNode(*AttributeGuard(AttributeVector::SP(new MultiInt16ExtAttribute("test")))) - .prepare(true).getResult().getClass().id(), + .prepare(true).getResult()->getClass().id(), uint32_t(Int16ResultNodeVector::classId)); EXPECT_EQUAL(AttributeNode(*AttributeGuard(AttributeVector::SP(new MultiInt32ExtAttribute("test")))) - .prepare(false).getResult().getClass().id(), + .prepare(false).getResult()->getClass().id(), uint32_t(Int64ResultNodeVector::classId)); EXPECT_EQUAL(AttributeNode(*AttributeGuard(AttributeVector::SP(new MultiInt32ExtAttribute("test")))) - .prepare(true).getResult().getClass().id(), + .prepare(true).getResult()->getClass().id(), uint32_t(Int32ResultNodeVector::classId)); EXPECT_EQUAL(AttributeNode(*AttributeGuard(AttributeVector::SP(new MultiInt64ExtAttribute("test")))) - .prepare(false).getResult().getClass().id(), + .prepare(false).getResult()->getClass().id(), uint32_t(Int64ResultNodeVector::classId)); EXPECT_EQUAL(AttributeNode(*AttributeGuard(AttributeVector::SP(new MultiInt64ExtAttribute("test")))) - .prepare(true).getResult().getClass().id(), + .prepare(true).getResult()->getClass().id(), uint32_t(Int64ResultNodeVector::classId)); } diff --git a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp index 8a702769312..7f0a88c9f86 100644 --- a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp +++ b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp @@ -218,12 +218,6 @@ TEST("require that config can be converted") a.fastsearch = true; EXPECT_TRUE(CC::convert(a).fastSearch()); } - { // huge - CACA a; - EXPECT_TRUE(!CC::convert(a).huge()); - a.huge = true; - EXPECT_TRUE(CC::convert(a).huge()); - } { // fastAccess CACA a; EXPECT_TRUE(!CC::convert(a).fastAccess()); diff --git a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp index 059f198a070..896fd27f90e 100644 --- a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp +++ b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp @@ -251,7 +251,7 @@ Fixture::assertInts(std::vector<IAttributeVector::largeint_t> expVals, const ves ++docId; node->setDocId(docId); node->execute(); - const auto &result = node->getResult(); + const auto &result = *node->getResult(); if (preserveAccurateTypes) { ASSERT_TRUE(result.inherits(Int8ResultNode::classId)); } else { @@ -271,7 +271,7 @@ Fixture::assertBools(std::vector<bool> expVals, const vespalib::string &attribut ++docId; node->setDocId(docId); node->execute(); - const auto &result = node->getResult(); + const auto &result = *node->getResult(); ASSERT_TRUE(result.inherits(BoolResultNode::classId)); const BoolResultNode & bResult = static_cast<const BoolResultNode &>(result); @@ -290,7 +290,7 @@ Fixture::assertStrings(std::vector<vespalib::string> expVals, const vespalib::st ++docId; node->setDocId(docId); node->execute(); - const auto &result = node->getResult(); + const auto &result = *node->getResult(); if (useEnumOptimization) { ASSERT_TRUE(result.inherits(EnumResultNode::classId)); } else { @@ -310,7 +310,7 @@ Fixture::assertFloats(std::vector<double> expVals, const vespalib::string &attri ++docId; node->setDocId(docId); node->execute(); - const auto &result = node->getResult(); + const auto &result = *node->getResult(); ASSERT_TRUE(result.inherits(FloatResultNode::classId)); double docVal = result.getFloat(); EXPECT_EQUAL(std::isnan(expDocVal), std::isnan(docVal)); @@ -329,7 +329,7 @@ Fixture::assertIntArrays(std::vector<std::vector<IAttributeVector::largeint_t>> ++docId; node->setDocId(docId); node->execute(); - const auto &result = node->getResult(); + const auto &result = *node->getResult(); ASSERT_TRUE(result.inherits(ResultNodeVector::classId)); const auto &resultVector = static_cast<const ResultNodeVector &>(result); if (preserveAccurateTypes) { @@ -354,7 +354,7 @@ Fixture::assertStringArrays(std::vector<std::vector<vespalib::string>> expVals, ++docId; node->setDocId(docId); node->execute(); - const auto &result = node->getResult(); + const auto &result = *node->getResult(); ASSERT_TRUE(result.inherits(ResultNodeVector::classId)); const auto &resultVector = static_cast<const ResultNodeVector &>(result); if (useEnumOptimization) { @@ -379,7 +379,7 @@ Fixture::assertFloatArrays(std::vector<std::vector<double>> expVals, const vespa ++docId; node->setDocId(docId); node->execute(); - const auto &result = node->getResult(); + const auto &result = *node->getResult(); ASSERT_TRUE(result.inherits(ResultNodeVector::classId)); const auto &resultVector = static_cast<const ResultNodeVector &>(result); ASSERT_TRUE(result.inherits(FloatResultNodeVector::classId)); diff --git a/searchlib/src/tests/features/bm25/bm25_test.cpp b/searchlib/src/tests/features/bm25/bm25_test.cpp index d3c812198e1..3a9dfffa8d7 100644 --- a/searchlib/src/tests/features/bm25/bm25_test.cpp +++ b/searchlib/src/tests/features/bm25/bm25_test.cpp @@ -94,6 +94,12 @@ TEST_F(Bm25BlueprintTest, blueprint_can_prepare_shared_state_with_average_field_ EXPECT_DOUBLE_EQ(10, as_value<double>(*store.get("bm25.afl.is"))); } +TEST_F(Bm25BlueprintTest, dump_features_for_all_index_fields) +{ + FtTestApp::FT_DUMP(factory, "bm25", index_env, + StringList().add("bm25(is)").add("bm25(ia)").add("bm25(iws)")); +} + struct Scorer { double avg_field_length; diff --git a/searchlib/src/tests/features/prod_features.cpp b/searchlib/src/tests/features/prod_features.cpp index 2c04a326e3e..f146abe5556 100644 --- a/searchlib/src/tests/features/prod_features.cpp +++ b/searchlib/src/tests/features/prod_features.cpp @@ -483,8 +483,6 @@ Test::testCloseness() TEST_DO(assertCloseness(1, "pos", 0)); assertCloseness(0.8, "pos", 1802661); assertCloseness(0, "pos", 9013306); - // use non-existing attribute -> default distance - TEST_DO(assertCloseness(0, "no", 0)); // two-argument version TEST_DO(assertCloseness(0.8, "field,pos", 1802661)); @@ -893,7 +891,11 @@ Test::testDistance() StringList params, in, out; FT_SETUP_FAIL(pt, params); - FT_SETUP_OK(pt, params.add("pos"), in, + FtIndexEnvironment idx_env; + idx_env + .getBuilder() + .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::INT64, "pos"); + FT_SETUP_OK(pt, idx_env, params.add("pos"), in, out.add("out").add("index").add("latitude").add("longitude").add("km")); FT_DUMP_EMPTY(_factory, "distance"); } @@ -1473,7 +1475,7 @@ Test::testNow() FT_SETUP_OK (pt, params, in, out.add("out")); FT_SETUP_FAIL(pt, params.add("foo")); - FT_DUMP(_factory, "now", StringList().add("now")); + FT_DUMP_EMPTY(_factory, "now"); } { @@ -1703,7 +1705,7 @@ Test::testMatches() FT_SETUP_OK(pt, ft.getIndexEnv(), params.clear().add("bar"), in, out); FT_SETUP_OK(pt, ft.getIndexEnv(), params.add("1"), in, out); - FT_DUMP_EMPTY(_factory, "matches"); + FT_DUMP(_factory, "matches", ft.getIndexEnv(), StringList().add("matches(foo)").add("matches(bar)")); } { // Test executor for index fields EXPECT_TRUE(assertMatches(0, "x", "a", "matches(foo)")); diff --git a/searchlib/src/tests/searchcommon/attribute/config/attribute_config_test.cpp b/searchlib/src/tests/searchcommon/attribute/config/attribute_config_test.cpp index f9e77376b25..2183138811b 100644 --- a/searchlib/src/tests/searchcommon/attribute/config/attribute_config_test.cpp +++ b/searchlib/src/tests/searchcommon/attribute/config/attribute_config_test.cpp @@ -20,9 +20,8 @@ struct Fixture Fixture(BasicType bt, CollectionType ct = CollectionType::SINGLE, - bool fastSearch_ = false, - bool huge_ = false) - : _config(bt, ct, fastSearch_, huge_) + bool fastSearch_ = false) + : _config(bt, ct, fastSearch_) { } }; @@ -32,7 +31,6 @@ TEST_F("test default attribute config", Fixture) EXPECT_EQUAL(CollectionType::Type::SINGLE, f._config.collectionType().type()); EXPECT_TRUE(!f._config.fastSearch()); - EXPECT_TRUE(!f._config.huge()); EXPECT_TRUE(!f._config.getEnableBitVectors()); EXPECT_TRUE(!f._config.getEnableOnlyBitVector()); EXPECT_TRUE(!f._config.getIsFilter()); @@ -48,7 +46,6 @@ TEST_F("test integer weightedset attribute config", EXPECT_EQUAL(CollectionType::Type::WSET, f._config.collectionType().type()); EXPECT_TRUE(!f._config.fastSearch()); - EXPECT_TRUE(!f._config.huge()); EXPECT_TRUE(!f._config.getEnableBitVectors()); EXPECT_TRUE(!f._config.getEnableOnlyBitVector()); EXPECT_TRUE(!f._config.getIsFilter()); diff --git a/searchlib/src/vespa/searchcommon/attribute/config.cpp b/searchlib/src/vespa/searchcommon/attribute/config.cpp index 0a50faa04c0..69dab6ad88e 100644 --- a/searchlib/src/vespa/searchcommon/attribute/config.cpp +++ b/searchlib/src/vespa/searchcommon/attribute/config.cpp @@ -11,15 +11,14 @@ static constexpr uint64_t MAX_UNCOMMITTED_MEMORY = 8000; } Config::Config() noexcept - : Config(BasicType::NONE, CollectionType::SINGLE, false, false) + : Config(BasicType::NONE, CollectionType::SINGLE, false) { } -Config::Config(BasicType bt, CollectionType ct, bool fastSearch_, bool huge_) noexcept +Config::Config(BasicType bt, CollectionType ct, bool fastSearch_) noexcept : _basicType(bt), _type(ct), _fastSearch(fastSearch_), - _huge(huge_), _enableBitVectors(false), _enableOnlyBitVector(false), _isFilter(false), @@ -49,7 +48,6 @@ Config::operator==(const Config &b) const { return _basicType == b._basicType && _type == b._type && - _huge == b._huge && _fastSearch == b._fastSearch && _enableBitVectors == b._enableBitVectors && _enableOnlyBitVector == b._enableOnlyBitVector && diff --git a/searchlib/src/vespa/searchcommon/attribute/config.h b/searchlib/src/vespa/searchcommon/attribute/config.h index f572f5038fc..edd5dbcc7a1 100644 --- a/searchlib/src/vespa/searchcommon/attribute/config.h +++ b/searchlib/src/vespa/searchcommon/attribute/config.h @@ -27,10 +27,7 @@ public: Config() noexcept; Config(BasicType bt) noexcept : Config(bt, CollectionType::SINGLE) { } Config(BasicType bt, CollectionType ct) noexcept : Config(bt, ct, false) { } - Config(BasicType bt, CollectionType ct, bool fastSearch_) noexcept - : Config(bt, ct, fastSearch_, false) - {} - Config(BasicType bt, CollectionType ct, bool fastSearch_, bool huge_) noexcept; + Config(BasicType bt, CollectionType ct, bool fastSearch_) noexcept; Config(const Config &); Config & operator = (const Config &); Config(Config &&) noexcept; @@ -40,7 +37,6 @@ public: BasicType basicType() const { return _basicType; } CollectionType collectionType() const { return _type; } bool fastSearch() const { return _fastSearch; } - bool huge() const { return _huge; } bool paged() const { return _paged; } const PredicateParams &predicateParams() const { return _predicateParams; } const vespalib::eval::ValueType & tensorType() const { return _tensorType; } @@ -72,7 +68,6 @@ public: const CompactionStrategy &getCompactionStrategy() const { return _compactionStrategy; } const DictionaryConfig & get_dictionary_config() const { return _dictionary; } Match get_match() const { return _match; } - Config & setHuge(bool v) { _huge = v; return *this;} Config & setFastSearch(bool v) { _fastSearch = v; return *this; } Config & setPredicateParams(const PredicateParams &v) { _predicateParams = v; return *this; } Config & setTensorType(const vespalib::eval::ValueType &tensorType_in) { @@ -137,7 +132,6 @@ private: BasicType _basicType; CollectionType _type; bool _fastSearch; - bool _huge; bool _enableBitVectors; bool _enableOnlyBitVector; bool _isFilter; diff --git a/searchlib/src/vespa/searchlib/aggregation/aggregation.cpp b/searchlib/src/vespa/searchlib/aggregation/aggregation.cpp index f78547b5b68..d4e7afb4252 100644 --- a/searchlib/src/vespa/searchlib/aggregation/aggregation.cpp +++ b/searchlib/src/vespa/searchlib/aggregation/aggregation.cpp @@ -60,7 +60,7 @@ void AggregationResult::aggregate(const document::Document & doc, HitRank rank) { bool ok(_expressionTree->execute(doc, rank)); if (ok) { - onAggregate(_expressionTree->getResult(), doc, rank); + onAggregate(*_expressionTree->getResult(), doc, rank); } else { throw std::runtime_error(vespalib::make_string("aggregate(%s, %f) failed ", doc.getId().toString().c_str(), rank)); } @@ -69,7 +69,7 @@ void AggregationResult::aggregate(DocId docId, HitRank rank) { bool ok(_expressionTree->execute(docId, rank)); if (ok) { - onAggregate(_expressionTree->getResult(), docId, rank); + onAggregate(*_expressionTree->getResult(), docId, rank); } else { throw std::runtime_error(vespalib::make_string("aggregate(%u, %f) failed ", docId, rank)); } @@ -92,7 +92,7 @@ AggregationResult & AggregationResult::setExpression(ExpressionNode::UP expr) { _expressionTree = std::make_shared<ExpressionTree>(std::move(expr)); - prepare(&_expressionTree->getResult(), false); + prepare(_expressionTree->getResult(), false); return *this; } diff --git a/searchlib/src/vespa/searchlib/aggregation/aggregationresult.h b/searchlib/src/vespa/searchlib/aggregation/aggregationresult.h index 35df257649e..db80798693e 100644 --- a/searchlib/src/vespa/searchlib/aggregation/aggregationresult.h +++ b/searchlib/src/vespa/searchlib/aggregation/aggregationresult.h @@ -62,7 +62,7 @@ public: } const ResultNode & getRank() const { return onGetRank(); } - const ResultNode & getResult() const override { return onGetRank(); } + const ResultNode * getResult() const override { return &onGetRank(); } virtual ResultNode & getResult() { return const_cast<ResultNode &>(onGetRank()); } virtual AggregationResult * clone() const override = 0; const ExpressionNode * getExpression() const { return _expressionTree->getRoot(); } @@ -73,7 +73,7 @@ private: void onPrepare(bool preserveAccurateTypes) override { (void) preserveAccurateTypes; } bool onExecute() const override { return true; } - void prepare() { if (getExpression() != nullptr) { prepare(&getExpression()->getResult(), false); } } + void prepare() { if (getExpression() != nullptr) { prepare(getExpression()->getResult(), false); } } void prepare(const ResultNode * result, bool useForInit) { if (result) { onPrepare(*result, useForInit); } } virtual void onPrepare(const ResultNode & result, bool useForInit) = 0; virtual void onMerge(const AggregationResult & b) = 0; diff --git a/searchlib/src/vespa/searchlib/aggregation/group.cpp b/searchlib/src/vespa/searchlib/aggregation/group.cpp index a3240ef8e25..e025d3b0f94 100644 --- a/searchlib/src/vespa/searchlib/aggregation/group.cpp +++ b/searchlib/src/vespa/searchlib/aggregation/group.cpp @@ -89,7 +89,7 @@ Group::groupNext(const GroupingLevel & level, const Doc & doc, HitRank rank) if (!selector.execute(doc, rank)) { throw std::runtime_error("Does not know how to handle failed select statements"); } - const ResultNode &selectResult = selector.getResult(); + const ResultNode &selectResult = *selector.getResult(); level.group(*this, selectResult, doc, rank); } @@ -220,7 +220,7 @@ Group::Value::cmp(const Value & rhs) const { int diff(0); for (size_t i(0), m(getOrderBySize()); (diff == 0) && (i < m); i++) { uint32_t index = std::abs(getOrderBy(i)) - 1; - diff = expr(index).getResult().cmp(rhs.expr(index).getResult()) * getOrderBy(i); + diff = expr(index).getResult()->cmp(*rhs.expr(index).getResult()) * getOrderBy(i); } return diff; } diff --git a/searchlib/src/vespa/searchlib/aggregation/grouping.cpp b/searchlib/src/vespa/searchlib/aggregation/grouping.cpp index c07e399b38e..daff16e9029 100644 --- a/searchlib/src/vespa/searchlib/aggregation/grouping.cpp +++ b/searchlib/src/vespa/searchlib/aggregation/grouping.cpp @@ -185,12 +185,12 @@ void Grouping::postProcess() bool hasEnums(false); for (size_t i(0), m(_levels.size()); !hasEnums && (i < m); i++) { const GroupingLevel & l = _levels[i]; - const ResultNode & id(l.getExpression().getResult()); + const ResultNode & id(*l.getExpression().getResult()); hasEnums = id.inherits(EnumResultNode::classId) || id.inherits(EnumResultNodeVector::classId); const Group & g(l.getGroupPrototype()); for (size_t j(0), n(g.getAggrSize()); !hasEnums && (j < n); j++) { - const ResultNode & r(g.getAggregationResult(j).getResult()); + const ResultNode & r(*g.getAggregationResult(j).getResult()); hasEnums = r.inherits(EnumResultNode::classId) || r.inherits(EnumResultNodeVector::classId); } diff --git a/searchlib/src/vespa/searchlib/aggregation/groupinglevel.cpp b/searchlib/src/vespa/searchlib/aggregation/groupinglevel.cpp index 294541c5f79..73938437f3e 100644 --- a/searchlib/src/vespa/searchlib/aggregation/groupinglevel.cpp +++ b/searchlib/src/vespa/searchlib/aggregation/groupinglevel.cpp @@ -96,7 +96,7 @@ void GroupingLevel::prepare(const Grouping * grouping, uint32_t level, bool isOr { _isOrdered = isOrdered_; _frozen = level < grouping->getFirstLevel(); - if (_classify.getResult().inherits(ResultNodeVector::classId)) { + if (_classify.getResult()->inherits(ResultNodeVector::classId)) { _grouper.reset(new MultiValueGrouper(grouping, level)); } else { _grouper.reset(new SingleValueGrouper(grouping, level)); diff --git a/searchlib/src/vespa/searchlib/aggregation/hitsaggregationresult.h b/searchlib/src/vespa/searchlib/aggregation/hitsaggregationresult.h index 204417d721c..97595a4119c 100644 --- a/searchlib/src/vespa/searchlib/aggregation/hitsaggregationresult.h +++ b/searchlib/src/vespa/searchlib/aggregation/hitsaggregationresult.h @@ -57,7 +57,7 @@ public: void visitMembers(vespalib::ObjectVisitor &visitor) const override; void selectMembers(const vespalib::ObjectPredicate &predicate, vespalib::ObjectOperation &operation) override; HitsAggregationResult & sort() { _hits.sort(); return *this; } - const ResultNode & getResult() const override { return _hits; } + const ResultNode * getResult() const override { return &_hits; } ResultNode & getResult() override { return _hits; } }; diff --git a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp index 9aac4f301d5..ed7d5abb8f8 100644 --- a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp @@ -103,7 +103,6 @@ ConfigConverter::convert(const AttributesConfig::Attribute & cfg) Config retval(bType, cType); PredicateParams predicateParams; retval.setFastSearch(cfg.fastsearch); - retval.setHuge(cfg.huge); retval.setEnableBitVectors(cfg.enablebitvectors); retval.setEnableOnlyBitVector(cfg.enableonlybitvector); retval.setIsFilter(cfg.enableonlybitvector); diff --git a/searchlib/src/vespa/searchlib/expression/aggregationrefnode.h b/searchlib/src/vespa/searchlib/expression/aggregationrefnode.h index 176ac0d585e..cae438c8351 100644 --- a/searchlib/src/vespa/searchlib/expression/aggregationrefnode.h +++ b/searchlib/src/vespa/searchlib/expression/aggregationrefnode.h @@ -31,7 +31,7 @@ public: AggregationRefNode & operator = (const AggregationRefNode & exprref); ExpressionNode *getExpression() { return _expressionNode; } - const ResultNode & getResult() const override { return _expressionNode->getResult(); } + const ResultNode * getResult() const override { return _expressionNode->getResult(); } void onPrepare(bool preserveAccurateTypes) override { _expressionNode->prepare(preserveAccurateTypes); } bool onExecute() const override; diff --git a/searchlib/src/vespa/searchlib/expression/arrayatlookupfunctionnode.cpp b/searchlib/src/vespa/searchlib/expression/arrayatlookupfunctionnode.cpp index bd8a617f6d2..3663d76ad51 100644 --- a/searchlib/src/vespa/searchlib/expression/arrayatlookupfunctionnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/arrayatlookupfunctionnode.cpp @@ -78,7 +78,7 @@ void ArrayAtLookup::onPrepareResult() bool ArrayAtLookup::onExecute() const { getArg().execute(); - int64_t idx = getArg().getResult().getInteger(); + int64_t idx = getArg().getResult()->getInteger(); // get attribute data size_t numValues = _attribute->getValueCount(_docId); if (idx < 0) { diff --git a/searchlib/src/vespa/searchlib/expression/constantnode.h b/searchlib/src/vespa/searchlib/expression/constantnode.h index 092981f4a08..53bd01fc651 100644 --- a/searchlib/src/vespa/searchlib/expression/constantnode.h +++ b/searchlib/src/vespa/searchlib/expression/constantnode.h @@ -15,7 +15,7 @@ public: ConstantNode() : ExpressionNode(), _result() { } ConstantNode(ResultNode::UP r) : ExpressionNode(), _result(r.release()) { } void visitMembers(vespalib::ObjectVisitor &visitor) const override; - const ResultNode & getResult() const override { return *_result; } + const ResultNode * getResult() const override { return _result.get(); } private: void onPrepare(bool preserveAccurateTypes) override { (void) preserveAccurateTypes; } bool onExecute() const override { return true; } diff --git a/searchlib/src/vespa/searchlib/expression/debugwaitfunctionnode.cpp b/searchlib/src/vespa/searchlib/expression/debugwaitfunctionnode.cpp index 5014464b88c..1b9aa40e1db 100644 --- a/searchlib/src/vespa/searchlib/expression/debugwaitfunctionnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/debugwaitfunctionnode.cpp @@ -32,7 +32,7 @@ DebugWaitFunctionNode::onExecute() const vespalib::Timer::waitAtLeast(vespalib::from_s(_waitTime), _busyWait); getArg().execute(); - updateResult().assign(getArg().getResult()); + updateResult().assign(*getArg().getResult()); return true; } diff --git a/searchlib/src/vespa/searchlib/expression/documentfieldnode.h b/searchlib/src/vespa/searchlib/expression/documentfieldnode.h index e1038f73fa3..cbe5fedf47a 100644 --- a/searchlib/src/vespa/searchlib/expression/documentfieldnode.h +++ b/searchlib/src/vespa/searchlib/expression/documentfieldnode.h @@ -66,7 +66,7 @@ private: void onPrimitive(uint32_t fid, const Content & c) override; }; - const ResultNode & getResult() const override { return *_value; } + const ResultNode * getResult() const override { return _value.get(); } void onPrepare(bool preserveAccurateTypes) override; bool onExecute() const override; void onDoc(const document::Document & doc) override; diff --git a/searchlib/src/vespa/searchlib/expression/expressionnode.h b/searchlib/src/vespa/searchlib/expression/expressionnode.h index 8a96192b04a..9c2c4c5d3e0 100644 --- a/searchlib/src/vespa/searchlib/expression/expressionnode.h +++ b/searchlib/src/vespa/searchlib/expression/expressionnode.h @@ -37,7 +37,7 @@ public: DECLARE_ABSTRACT_EXPRESSIONNODE(ExpressionNode); using UP = std::unique_ptr<ExpressionNode>; using CP = vespalib::IdentifiablePtr<ExpressionNode>; - virtual const ResultNode & getResult() const = 0; + virtual const ResultNode * getResult() const = 0; bool execute() const { return onExecute(); } ExpressionNode & prepare(bool preserveAccurateTypes) { onPrepare(preserveAccurateTypes); return *this; } virtual ExpressionNode * clone() const = 0; diff --git a/searchlib/src/vespa/searchlib/expression/expressiontree.h b/searchlib/src/vespa/searchlib/expression/expressiontree.h index 9f75f1734cd..9497772059e 100644 --- a/searchlib/src/vespa/searchlib/expression/expressiontree.h +++ b/searchlib/src/vespa/searchlib/expression/expressiontree.h @@ -54,7 +54,7 @@ public: bool execute(const document::Document & doc, HitRank rank) const; const ExpressionNode * getRoot() const { return _root.get(); } ExpressionNode * getRoot() { return _root.get(); } - const ResultNode & getResult() const override { return _root->getResult(); } + const ResultNode * getResult() const override { return _root->getResult(); } friend vespalib::Serializer & operator << (vespalib::Serializer & os, const ExpressionTree & et); friend vespalib::Deserializer & operator >> (vespalib::Deserializer & is, ExpressionTree & et); void swap(ExpressionTree &); diff --git a/searchlib/src/vespa/searchlib/expression/fixedwidthbucketfunctionnode.cpp b/searchlib/src/vespa/searchlib/expression/fixedwidthbucketfunctionnode.cpp index 723371a2ff6..f7c8941c3e2 100644 --- a/searchlib/src/vespa/searchlib/expression/fixedwidthbucketfunctionnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/fixedwidthbucketfunctionnode.cpp @@ -85,7 +85,7 @@ void FixedWidthBucketFunctionNode::onPrepareResult() { const ExpressionNode &child = getArg(); - const ResultNode &input = child.getResult(); + const ResultNode &input = *child.getResult(); if (input.getClass().inherits(IntegerResultNode::classId)) { ResultNode::UP res(new IntegerBucketResultNode()); setResultType(std::move(res)); @@ -111,7 +111,7 @@ bool FixedWidthBucketFunctionNode::onExecute() const { getArg().execute(); - _bucketHandler->update(updateResult(), getArg().getResult()); + _bucketHandler->update(updateResult(), *getArg().getResult()); return true; } diff --git a/searchlib/src/vespa/searchlib/expression/functionnode.h b/searchlib/src/vespa/searchlib/expression/functionnode.h index 486f0d5428b..4e23a2f3c60 100644 --- a/searchlib/src/vespa/searchlib/expression/functionnode.h +++ b/searchlib/src/vespa/searchlib/expression/functionnode.h @@ -13,7 +13,7 @@ public: DECLARE_NBO_SERIALIZE; void visitMembers(vespalib::ObjectVisitor & visitor) const override; DECLARE_ABSTRACT_EXPRESSIONNODE(FunctionNode); - const ResultNode & getResult() const override { return *_tmpResult; } + const ResultNode * getResult() const override { return _tmpResult.get(); } ResultNode & updateResult() const { return *_tmpResult; } virtual void reset() { _tmpResult.reset(NULL); } diff --git a/searchlib/src/vespa/searchlib/expression/functionnodes.cpp b/searchlib/src/vespa/searchlib/expression/functionnodes.cpp index 4e929be0a64..9046a5427ce 100644 --- a/searchlib/src/vespa/searchlib/expression/functionnodes.cpp +++ b/searchlib/src/vespa/searchlib/expression/functionnodes.cpp @@ -208,19 +208,12 @@ void MultiArgFunctionNode::onPrepare(bool preserveAccurateTypes) void MultiArgFunctionNode::onPrepareResult() { if (_args.size() == 1) { - setResultType(ArithmeticTypeConversion::getType(_args[0]->getResult())); + setResultType(ArithmeticTypeConversion::getType(*_args[0]->getResult())); } else if (_args.size() > 1) { - setResultType(std::unique_ptr<ResultNode>(static_cast<ResultNode *>(_args[0]->getResult().clone()))); + setResultType(std::unique_ptr<ResultNode>(static_cast<ResultNode *>(_args[0]->getResult()->clone()))); for(size_t i(1), m(_args.size()); i < m; i++) { -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wtautological-undefined-compare" -#endif - if (&_args[i]->getResult() != NULL) { -#ifdef __clang__ -#pragma clang diagnostic pop -#endif - setResultType(_ArithmeticTypeConversion.getType(getResult(), _args[i]->getResult())); + if (_args[i]->getResult() != nullptr) { + setResultType(_ArithmeticTypeConversion.getType(*getResult(), *_args[i]->getResult())); } } } @@ -236,9 +229,9 @@ bool MultiArgFunctionNode::onExecute() const bool MultiArgFunctionNode::onCalculate(const ExpressionNodeVector & args, ResultNode & result) const { - result.set(args[0]->getResult()); + result.set(*args[0]->getResult()); for (size_t i(1), m(args.size()); i < m; i++) { - executeIterative(args[i]->getResult(), result); + executeIterative(*args[i]->getResult(), result); } return true; } @@ -282,7 +275,7 @@ void XorFunctionNode::onArgument(const ResultNode & arg, Int64ResultNode & resul ResultNode::CP MaxFunctionNode::getInitialValue() const { ResultNode::CP initial; - const ResultNode & arg(getArg(0).getResult()); + const ResultNode & arg(*getArg(0).getResult()); if (arg.inherits(FloatResultNodeVector::classId)) { initial.reset(new FloatResultNode(std::numeric_limits<double>::min())); } else if (arg.inherits(IntegerResultNodeVector::classId)) { @@ -296,7 +289,7 @@ ResultNode::CP MaxFunctionNode::getInitialValue() const ResultNode::CP MinFunctionNode::getInitialValue() const { ResultNode::CP initial; - const ResultNode & arg(getArg(0).getResult()); + const ResultNode & arg(*getArg(0).getResult()); if (arg.inherits(FloatResultNodeVector::classId)) { initial.reset(new FloatResultNode(std::numeric_limits<double>::max())); } else if (arg.inherits(IntegerResultNodeVector::classId)) { @@ -340,7 +333,7 @@ void UnaryBitFunctionNode::onPrepare(bool preserveAccurateTypes) void UnaryFunctionNode::onPrepareResult() { - setResultType(std::unique_ptr<ResultNode>(getArg().getResult().clone())); + setResultType(std::unique_ptr<ResultNode>(getArg().getResult()->clone())); } void ToStringFunctionNode::onPrepareResult() @@ -351,7 +344,7 @@ void ToStringFunctionNode::onPrepareResult() bool ToStringFunctionNode::onExecute() const { getArg().execute(); - updateResult().set(getArg().getResult()); + updateResult().set(*getArg().getResult()); return true; } @@ -363,7 +356,7 @@ void ToRawFunctionNode::onPrepareResult() bool ToRawFunctionNode::onExecute() const { getArg().execute(); - updateResult().set(getArg().getResult()); + updateResult().set(*getArg().getResult()); return true; } @@ -375,7 +368,7 @@ void ToIntFunctionNode::onPrepareResult() bool ToIntFunctionNode::onExecute() const { getArg().execute(); - updateResult().set(getArg().getResult()); + updateResult().set(*getArg().getResult()); return true; } @@ -387,7 +380,7 @@ void ToFloatFunctionNode::onPrepareResult() bool ToFloatFunctionNode::onExecute() const { getArg().execute(); - updateResult().set(getArg().getResult()); + updateResult().set(*getArg().getResult()); return true; } @@ -400,7 +393,7 @@ bool StrLenFunctionNode::onExecute() const { getArg().execute(); char buf[32]; - static_cast<Int64ResultNode &> (updateResult()).set(getArg().getResult().getString(BufferRef(buf, sizeof(buf))).size()); + static_cast<Int64ResultNode &> (updateResult()).set(getArg().getResult()->getString(BufferRef(buf, sizeof(buf))).size()); return true; } @@ -413,7 +406,7 @@ bool NormalizeSubjectFunctionNode::onExecute() const { getArg().execute(); char buf[32]; - ConstBufferRef tmp(getArg().getResult().getString(BufferRef(buf, sizeof(buf)))); + ConstBufferRef tmp(getArg().getResult()->getString(BufferRef(buf, sizeof(buf)))); int pos = 0; if (tmp.size() >= 4) { @@ -439,8 +432,8 @@ void NumElemFunctionNode::onPrepareResult() bool NumElemFunctionNode::onExecute() const { getArg().execute(); - if (getArg().getResult().inherits(ResultNodeVector::classId)) { - static_cast<Int64ResultNode &> (updateResult()).set(static_cast<const ResultNodeVector &>(getArg().getResult()).size()); + if (getArg().getResult()->inherits(ResultNodeVector::classId)) { + static_cast<Int64ResultNode &> (updateResult()).set(static_cast<const ResultNodeVector &>(*getArg().getResult()).size()); } return true; } @@ -448,7 +441,7 @@ bool NumElemFunctionNode::onExecute() const bool NegateFunctionNode::onExecute() const { getArg().execute(); - updateResult().assign(getArg().getResult()); + updateResult().assign(*getArg().getResult()); updateResult().negate(); return true; } @@ -456,7 +449,7 @@ bool NegateFunctionNode::onExecute() const bool SortFunctionNode::onExecute() const { getArg().execute(); - updateResult().assign(getArg().getResult()); + updateResult().assign(*getArg().getResult()); updateResult().sort(); return true; } @@ -464,7 +457,7 @@ bool SortFunctionNode::onExecute() const bool ReverseFunctionNode::onExecute() const { getArg().execute(); - updateResult().assign(getArg().getResult()); + updateResult().assign(*getArg().getResult()); updateResult().reverse(); return true; } @@ -475,7 +468,7 @@ bool StrCatFunctionNode::onExecute() const StrCatSerializer nos(os); for(size_t i(0), m(getNumArgs()); i < m; i++) { getArg(i).execute(); - getArg(i).getResult().serialize(nos); + getArg(i).getResult()->serialize(nos); } static_cast<StringResultNode &>(updateResult()).set(os.str()); return true; @@ -487,7 +480,7 @@ bool CatFunctionNode::onExecute() const CatSerializer nos(os); for(size_t i(0), m(getNumArgs()); i < m; i++) { getArg(i).execute(); - getArg(i).getResult().serialize(nos); + getArg(i).getResult()->serialize(nos); } static_cast<RawResultNode &>(updateResult()).setBuffer(os.data(), os.size()); return true; @@ -506,7 +499,7 @@ bool UnaryBitFunctionNode::onExecute() const _tmpOs.clear(); getArg().execute(); CatSerializer os(_tmpOs); - getArg().getResult().serialize(os); + getArg().getResult()->serialize(os); return internalExecute(_tmpOs); } diff --git a/searchlib/src/vespa/searchlib/expression/getdocidnamespacespecificfunctionnode.h b/searchlib/src/vespa/searchlib/expression/getdocidnamespacespecificfunctionnode.h index 8d508fc9456..cc3d6dd697c 100644 --- a/searchlib/src/vespa/searchlib/expression/getdocidnamespacespecificfunctionnode.h +++ b/searchlib/src/vespa/searchlib/expression/getdocidnamespacespecificfunctionnode.h @@ -15,7 +15,7 @@ public: GetDocIdNamespaceSpecificFunctionNode() : _value(new StringResultNode("")) { } GetDocIdNamespaceSpecificFunctionNode(ResultNode::UP resultNode) : _value(resultNode.release()) { } private: - const ResultNode & getResult() const override { return *_value; } + const ResultNode * getResult() const override { return _value.get(); } void onDocType(const document::DocumentType & docType) override { (void) docType; } void onDoc(const document::Document & doc) override; void onPrepare(bool preserveAccurateTypes) override { (void) preserveAccurateTypes; } diff --git a/searchlib/src/vespa/searchlib/expression/getymumchecksumfunctionnode.h b/searchlib/src/vespa/searchlib/expression/getymumchecksumfunctionnode.h index 55a1d19bd53..624edb4f9f8 100644 --- a/searchlib/src/vespa/searchlib/expression/getymumchecksumfunctionnode.h +++ b/searchlib/src/vespa/searchlib/expression/getymumchecksumfunctionnode.h @@ -14,7 +14,7 @@ public: DECLARE_EXPRESSIONNODE(GetYMUMChecksumFunctionNode); private: void onPrepare(bool preserveAccurateTypes) override { (void) preserveAccurateTypes; } - const ResultNode & getResult() const override { return _checkSum; } + const ResultNode * getResult() const override { return &_checkSum; } void onDocType(const document::DocumentType & docType) override { (void) docType; } void onDoc(const document::Document & doc) override; bool onExecute() const override { return true; } diff --git a/searchlib/src/vespa/searchlib/expression/interpolatedlookupfunctionnode.cpp b/searchlib/src/vespa/searchlib/expression/interpolatedlookupfunctionnode.cpp index 993df97df8f..7f55573afbf 100644 --- a/searchlib/src/vespa/searchlib/expression/interpolatedlookupfunctionnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/interpolatedlookupfunctionnode.cpp @@ -88,7 +88,7 @@ simpleInterpolate(size_t sz, std::vector<double> v, double lookup) bool InterpolatedLookup::onExecute() const { getArg().execute(); - double lookup = getArg().getResult().getFloat(); + double lookup = getArg().getResult()->getFloat(); // get attribute data size_t numValues = _attribute->getValueCount(_docId); std::vector<double> valueVector; diff --git a/searchlib/src/vespa/searchlib/expression/mathfunctionnode.cpp b/searchlib/src/vespa/searchlib/expression/mathfunctionnode.cpp index ac0e3261088..7fe247bc1a4 100644 --- a/searchlib/src/vespa/searchlib/expression/mathfunctionnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/mathfunctionnode.cpp @@ -36,27 +36,27 @@ bool MathFunctionNode::onExecute() const getArg(0).execute(); double result(0.0); switch (_function) { - case EXP: result = std::exp(getArg(0).getResult().getFloat()); break; - case POW: getArg(1).execute(); result = std::pow(getArg(0).getResult().getFloat(), getArg(1).getResult().getFloat()); break; - case LOG: result = std::log(getArg(0).getResult().getFloat()); break; - case LOG1P: result = std::log1p(getArg(0).getResult().getFloat()); break; - case LOG10: result = std::log10(getArg(0).getResult().getFloat()); break; - case SIN: result = std::sin(getArg(0).getResult().getFloat()); break; - case ASIN: result = std::asin(getArg(0).getResult().getFloat()); break; - case COS: result = std::cos(getArg(0).getResult().getFloat()); break; - case ACOS: result = std::acos(getArg(0).getResult().getFloat()); break; - case TAN: result = std::tan(getArg(0).getResult().getFloat()); break; - case ATAN: result = std::atan(getArg(0).getResult().getFloat()); break; - case SQRT: result = std::sqrt(getArg(0).getResult().getFloat()); break; - case SINH: result = std::sinh(getArg(0).getResult().getFloat()); break; - case ASINH: result = std::asinh(getArg(0).getResult().getFloat()); break; - case COSH: result = std::cosh(getArg(0).getResult().getFloat()); break; - case ACOSH: result = std::acosh(getArg(0).getResult().getFloat()); break; - case TANH: result = std::tanh(getArg(0).getResult().getFloat()); break; - case ATANH: result = std::atanh(getArg(0).getResult().getFloat()); break; - case CBRT: result = std::cbrt(getArg(0).getResult().getFloat()); break; - case HYPOT: getArg(1).execute(); result = std::hypot(getArg(0).getResult().getFloat(), getArg(1).getResult().getFloat()); break; - case FLOOR: result = std::floor(getArg(0).getResult().getFloat()); break; + case EXP: result = std::exp(getArg(0).getResult()->getFloat()); break; + case POW: getArg(1).execute(); result = std::pow(getArg(0).getResult()->getFloat(), getArg(1).getResult()->getFloat()); break; + case LOG: result = std::log(getArg(0).getResult()->getFloat()); break; + case LOG1P: result = std::log1p(getArg(0).getResult()->getFloat()); break; + case LOG10: result = std::log10(getArg(0).getResult()->getFloat()); break; + case SIN: result = std::sin(getArg(0).getResult()->getFloat()); break; + case ASIN: result = std::asin(getArg(0).getResult()->getFloat()); break; + case COS: result = std::cos(getArg(0).getResult()->getFloat()); break; + case ACOS: result = std::acos(getArg(0).getResult()->getFloat()); break; + case TAN: result = std::tan(getArg(0).getResult()->getFloat()); break; + case ATAN: result = std::atan(getArg(0).getResult()->getFloat()); break; + case SQRT: result = std::sqrt(getArg(0).getResult()->getFloat()); break; + case SINH: result = std::sinh(getArg(0).getResult()->getFloat()); break; + case ASINH: result = std::asinh(getArg(0).getResult()->getFloat()); break; + case COSH: result = std::cosh(getArg(0).getResult()->getFloat()); break; + case ACOSH: result = std::acosh(getArg(0).getResult()->getFloat()); break; + case TANH: result = std::tanh(getArg(0).getResult()->getFloat()); break; + case ATANH: result = std::atanh(getArg(0).getResult()->getFloat()); break; + case CBRT: result = std::cbrt(getArg(0).getResult()->getFloat()); break; + case HYPOT: getArg(1).execute(); result = std::hypot(getArg(0).getResult()->getFloat(), getArg(1).getResult()->getFloat()); break; + case FLOOR: result = std::floor(getArg(0).getResult()->getFloat()); break; } static_cast<FloatResultNode &>(updateResult()).set(result); return true; diff --git a/searchlib/src/vespa/searchlib/expression/numericfunctionnode.cpp b/searchlib/src/vespa/searchlib/expression/numericfunctionnode.cpp index 9cb10a81335..d9c664e5cde 100644 --- a/searchlib/src/vespa/searchlib/expression/numericfunctionnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/numericfunctionnode.cpp @@ -28,32 +28,32 @@ void NumericFunctionNode::onPrepare(bool preserveAccurateTypes) { MultiArgFunctionNode::onPrepare(preserveAccurateTypes); if (getNumArgs() == 1) { - if (getArg(0).getResult().getClass().inherits(IntegerResultNodeVector::classId)) { + if (getArg(0).getResult()->getClass().inherits(IntegerResultNodeVector::classId)) { _handler.reset(new FlattenIntegerHandler(*this)); - } else if (getArg(0).getResult().getClass().inherits(FloatResultNodeVector::classId)) { + } else if (getArg(0).getResult()->getClass().inherits(FloatResultNodeVector::classId)) { _handler.reset(new FlattenFloatHandler(*this)); - } else if (getArg(0).getResult().getClass().inherits(StringResultNodeVector::classId)) { + } else if (getArg(0).getResult()->getClass().inherits(StringResultNodeVector::classId)) { _handler.reset(new FlattenStringHandler(*this)); } else { - throw std::runtime_error(vespalib::string("No FlattenHandler for ") + getArg(0).getResult().getClass().name()); + throw std::runtime_error(vespalib::string("No FlattenHandler for ") + getArg(0).getResult()->getClass().name()); } } else { - if (getResult().getClass().inherits(IntegerResultNodeVector::classId)) { + if (getResult()->getClass().inherits(IntegerResultNodeVector::classId)) { _handler.reset(new VectorIntegerHandler(*this)); - } else if (getResult().getClass().inherits(FloatResultNodeVector::classId)) { + } else if (getResult()->getClass().inherits(FloatResultNodeVector::classId)) { _handler.reset(new VectorFloatHandler(*this)); - } else if (getResult().getClass().inherits(StringResultNodeVector::classId)) { + } else if (getResult()->getClass().inherits(StringResultNodeVector::classId)) { _handler.reset(new VectorStringHandler(*this)); - } else if (getResult().getClass().inherits(IntegerResultNode::classId)) { + } else if (getResult()->getClass().inherits(IntegerResultNode::classId)) { _handler.reset(new ScalarIntegerHandler(*this)); - } else if (getResult().getClass().inherits(FloatResultNode::classId)) { + } else if (getResult()->getClass().inherits(FloatResultNode::classId)) { _handler.reset(new ScalarFloatHandler(*this)); - } else if (getResult().getClass().inherits(StringResultNode::classId)) { + } else if (getResult()->getClass().inherits(StringResultNode::classId)) { _handler.reset(new ScalarStringHandler(*this)); - } else if (getResult().getClass().inherits(RawResultNode::classId)) { + } else if (getResult()->getClass().inherits(RawResultNode::classId)) { _handler.reset(new ScalarRawHandler(*this)); } else { - throw std::runtime_error(vespalib::make_string("NumericFunctionNode::onPrepare does not handle results of type %s", getResult().getClass().name())); + throw std::runtime_error(vespalib::make_string("NumericFunctionNode::onPrepare does not handle results of type %s", getResult()->getClass().name())); } } } @@ -62,9 +62,9 @@ bool NumericFunctionNode::onCalculate(const ExpressionNodeVector & args, ResultN { bool retval(true); (void) result; - _handler->handleFirst(args[0]->getResult()); + _handler->handleFirst(*args[0]->getResult()); for (size_t i(1), m(args.size()); i < m; i++) { - _handler->handle(args[i]->getResult()); + _handler->handle(*args[i]->getResult()); } return retval; } diff --git a/searchlib/src/vespa/searchlib/expression/rangebucketpredef.cpp b/searchlib/src/vespa/searchlib/expression/rangebucketpredef.cpp index fbd4879370b..a095a3cd46b 100644 --- a/searchlib/src/vespa/searchlib/expression/rangebucketpredef.cpp +++ b/searchlib/src/vespa/searchlib/expression/rangebucketpredef.cpp @@ -33,10 +33,10 @@ void RangeBucketPreDefFunctionNode::onPrepareResult() { // Use the type of the predefined buckets for the null bucket - const ResultNode& resultNode = _predef->empty() ? getArg().getResult() : _predef->get(0); + const ResultNode& resultNode = _predef->empty() ? *getArg().getResult() : _predef->get(0); _nullResult = &resultNode.getNullBucket(); - const vespalib::Identifiable::RuntimeClass & cInfo(getArg().getResult().getClass()); + const vespalib::Identifiable::RuntimeClass & cInfo(getArg().getResult()->getClass()); if (cInfo.inherits(ResultNodeVector::classId)) { setResultType(ResultNode::UP(_predef->clone())); static_cast<ResultNodeVector &>(updateResult()).clear(); @@ -52,7 +52,7 @@ bool RangeBucketPreDefFunctionNode::onExecute() const { getArg().execute(); - const ResultNode * result = _handler->handle(getArg().getResult()); + const ResultNode * result = _handler->handle(*getArg().getResult()); _result = result ? result : _nullResult; return true; } diff --git a/searchlib/src/vespa/searchlib/expression/rangebucketpredef.h b/searchlib/src/vespa/searchlib/expression/rangebucketpredef.h index 4e220733b63..313ef62404d 100644 --- a/searchlib/src/vespa/searchlib/expression/rangebucketpredef.h +++ b/searchlib/src/vespa/searchlib/expression/rangebucketpredef.h @@ -62,7 +62,7 @@ public: RangeBucketPreDefFunctionNode(const RangeBucketPreDefFunctionNode & rhs); RangeBucketPreDefFunctionNode & operator = (const RangeBucketPreDefFunctionNode & rhs); ~RangeBucketPreDefFunctionNode(); - const ResultNode & getResult() const override { return *_result; } + const ResultNode * getResult() const override { return _result; } const ResultNodeVector & getBucketList() const { return *_predef; } ResultNodeVector & getBucketList() { return *_predef; } RangeBucketPreDefFunctionNode & setBucketList(const ResultNodeVector & predef) { diff --git a/searchlib/src/vespa/searchlib/expression/relevancenode.h b/searchlib/src/vespa/searchlib/expression/relevancenode.h index a20059757f7..05ac3fe6cc4 100644 --- a/searchlib/src/vespa/searchlib/expression/relevancenode.h +++ b/searchlib/src/vespa/searchlib/expression/relevancenode.h @@ -13,7 +13,7 @@ public: DECLARE_EXPRESSIONNODE(RelevanceNode); RelevanceNode() : ExpressionNode(), _relevance() { } void visitMembers(vespalib::ObjectVisitor &visitor) const override; - const ResultNode & getResult() const override { return _relevance; } + const ResultNode * getResult() const override { return &_relevance; } void setRelevance(double relevance) { _relevance.set(relevance); } private: void onPrepare(bool preserveAccurateTypes) override { (void) preserveAccurateTypes; } diff --git a/searchlib/src/vespa/searchlib/expression/timestamp.cpp b/searchlib/src/vespa/searchlib/expression/timestamp.cpp index 0a24b24c4e0..acb621c4b9d 100644 --- a/searchlib/src/vespa/searchlib/expression/timestamp.cpp +++ b/searchlib/src/vespa/searchlib/expression/timestamp.cpp @@ -41,7 +41,7 @@ TimeStampFunctionNode & TimeStampFunctionNode::operator = (const TimeStampFuncti void TimeStampFunctionNode::onPrepareResult() { - if (getArg().getResult().inherits(ResultNodeVector::classId)) { + if (getArg().getResult()->inherits(ResultNodeVector::classId)) { setResultType(std::unique_ptr<ResultNode>(new IntegerResultNodeVector)); _handler.reset(new MultiValueHandler(*this)); } else { @@ -75,7 +75,7 @@ unsigned TimeStampFunctionNode::getTimePart(time_t secSince70, TimePart tp, bool bool TimeStampFunctionNode::onExecute() const { getArg().execute(); - _handler->handle(getArg().getResult()); + _handler->handle(*getArg().getResult()); return true; } diff --git a/searchlib/src/vespa/searchlib/expression/timestamp.h b/searchlib/src/vespa/searchlib/expression/timestamp.h index 4e1245513f9..50359be6209 100644 --- a/searchlib/src/vespa/searchlib/expression/timestamp.h +++ b/searchlib/src/vespa/searchlib/expression/timestamp.h @@ -18,7 +18,7 @@ public: ~TimeStampFunctionNode(); TimeStampFunctionNode(const TimeStampFunctionNode & rhs); TimeStampFunctionNode & operator = (const TimeStampFunctionNode & rhs); - unsigned int getTime() const { return getResult().getInteger(); } // Not valid until after node has been prepared + unsigned int getTime() const { return getResult()->getInteger(); } // Not valid until after node has been prepared TimePart getTimePart() const { return _timePart; } TimeStampFunctionNode & setTimePart(TimePart timePart) { _timePart = timePart; return *this; } bool isGmt() const { return _isGmt; } @@ -64,7 +64,7 @@ private: IntegerResultNodeVector & _result; }; - const ResultNode & getTimeStamp() const { return getArg().getResult(); } + const ResultNode & getTimeStamp() const { return *getArg().getResult(); } void init(); Int64ResultNode & updateIntegerResult() const { return static_cast<Int64ResultNode &>(updateResult()); } static unsigned getTimePart(time_t time, TimePart, bool gmt); diff --git a/searchlib/src/vespa/searchlib/expression/zcurve.cpp b/searchlib/src/vespa/searchlib/expression/zcurve.cpp index cc9f8656dda..e50a2245426 100644 --- a/searchlib/src/vespa/searchlib/expression/zcurve.cpp +++ b/searchlib/src/vespa/searchlib/expression/zcurve.cpp @@ -31,7 +31,7 @@ ZCurveFunctionNode & ZCurveFunctionNode::operator = (const ZCurveFunctionNode & void ZCurveFunctionNode::onPrepareResult() { - if (getArg().getResult().inherits(ResultNodeVector::classId)) { + if (getArg().getResult()->inherits(ResultNodeVector::classId)) { setResultType(std::make_unique<IntegerResultNodeVector>()); _handler = std::make_unique<MultiValueHandler>(*this); } else { @@ -50,7 +50,7 @@ int32_t ZCurveFunctionNode::Handler::getXorY(uint64_t z) const bool ZCurveFunctionNode::onExecute() const { getArg().execute(); - _handler->handle(getArg().getResult()); + _handler->handle(*getArg().getResult()); return true; } diff --git a/searchlib/src/vespa/searchlib/features/bm25_feature.cpp b/searchlib/src/vespa/searchlib/features/bm25_feature.cpp index 22b29014a81..66658fd2ca5 100644 --- a/searchlib/src/vespa/searchlib/features/bm25_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/bm25_feature.cpp @@ -2,6 +2,7 @@ #include "bm25_feature.h" #include "utils.h" +#include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/itermdata.h> #include <vespa/searchlib/fef/itermfielddata.h> #include <vespa/searchlib/fef/objectstore.h> @@ -18,7 +19,9 @@ namespace search::features { using fef::AnyWrapper; using fef::Blueprint; using fef::FeatureExecutor; +using fef::FeatureNameBuilder; using fef::FieldInfo; +using fef::FieldType; using fef::ITermData; using fef::ITermFieldData; using fef::MatchDataDetails; @@ -127,9 +130,14 @@ Bm25Blueprint::Bm25Blueprint() void Bm25Blueprint::visitDumpFeatures(const fef::IIndexEnvironment& env, fef::IDumpFeatureVisitor& visitor) const { - (void) env; - (void) visitor; - // TODO: Implement when feature is supported end-2-end with both memory and disk index. + for (uint32_t i = 0; i < env.getNumFields(); ++i) { + const auto* field = env.getField(i); + if (field->type() == FieldType::INDEX) { + FeatureNameBuilder fnb; + fnb.baseName(getBaseName()).parameter(field->name()); + visitor.visitDumpFeature(fnb.buildName()); + } + } } fef::Blueprint::UP diff --git a/searchlib/src/vespa/searchlib/features/distancefeature.cpp b/searchlib/src/vespa/searchlib/features/distancefeature.cpp index 7429f32cf4f..d0a2c1a3838 100644 --- a/searchlib/src/vespa/searchlib/features/distancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/distancefeature.cpp @@ -206,7 +206,6 @@ DistanceBlueprint::setup(const IIndexEnvironment & env, { // params[0] = attribute name vespalib::string arg = params[0].getValue(); - bool allow_bad_field = true; if (params.size() == 2) { // params[0] = field / label // params[1] = attribute name / label value @@ -217,7 +216,6 @@ DistanceBlueprint::setup(const IIndexEnvironment & env, return true; } else if (arg == "field") { arg = params[1].getValue(); - allow_bad_field = false; } else { LOG(error, "first argument must be 'field' or 'label', but was '%s'", arg.c_str()); return false; @@ -243,11 +241,6 @@ DistanceBlueprint::setup(const IIndexEnvironment & env, return setup_geopos(env, arg); } } - if (allow_bad_field) { - // TODO remove on Vespa 8 - // backwards compatibility fallback: - return setup_geopos(env, arg); - } if (env.getFieldByName(arg) == nullptr) { LOG(error, "unknown field '%s' for rank feature %s\n", arg.c_str(), getName().c_str()); } else { diff --git a/searchlib/src/vespa/searchlib/features/matchesfeature.cpp b/searchlib/src/vespa/searchlib/features/matchesfeature.cpp index cbbb12e8314..07151e5ae1c 100644 --- a/searchlib/src/vespa/searchlib/features/matchesfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/matchesfeature.cpp @@ -3,6 +3,7 @@ #include "matchesfeature.h" #include "utils.h" #include "valuefeature.h" +#include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/vespalib/util/stash.h> @@ -75,9 +76,17 @@ MatchesBlueprint::MatchesBlueprint() : } void -MatchesBlueprint::visitDumpFeatures(const IIndexEnvironment &, - IDumpFeatureVisitor &) const +MatchesBlueprint::visitDumpFeatures(const IIndexEnvironment& env, + IDumpFeatureVisitor& visitor) const { + for (uint32_t i = 0; i < env.getNumFields(); ++i) { + const auto* field = env.getField(i); + if (field->type() == FieldType::INDEX || field->type() == FieldType::ATTRIBUTE) { + FeatureNameBuilder fnb; + fnb.baseName(getBaseName()).parameter(field->name()); + visitor.visitDumpFeature(fnb.buildName()); + } + } } bool diff --git a/searchlib/src/vespa/searchlib/features/nowfeature.cpp b/searchlib/src/vespa/searchlib/features/nowfeature.cpp index 49495612af3..21415555ceb 100644 --- a/searchlib/src/vespa/searchlib/features/nowfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/nowfeature.cpp @@ -20,9 +20,8 @@ NowExecutor::execute(uint32_t) { } void -NowBlueprint::visitDumpFeatures(const fef::IIndexEnvironment &, fef::IDumpFeatureVisitor &visitor) const +NowBlueprint::visitDumpFeatures(const fef::IIndexEnvironment &, fef::IDumpFeatureVisitor &) const { - visitor.visitDumpFeature(getBaseName()); } bool diff --git a/searchlib/src/vespa/searchlib/grouping/collect.cpp b/searchlib/src/vespa/searchlib/grouping/collect.cpp index 1e8d20dafee..11e297fef2a 100644 --- a/searchlib/src/vespa/searchlib/grouping/collect.cpp +++ b/searchlib/src/vespa/searchlib/grouping/collect.cpp @@ -21,7 +21,7 @@ Collect::ResultAccessor::ResultAccessor(const AggregationResult & aggregator, si void Collect::ResultAccessor::create(uint8_t * base) { _aggregator->getResult().create(base+_offset); - _bluePrint->getResult().encode(base+_offset); + _bluePrint->getResult()->encode(base+_offset); } Collect::Collect(const Group & gp) : @@ -103,7 +103,7 @@ Collect::preFill(GroupRef gr, const Group & g) uint8_t * base(&_aggrBacking[offset]); for (size_t i(0), m(_aggregator.size()); i < m; i++) { ResultAccessor & r = _aggregator[i]; - r.setResult(g.getAggregationResult(i).getResult(), base); + r.setResult(*g.getAggregationResult(i).getResult(), base); } } } diff --git a/searchlib/src/vespa/searchlib/grouping/groupengine.cpp b/searchlib/src/vespa/searchlib/grouping/groupengine.cpp index 180b4d88887..853548b47f7 100644 --- a/searchlib/src/vespa/searchlib/grouping/groupengine.cpp +++ b/searchlib/src/vespa/searchlib/grouping/groupengine.cpp @@ -24,7 +24,7 @@ GroupEngine::GroupEngine(const GroupingLevel * request, size_t level, GroupEngin _frozen(frozen) { if ((request != NULL) && (level > 0)) { - _idScratch.reset(request->getExpression().getResult().clone()); + _idScratch.reset(request->getExpression().getResult()->clone()); } else { _idScratch.reset(new NullResultNode()); } @@ -49,7 +49,7 @@ GroupRef GroupEngine::group(Children & children, uint32_t docId, double rank) if (!selector.execute(docId, rank)) { throw std::runtime_error("Does not know how to handle failed select statements"); } - const ResultNode &selectResult = selector.getResult(); + const ResultNode &selectResult = *selector.getResult(); Children::iterator found = children.find(selectResult); GroupRef gr; if (found == children.end()) { diff --git a/searchlib/src/vespa/searchlib/uca/ucafunctionnode.cpp b/searchlib/src/vespa/searchlib/uca/ucafunctionnode.cpp index 66245595f34..190875eb10c 100644 --- a/searchlib/src/vespa/searchlib/uca/ucafunctionnode.cpp +++ b/searchlib/src/vespa/searchlib/uca/ucafunctionnode.cpp @@ -43,7 +43,7 @@ UcaFunctionNode & UcaFunctionNode::operator = (const UcaFunctionNode & rhs) void UcaFunctionNode::onPrepareResult() { - if (getArg().getResult().inherits(ResultNodeVector::classId)) { + if (getArg().getResult()->inherits(ResultNodeVector::classId)) { setResultType(std::make_unique<RawResultNodeVector>()); _handler = std::make_unique<MultiValueHandler>(*this); } else { @@ -68,7 +68,7 @@ void UcaFunctionNode::Handler::handleOne(const ResultNode & arg, RawResultNode & bool UcaFunctionNode::onExecute() const { getArg().execute(); - _handler->handle(getArg().getResult()); + _handler->handle(*getArg().getResult()); return true; } diff --git a/searchsummary/pom.xml b/searchsummary/pom.xml index 2cfe3ef632f..d7501d19618 100644 --- a/searchsummary/pom.xml +++ b/searchsummary/pom.xml @@ -7,11 +7,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>searchsummary</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/security-tools/pom.xml b/security-tools/pom.xml index 424b5002d1d..7f248c185a2 100644 --- a/security-tools/pom.xml +++ b/security-tools/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>security-tools</artifactId> <packaging>jar</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <!-- compile scope --> <dependency> diff --git a/security-utils/pom.xml b/security-utils/pom.xml index ac856dff6c3..d4b4bc8d0f6 100644 --- a/security-utils/pom.xml +++ b/security-utils/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>security-utils</artifactId> <packaging>bundle</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <!-- provided --> diff --git a/service-monitor/pom.xml b/service-monitor/pom.xml index 067ea451e35..6a46838a0ce 100644 --- a/service-monitor/pom.xml +++ b/service-monitor/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>service-monitor</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>${project.artifactId}</name> <description>Service monitor component for hosted vespa.</description> diff --git a/socket_test/pom.xml b/socket_test/pom.xml index 8e8797564fe..f68b04d01c8 100644 --- a/socket_test/pom.xml +++ b/socket_test/pom.xml @@ -7,11 +7,11 @@ <modelVersion>4.0.0</modelVersion> <groupId>com.yahoo.vespa</groupId> <artifactId>socket_test</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <packaging>jar</packaging> diff --git a/standalone-container/pom.xml b/standalone-container/pom.xml index 25c4b8f801f..86cad2ad012 100644 --- a/standalone-container/pom.xml +++ b/standalone-container/pom.xml @@ -6,11 +6,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>standalone-container</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> @@ -72,6 +72,15 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>org.apache.felix</groupId> + <artifactId>org.apache.felix.framework</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> + </dependency> </dependencies> <build> diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneSubscriberFactory.java b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneSubscriberFactory.java index 5555b3b4077..b1d3f1a6e2c 100644 --- a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneSubscriberFactory.java +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneSubscriberFactory.java @@ -57,7 +57,6 @@ public class StandaloneSubscriberFactory implements SubscriberFactory { return ret; } - @SuppressWarnings("removal") // TODO Vespa 8: remove @Override public long waitNextGeneration(boolean isInitializing) { generation++; diff --git a/standalone-container/src/main/sh/standalone-container.sh b/standalone-container/src/main/sh/standalone-container.sh index bee4c06c38d..cb5d7e3d060 100755 --- a/standalone-container/src/main/sh/standalone-container.sh +++ b/standalone-container/src/main/sh/standalone-container.sh @@ -167,10 +167,15 @@ StartCommand() { FixDataDirectory "$bundlecachedir" FixDataDirectory "$VESPA_HOME/var/crash" + # Note that the hardcoded memory Xmx=2048 migh be overridden by jvm_arguments, + # hence rendering get_jvm_hugepage_settings incorrect, but it is better than not. + # TODO Better way of extracting heap size arguments. + heap_min=128 + heap_max=2048 java \ - -Xms128m -Xmx2048m \ + -Xms${heap_min}m -Xmx${heap_max}m \ -XX:+PreserveFramePointer \ - -XX:+UseTransparentHugePages \ + $(get_jvm_hugepage_settings $heap_max) \ -XX:+HeapDumpOnOutOfMemoryError \ -XX:HeapDumpPath="$VESPA_HOME/var/crash" \ -XX:+ExitOnOutOfMemoryError \ diff --git a/statistics/.gitignore b/statistics/.gitignore deleted file mode 100644 index 324d6034fd1..00000000000 --- a/statistics/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -statistics.iml -archive -build -log -target -.settings -.classpath -.project -/pom.xml.build diff --git a/statistics/OWNERS b/statistics/OWNERS deleted file mode 100644 index 67cd2820bb8..00000000000 --- a/statistics/OWNERS +++ /dev/null @@ -1 +0,0 @@ -arnej27959 diff --git a/statistics/README b/statistics/README deleted file mode 100644 index 0b90b092125..00000000000 --- a/statistics/README +++ /dev/null @@ -1,2 +0,0 @@ -Old metrics gathering and aggregation. -To be replaced with simplemetrics where possible. diff --git a/statistics/abi-spec.json b/statistics/abi-spec.json deleted file mode 100644 index b2ffb43d9cc..00000000000 --- a/statistics/abi-spec.json +++ /dev/null @@ -1,241 +0,0 @@ -{ - "com.yahoo.statistics.Callback": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void run(com.yahoo.statistics.Handle, boolean)" - ], - "fields": [] - }, - "com.yahoo.statistics.Counter": { - "superClass": "com.yahoo.statistics.Handle", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String, com.yahoo.statistics.Statistics, boolean)", - "public void <init>(java.lang.String, com.yahoo.statistics.Statistics, boolean, com.yahoo.statistics.Callback, boolean)", - "public void increment()", - "public void increment(long)", - "public long get()", - "public boolean getResetCounter()", - "public void reset()", - "public void runHandle()", - "public java.lang.String toString()", - "public boolean equals(java.lang.Object)", - "public int hashCode()" - ], - "fields": [] - }, - "com.yahoo.statistics.CounterGroup": { - "superClass": "com.yahoo.statistics.Group", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String, com.yahoo.statistics.Statistics)", - "public void <init>(java.lang.String, com.yahoo.statistics.Statistics, boolean)", - "public void <init>(java.lang.String, com.yahoo.statistics.Statistics, boolean, com.yahoo.statistics.Callback, boolean)", - "public void increment(java.lang.String)", - "public void increment(java.lang.String, long)", - "public void runHandle()", - "public boolean equals(java.lang.Object)", - "public int hashCode()" - ], - "fields": [] - }, - "com.yahoo.statistics.Handle": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "abstract" - ], - "methods": [ - "public final void runCallback()", - "public final void run()", - "public abstract void runHandle()", - "public final boolean cancel()", - "public final boolean isCancelled()", - "public abstract boolean equals(java.lang.Object)", - "public abstract int hashCode()" - ], - "fields": [] - }, - "com.yahoo.statistics.Histogram": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.statistics.Bucket" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.statistics.Limits)", - "public synchronized void put(double[])", - "public void put(double[], int)", - "public java.lang.String toString()", - "public void reset()", - "public double lowerLimit()", - "public double upperLimit()", - "public java.util.List getBuckets()", - "public long getSum()", - "public boolean isLeaf()", - "public void add(long)" - ], - "fields": [] - }, - "com.yahoo.statistics.HistogramType": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public java.lang.String toString()" - ], - "fields": [ - "public static final com.yahoo.statistics.HistogramType REGULAR", - "public static final com.yahoo.statistics.HistogramType CUMULATIVE", - "public static final com.yahoo.statistics.HistogramType REVERSE_CUMULATIVE" - ] - }, - "com.yahoo.statistics.Limits": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(double[])", - "public void addAxis(java.lang.String, double[])", - "public void freeze()", - "public boolean isFrozen()" - ], - "fields": [] - }, - "com.yahoo.statistics.Statistics$NullImplementation": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.statistics.Statistics" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void register(com.yahoo.statistics.Handle)", - "public void remove(java.lang.String)", - "public com.yahoo.container.StatisticsConfig getConfig()", - "public int purge()" - ], - "fields": [] - }, - "com.yahoo.statistics.Statistics": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void register(com.yahoo.statistics.Handle)", - "public abstract void remove(java.lang.String)", - "public abstract com.yahoo.container.StatisticsConfig getConfig()", - "public abstract int purge()" - ], - "fields": [ - "public static final com.yahoo.statistics.Statistics nullImplementation" - ] - }, - "com.yahoo.statistics.StatisticsImpl": { - "superClass": "com.yahoo.component.AbstractComponent", - "interfaces": [ - "com.yahoo.statistics.Statistics" - ], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(com.yahoo.container.StatisticsConfig)", - "public void deconstruct()", - "public void register(com.yahoo.statistics.Handle)", - "public void remove(java.lang.String)", - "public com.yahoo.container.StatisticsConfig getConfig()", - "public int purge()" - ], - "fields": [] - }, - "com.yahoo.statistics.Value$Parameters": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public com.yahoo.statistics.Value$Parameters setLogRaw(java.lang.Boolean)", - "public com.yahoo.statistics.Value$Parameters setLogSum(java.lang.Boolean)", - "public com.yahoo.statistics.Value$Parameters setLogMean(java.lang.Boolean)", - "public com.yahoo.statistics.Value$Parameters setLogMax(java.lang.Boolean)", - "public com.yahoo.statistics.Value$Parameters setLogMin(java.lang.Boolean)", - "public com.yahoo.statistics.Value$Parameters setLogInsertions(java.lang.Boolean)", - "public com.yahoo.statistics.Value$Parameters setNameExtension(java.lang.Boolean)", - "public com.yahoo.statistics.Value$Parameters setLogHistogram(java.lang.Boolean)", - "public com.yahoo.statistics.Value$Parameters setHistogramId(com.yahoo.statistics.HistogramType)", - "public com.yahoo.statistics.Value$Parameters setLimits(com.yahoo.statistics.Limits)", - "public com.yahoo.statistics.Value$Parameters setAppendChar(java.lang.Character)", - "public com.yahoo.statistics.Value$Parameters setCallback(com.yahoo.statistics.Callback)" - ], - "fields": [] - }, - "com.yahoo.statistics.Value": { - "superClass": "com.yahoo.statistics.Handle", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String, com.yahoo.statistics.Statistics, com.yahoo.statistics.Value$Parameters)", - "public static com.yahoo.statistics.Value buildValue(java.lang.String, com.yahoo.statistics.Statistics, com.yahoo.statistics.Value$Parameters)", - "public void put(double)", - "public double getMean()", - "public double getMin()", - "public double getMax()", - "public double get()", - "public void reset()", - "public void runHandle()", - "public java.lang.String toString()", - "public boolean equals(java.lang.Object)", - "public int hashCode()" - ], - "fields": [] - }, - "com.yahoo.statistics.ValueGroup": { - "superClass": "com.yahoo.statistics.Group", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String, com.yahoo.statistics.Statistics)", - "public void <init>(java.lang.String, com.yahoo.statistics.Statistics, com.yahoo.statistics.Callback)", - "public void put(java.lang.String, double)", - "public void runHandle()", - "public boolean equals(java.lang.Object)", - "public int hashCode()" - ], - "fields": [] - } -}
\ No newline at end of file diff --git a/statistics/pom.xml b/statistics/pom.xml deleted file mode 100644 index c32b42ad5be..00000000000 --- a/statistics/pom.xml +++ /dev/null @@ -1,79 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 - http://maven.apache.org/maven-v4_0_0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> - </parent> - <artifactId>statistics</artifactId> - <version>7-SNAPSHOT</version> - <packaging>jar</packaging> - <dependencies> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespalog</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>annotations</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-bundle</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>component</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>config-class-plugin</artifactId> - <version>${project.version}</version> - <executions> - <execution> - <id>config-gen</id> - <goals> - <goal>config-gen</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>abi-check-plugin</artifactId> - </plugin> - </plugins> - </build> -</project> diff --git a/statistics/src/main/java/com/yahoo/statistics/Axis.java b/statistics/src/main/java/com/yahoo/statistics/Axis.java deleted file mode 100644 index f0daff92d00..00000000000 --- a/statistics/src/main/java/com/yahoo/statistics/Axis.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - -import java.util.Arrays; - - -/** - * A wrapper class for representing a single axis of an n-dimensional - * histogram. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -@Deprecated -class Axis { - private final double[] limits; - private final String name; - - Axis(String name, double[] limits) { - this.limits = Arrays.copyOf(limits, limits.length); - this.name = name; - } - - double[] getLimits() { - return limits; - } - - String getName() { - return name; - } - -} diff --git a/statistics/src/main/java/com/yahoo/statistics/Bucket.java b/statistics/src/main/java/com/yahoo/statistics/Bucket.java deleted file mode 100644 index fd69f202d2b..00000000000 --- a/statistics/src/main/java/com/yahoo/statistics/Bucket.java +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - -import java.util.List; - - -/** - * A bucket in a multidimensional histogram. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -@Deprecated -interface Bucket { - void put(double[] value, int dim); - void reset(); - double lowerLimit(); - double upperLimit(); - boolean isLeaf(); - List<Bucket> getBuckets(); - long getSum(); - void add(long n); -} diff --git a/statistics/src/main/java/com/yahoo/statistics/Callback.java b/statistics/src/main/java/com/yahoo/statistics/Callback.java deleted file mode 100644 index fad86d38fd7..00000000000 --- a/statistics/src/main/java/com/yahoo/statistics/Callback.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - -/** - * Definition of the callback interface for the statistics API. It is a common - * use case to need a reference to the Handle (e.g. Value or Counter) which a - * callback is related to. Since everything in a Handle since 5.1.4 is fully - * initialized from the constructor, it became cumbersome to use Runnable for - * the callback and this interface came into use. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @since 5.1.4 - */ -@Deprecated -public interface Callback { - /** - * Invoked each logging cycle right before the events for a Handle are - * emitted to the log. - * - * @param h - * the handle which invoked this callback - * @param firstTime - * true the first time the method is invoked from h, false later - */ - public void run(Handle h, boolean firstTime); -} diff --git a/statistics/src/main/java/com/yahoo/statistics/Counter.java b/statistics/src/main/java/com/yahoo/statistics/Counter.java deleted file mode 100644 index 1425c2a4375..00000000000 --- a/statistics/src/main/java/com/yahoo/statistics/Counter.java +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - -import java.util.concurrent.atomic.AtomicLong; - -import com.yahoo.container.StatisticsConfig; - - -/** - * A single integer value which can be incremented. - * - * @author Steinar Knutsen - */ -@Deprecated -public class Counter extends Handle { - // The current value of this counter - private AtomicLong current = new AtomicLong(0L); - - // Whether or not this counter shall be reset between each logging - // interval - private final boolean resetCounter; - - /** - * A monotonically increasing 64 bit integer value. - * - * @param name - * The name of this counter, for use in logging. - * @param manager - * the statistics manager acquired by injection - * @param fetchParametersFromConfig - * Whether or not this counter should be initialized from config. - */ - public Counter(String name, Statistics manager, boolean fetchParametersFromConfig) { - this(name, manager, fetchParametersFromConfig, null, false, true); - } - - /** - * A monotonically increasing 64 bit integer value. - * - * @param name - * The name of this counter, for use in logging. - * @param manager - * the statistics manager acquired by injection - * @param fetchParametersFromConfig - * Whether or not this counter should be initialized from config. - * @param callback - * will be invoked each time this counter is written to the log - * @param resetCounter - * Control for if this Counter should be reset between each - * logging interval. - */ - public Counter(String name, Statistics manager, - boolean fetchParametersFromConfig, Callback callback, boolean resetCounter) { - this(name, manager, fetchParametersFromConfig, callback, - resetCounter, true); - } - - /** - * A monotonically increasing 64 bit integer value. Do not make this - * constructor public, it is used for creating unregistered counters. - * - * @param name - * The name of this counter, for use in logging. - * @param manager - * the statistics manager acquired by injection - * @param fetchParametersFromConfig - * Whether or not this counter should be initialized from config. - * @param callback - * will be invoked each time this counter is written to the log - * @param resetCounter - * Control for if this Counter should be reset between each - * logging interval. - * @param register - * Whether to register the counter in the statistics manager - */ - private Counter(String name, Statistics manager, - boolean fetchParametersFromConfig, Callback callback, - boolean resetCounter, boolean register) { - super(name, manager, callback); - if (fetchParametersFromConfig) { - StatisticsConfig config = manager.getConfig(); - this.resetCounter = getResetCounter(name, config); - } else { - this.resetCounter = resetCounter; - } - if (register) { - manager.register(this); - } - } - - - /** - * Get a Counter instance not registered in the statistics manager. This is - * used by CounterGroup and should not be made public. - * - * @param name - * The name of this counter, for use in logging. - * @param resetCounter - * Control for if this Counter should be reset between each - * logging interval. - */ - static Counter intializeUnregisteredCounter(String name, - boolean resetCounter) { - return new Counter(name, null, false, null, resetCounter, false); - } - - /** - * If this Counter is set up to read config, configure it - * according to the config given. - */ - private static boolean getResetCounter(String name, StatisticsConfig config) { - for (int i = 0; i < config.counterresets().size(); i++) { - String configName = config.counterresets(i).name(); - if (configName.equals(name)) { - return true; - } - } - return false; - } - - /** - * Increment by 1. - */ - public void increment() { - current.incrementAndGet(); - } - - /** - * Increment by n. - */ - public void increment(long n) { - current.addAndGet(n); - } - - /** - * @return current value of this counter - */ - public long get() { - return current.get(); - } - - /** - * The reset counter is true if this is counter is reset to 0 between each - * logging interval. - * - * @return whether this counter is reset between each logging interval. - */ - public boolean getResetCounter() { - return resetCounter; - } - - /** - * If this counter should be set to 0 between each logging interval, - * do that. - */ - public void reset() { - if (resetCounter) { - current.set(0L); - } - } - - /** - * Log current state and reset. - */ - @Override - public void runHandle() { - boolean resetState = getResetCounter(); - - if (resetState) { - current.set(0L); - } - } - - @Override - public String toString() { - return super.toString() + " " + getName() + " " + current; - } - - CounterProxy getProxyAndReset() { - CounterProxy c = new CounterProxy(getName()); - if (getResetCounter()) { - c.setRaw(current.getAndSet(0L)); - } else { - c.setRaw(current.get()); - } - return c; - } - - @Override - public boolean equals(Object o) { - if (o.getClass() != this.getClass()) { - return false; - } - Counter other = (Counter) o; - return getName().equals(other.getName()); - } - - @Override - public int hashCode() { - return getName().hashCode() + 31 * "Counter".hashCode(); - } - -} diff --git a/statistics/src/main/java/com/yahoo/statistics/CounterGroup.java b/statistics/src/main/java/com/yahoo/statistics/CounterGroup.java deleted file mode 100644 index 08b700dbe3c..00000000000 --- a/statistics/src/main/java/com/yahoo/statistics/CounterGroup.java +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - - -import com.yahoo.container.StatisticsConfig; -import com.yahoo.log.event.Event; - -import java.util.Map; -import java.util.HashMap; -import java.util.Iterator; - - -/** - * A set of associated counters. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -@SuppressWarnings("removal") // TODO Vespa 8: remove (com.yahoo.log.event) -@Deprecated -public class CounterGroup extends Group { - private final boolean resetCounter; - - // A map for names of subevents and Value instances - private Map<String, Counter> subEvents = new HashMap<>(); - - /** - * @param name The symbolic name of this group of counters. - */ - public CounterGroup(String name, Statistics manager) { - this(name, manager, true); - } - - /** - * Create a basic group of counter which may or may not depend on config. - * - * @param name - * The symbolic name of this group of counters. - * @param manager - * the statistics manager acquired by injection - * @param fetchParametersFromConfig - * Whether this Group should be configured from config. - */ - public CounterGroup(String name, Statistics manager, - boolean fetchParametersFromConfig) { - this(name, manager, fetchParametersFromConfig, null, false); - } - - /** - * Create a group of counters with a callback included. - * - * @param name - * The symbolic name of this group of counters. - * @param manager - * the statistics manager acquired by injection - * @param fetchParametersFromConfig - * Whether this Group should be configured from config. - * @param callback - * will be invoked each time data is written to the log - * @param resetCounter - * Control for if this group should be reset between each - * logging interval. - */ - public CounterGroup(String name, Statistics manager, - boolean fetchParametersFromConfig, Callback callback, boolean resetCounter) { - - super(name, manager, callback); - if (fetchParametersFromConfig) { - StatisticsConfig config = manager.getConfig(); - this.resetCounter = getResetCounter(name, config); - } else { - this.resetCounter = resetCounter; - } - manager.register(this); - } - - private static boolean getResetCounter(String name, StatisticsConfig config) { - for (int i = 0; i < config.counterresets().size(); i++) { - String configName = config.counterresets(i).name(); - if (configName.equals(name)) { - return true; - } - } - return false; - } - - - /** - * Increment named contained counter by 1. - */ - public void increment(String name) { - Counter c = getCounter(name); - c.increment(); - } - - /** - * Increment named contained counter by n. - */ - public void increment(String name, long n) { - Counter c = getCounter(name); - c.increment(n); - } - - /** - * Get a counter with a given name, creates a new counter if no - * counter with the name given exists. - */ - synchronized Counter getCounter(String name) { - Counter c = subEvents.get(name); - if (c == null) { - c = getNewCounter(name); - } - return c; - } - - private Counter getNewCounter(String subName) { - Counter c = Counter.intializeUnregisteredCounter(subName, resetCounter); - subEvents.put(subName, c); - return c; - } - - /** - * Dump contained counters to log and reset. - */ - @Override - public void runHandle() { - StringBuilder multi = new StringBuilder(); - CounterProxy[] proxies; - int i = 0; - - // this is to make sure the number of events does not change while logging - synchronized (this) { - proxies = new CounterProxy[subEvents.size()]; - i = 0; - for (Iterator<Counter> j = subEvents.values().iterator(); j - .hasNext();) { - Counter c = j.next(); - proxies[i] = c.getProxyAndReset(); - i++; - } - } - - while (i > 0) { - i--; - if (multi.length() > 0) { - multi.append(", "); - } - multi.append(proxies[i].getName()); - multi.append("="); - multi.append(proxies[i].getRaw()); - } - - Event.countGroup(getName(), multi.toString()); - } - - @Override - public boolean equals(Object o) { - if (o.getClass() != this.getClass()) { - return false; - } - CounterGroup other = (CounterGroup) o; - return getName().equals(other.getName()); - } - - @Override - public int hashCode() { - return getName().hashCode() + 31 * "CounterGroup".hashCode(); - } -} - diff --git a/statistics/src/main/java/com/yahoo/statistics/CounterProxy.java b/statistics/src/main/java/com/yahoo/statistics/CounterProxy.java deleted file mode 100644 index da8a029982a..00000000000 --- a/statistics/src/main/java/com/yahoo/statistics/CounterProxy.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - - -/** - * To be able to cache events concerning Counters internally, group them - * together and similar. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -@Deprecated -class CounterProxy extends Proxy { - private long raw; - private boolean hasRaw = false; - - CounterProxy(String name) { - super(name); - } - - boolean hasRaw() { - return hasRaw; - } - long getRaw() { - return raw; - } - void setRaw(long raw) { - hasRaw = true; - this.raw = raw; - } - -} - diff --git a/statistics/src/main/java/com/yahoo/statistics/Group.java b/statistics/src/main/java/com/yahoo/statistics/Group.java deleted file mode 100644 index 2935521b1e0..00000000000 --- a/statistics/src/main/java/com/yahoo/statistics/Group.java +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - - -/** - * The base class for groups of counters and values. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -@Deprecated -abstract class Group extends Handle { - Group(String name, Statistics manager, Callback parametrizedCallback) { - super(name, manager, parametrizedCallback); - } -} diff --git a/statistics/src/main/java/com/yahoo/statistics/Handle.java b/statistics/src/main/java/com/yahoo/statistics/Handle.java deleted file mode 100644 index d1ad0e26f40..00000000000 --- a/statistics/src/main/java/com/yahoo/statistics/Handle.java +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - -import java.util.TimerTask; - -/** - * Base class for the interface to the statistics framework. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -@Deprecated -public abstract class Handle { - - private TimerTask task; - - private final String name; - private boolean cancelled; - private final Statistics manager; - private final Callback parametrizedCallback; - private boolean firstTime; - - Handle(String name, Statistics manager, Callback parametrizedCallback) { - this.name = name; - this.manager = manager; - this.parametrizedCallback = parametrizedCallback; - firstTime = true; - } - - String getName() { - return name; - } - - TimerTask makeTask() { - final Handle self = this; - synchronized (self) { - if (task != null) { - task.cancel(); - } - task = new TimerTask() { - public void run() { - self.run(); - } - }; - return task; - } - } - - /** - * Run the callback object. - * - * This will happen at the start of each invocation of a Handle's - * run() method. The callback is presumed to be exception safe. - * If no callback is set, this is a no-op. The callback will need - * to handle any necessary synchronization itself. - */ - public final void runCallback() { - if (parametrizedCallback == null) { - return; - } - parametrizedCallback.run(this, firstTime); - firstTime = false; - } - - /** - * Run the callback object first, then invoke runHandle(). - */ - public final void run() { - runCallback(); - runHandle(); - } - - /** - * Invoke an action to be performed periodically for a statistics Handle. - * - * <p>Synchronization has to be handled by the method itself. - */ - public abstract void runHandle(); - - /** - * Cancel this Handle and remove it from internal state in Statistics. - * - * @return value of java.util.TimerTask.cancel() - */ - public final boolean cancel() { - boolean ok = (task == null ? false : task.cancel()); - cancelled = true; - manager.purge(); - return ok; - } - - /** - * Returns whether this object has been cancelled or not. - * - * @return true if cancelled - */ - public final boolean isCancelled() { - return cancelled; - } - - @Override - public abstract boolean equals(Object o); - - @Override - public abstract int hashCode(); -} diff --git a/statistics/src/main/java/com/yahoo/statistics/Histogram.java b/statistics/src/main/java/com/yahoo/statistics/Histogram.java deleted file mode 100644 index 98330e80efc..00000000000 --- a/statistics/src/main/java/com/yahoo/statistics/Histogram.java +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - - -import java.util.ArrayDeque; -import java.util.Deque; -import java.util.Iterator; -import java.util.List; -import java.util.ArrayList; - -/** - * A set of sums or other histograms. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -@Deprecated -public class Histogram implements Bucket { - // The upper and lower limit for the bucket in another histogram - // this histogram represents. The "outermost" histogram in a - // multidimensional histogram will effectively ignore this, but set - // them to -Inf and Inf for consistency. - private final double lower; - private final double upper; - - // The names of all the axes, only used in "outermost" histogram in - // multi dimensional histogram. - private String axes = null; - - private List<Bucket> buckets = new ArrayList<>(); - - /** - * Build a new histogram using bucket limits from the given Limits - * object. - */ - public Histogram(Limits limits) { - // lower and upper will never be used here, - // but it's nicer with defined values - this(limits, 0, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); - - // generate axis names, but only if there is more than one - int guard = limits.getDimensions(); - if (guard == 1) { - return; - } - StringBuffer names = new StringBuffer(); - int i = 0; - while (i < guard) { - names.append(limits.getAxis(i).getName()); - i++; - if (i < guard) { - names.append(","); - } - } - axes = names.toString(); - } - - private Histogram(Limits limits, int dim, double lower, double upper) { - this.lower = lower; - this.upper = upper; - // last axis, create sum objects instead of histograms - boolean lastAxis = dim == (limits.getDimensions() - 1); - Axis next = limits.getAxis(dim); - double[] delimiters = next.getLimits(); - int i = 0; - double previousBucket = Double.NEGATIVE_INFINITY; - dim++; - while (i < delimiters.length) { - if (lastAxis) { - buckets.add(new Sum(previousBucket, delimiters[i])); - } else { - buckets.add(new Histogram(limits, dim, - previousBucket, delimiters[i])); - } - previousBucket = delimiters[i]; - i++; - } - if (lastAxis) { - buckets.add(new Sum(previousBucket, Double.POSITIVE_INFINITY)); - } else { - buckets.add(new Histogram(limits, dim, - previousBucket, - Double.POSITIVE_INFINITY)); - } - - } - - /** - * Increment the corresponding bucket for this data point by 1. - */ - public synchronized void put(double[] value) { - put(value, 0); - } - - /** - * Increment the corresponding bucket for this data point by 1. - * - * @param dim the index of the first value to consider in value array - */ - @Override - public void put(double[] value, int dim) { - Bucket bucket = findBucket(0, buckets.size(), value[dim]); - bucket.put(value, ++dim); - } - - private Bucket findBucket(int offset, int limit, double value) { - int index = offset + (limit - offset) / 2; - Bucket bucket = buckets.get(index); - if (bucket.lowerLimit() <= value && value < bucket.upperLimit()) { - return bucket; - } else if (bucket.upperLimit() <= value) { - return findBucket(index + 1, limit, value); - } else { // value < bucket.lowerLimit() - return findBucket(offset, index, value); - } - } - - @Override - public String toString() { - StringBuffer s; - int i, t; - - s = new StringBuffer(); - if (axes != null) { - s.append(axes).append(" "); - } - s.append("("); - s.append(buckets.get(0).toString()); - s.append(")"); - - t = buckets.size(); - i = 1; - while (i < t) { - Bucket b = buckets.get(i); - s.append(" < "); - s.append(b.lowerLimit()); - s.append(" ("); - s.append(b.toString()); - s.append(")"); - i += 1; - } - - return s.toString(); - } - - /** - * Reset all contained buckets. - */ - @Override - public void reset() { - for (Iterator<Bucket> i = buckets.iterator(); i.hasNext(); ) { - i.next().reset(); - } - } - - /** - * The lower limit for the bucket this histogram represents. - * - * @return the lower limit for the bucket this histogram represents - */ - @Override - public double lowerLimit() { - return lower; - } - - /** - * The upper limit for the bucket this histogram represents. - * - * @return the upper limit for the bucket this histogram represents - */ - @Override - public double upperLimit() { - return upper; - } - - @Override - public List<Bucket> getBuckets() { - return buckets; - } - - private List<Bucket> getLeaves() { - final class Bookmark { - final int i; - final List<Bucket> buckets; - Bookmark(int i, List<Bucket> buckets) { - this.i = i; - this.buckets = buckets; - } - } - List<Bucket> sums = new ArrayList<>(); - Deque<Bookmark> stack = new ArrayDeque<>(); - List<Bucket> current; - int i = 0; - stack.addFirst(new Bookmark(i, buckets)); - while (stack.size() > 0) { - Bookmark currentMark = stack.removeFirst(); - i = currentMark.i; - current = currentMark.buckets; - while (i < current.size()) { - Bucket b = current.get(i++); - if (b.isLeaf()) { - sums.add(b); - } else { - Bookmark marker = new Bookmark(i, current); - stack.addFirst(marker); - i = 0; - current = b.getBuckets(); - } - } - } - return sums; - } - - void merge(Histogram source) { - List<Bucket> src = source.getLeaves(); - List<Bucket> dst = getLeaves(); - if (dst.size() != src.size()) { - throw new IllegalStateException( - "Number of buckets in destination and source not equal. (Source " - + src.size() + ", destination " + dst.size() + "."); - } - for (int i = 0; i < dst.size(); ++i) { - dst.get(i).add(src.get(i).getSum()); - } - } - - @Override - public long getSum() { - throw new RuntimeException("Not implemented."); - } - - @Override - public boolean isLeaf() { - return false; - } - - @Override - public void add(long n) { - throw new IllegalStateException("Can not add directly to a Histogram instance."); - } -} diff --git a/statistics/src/main/java/com/yahoo/statistics/HistogramType.java b/statistics/src/main/java/com/yahoo/statistics/HistogramType.java deleted file mode 100644 index cc7ca04dbbf..00000000000 --- a/statistics/src/main/java/com/yahoo/statistics/HistogramType.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - - -/** - * Enumeration of how a histogram should be represented from - * admin server. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -@Deprecated -public final class HistogramType { - private final String representation; - - private HistogramType(String representation) { - this.representation = representation; - } - - public String toString() { return representation; } - - /** - * Basic histograms, each bucket is count representing the bucket's - * defined interval. - */ - public static final HistogramType REGULAR = new HistogramType("REGULAR"); - - /** - * Cumulative histograms, that is, a given bucket contains the count - * for values corresponding to "itself" and all preceding buckets. - */ - public static final HistogramType CUMULATIVE = - new HistogramType("CUMULATIVE"); - - - /** - * Reverse cumulative histograms, that is, a given bucket contains - * the count for values corresponding to "itself" and all following - * buckets. - */ - public static final HistogramType REVERSE_CUMULATIVE = - new HistogramType("REVERSE_CUMULATIVE"); - -} diff --git a/statistics/src/main/java/com/yahoo/statistics/Limits.java b/statistics/src/main/java/com/yahoo/statistics/Limits.java deleted file mode 100644 index fc79fd00d19..00000000000 --- a/statistics/src/main/java/com/yahoo/statistics/Limits.java +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - - -import java.util.List; -import java.util.ArrayList; - - -/** - * Limits for a histogram, this is only a wrapper for initializing - * a histogram. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -@Deprecated -public class Limits { - private final List<Axis> axes = new ArrayList<>(1); - private boolean frozen = false; - - /** - * Create an empty Limits instance. - */ - public Limits() { - } - - /** - * Create a Limits instance suitable for use in a Value.Parameters instance. - * The instance will be frozen. - */ - public Limits(double[] limits) { - addAxis(null, limits); - freeze(); - } - - /** - * @param name the name of the variable for this axis - * @param limits the bucket limits for this axis - */ - public void addAxis(String name, double[] limits) { - if (frozen) { - throw new IllegalStateException("Can not add more axes to a frozen Limits instance."); - } - axes.add(new Axis(name, limits)); - } - - int getDimensions() { - return axes.size(); - } - - Axis getAxis(int i) { - return axes.get(i); - } - - /** - * Prevent adding any mores axes. - */ - public void freeze() { - this.frozen = true; - } - - /** - * True if further change is not permitted. - * - * @return whether this object now should be considered immutable - */ - public boolean isFrozen() { - return frozen; - } - -} diff --git a/statistics/src/main/java/com/yahoo/statistics/Proxy.java b/statistics/src/main/java/com/yahoo/statistics/Proxy.java deleted file mode 100644 index c4146f4dbf8..00000000000 --- a/statistics/src/main/java/com/yahoo/statistics/Proxy.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - - -/** - * Base class for event proxies, which are used to cache and group - * events internally. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -@Deprecated -abstract class Proxy { - private long timestamp; - private String name; - - Proxy(String name) { - this(name, System.currentTimeMillis()); - } - - Proxy(String name, long timestamp) { - this.timestamp = timestamp; - this.name = name; - } - - long getTimestamp() { - return timestamp; - } - - String getName() { - return name; - } -} diff --git a/statistics/src/main/java/com/yahoo/statistics/SampleDirectory.java b/statistics/src/main/java/com/yahoo/statistics/SampleDirectory.java deleted file mode 100644 index bac08fe17b6..00000000000 --- a/statistics/src/main/java/com/yahoo/statistics/SampleDirectory.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - -import java.util.ArrayList; -import java.util.List; - -/** - * Book-keeping class to know which SampleSet instances have unlogged data. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -@Deprecated -final class SampleDirectory { - private final Object directoryLock = new Object(); - private List<SampleSet> directory = new ArrayList<>(200); - - void put(SampleSet s) { - synchronized (directoryLock) { - directory.add(s); - s.setRegisteredForLogging(true); - } - } - - /** - * Get a view of the current generation of data and instantiate a new - * generation. This does the memory barrier two-step for the - * client. - */ - SampleSet.Sampling[] fetchValues() { - SampleSet.Sampling[] copyToReturn; - synchronized (directoryLock) { - List<SampleSet> tmpDir = directory; - copyToReturn = new SampleSet.Sampling[tmpDir.size()]; - List<SampleSet> newDir = new ArrayList<>(200); - for (int i = 0; i < copyToReturn.length; ++i) { - copyToReturn[i] = tmpDir.get(i).getAndReset(); - } - directory = newDir; - } - return copyToReturn; - } - - /** - * Return a view of the current generation of data. This does the memory - * barrier two-step for the client. - */ - SampleSet.Sampling[] viewValues() { - SampleSet.Sampling[] copy; - synchronized (directoryLock) { - copy = new SampleSet.Sampling[directory.size()]; - for (int i = 0; i < copy.length; ++i) { - copy[i] = directory.get(i).values; - } - } - return copy; - } - -} diff --git a/statistics/src/main/java/com/yahoo/statistics/SampleSet.java b/statistics/src/main/java/com/yahoo/statistics/SampleSet.java deleted file mode 100644 index 768ae1433ea..00000000000 --- a/statistics/src/main/java/com/yahoo/statistics/SampleSet.java +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - -/** - * A running set of samples for a Value instance. It is - * used only in a very specific context between the sampling threads (each instance - * is only used by on sampling thread) and the single logging thread. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -@Deprecated -final class SampleSet { - Sampling values; - final Limits histogramLimits; - private boolean isRegisteredForLogging = false; - - SampleSet(Limits histogramLimits) { - this.histogramLimits = histogramLimits; - values = new Sampling(0.0d, 0L, 0.0d, 0.0d, histogramLimits); - } - - static final class Sampling { - final double sum; - final long insertions; - final double max; - final double min; - final Histogram histogram; - - Sampling(double sum, long insertions, double max, double min, Limits histogramLimits) { - if (histogramLimits != null) { - this.histogram = new Histogram(histogramLimits); - } else { - this.histogram = null; - } - this.sum = sum; - this.insertions = insertions; - this.max = max; - this.min = min; - } - - Sampling(double sum, long insertions, double max, double min, Histogram histogram) { - this.histogram = histogram; - this.sum = sum; - this.insertions = insertions; - this.max = max; - this.min = min; - } - } - - - private Sampling createSampling(double x, Sampling previous) { - double sum = previous.sum; - long insertions = previous.insertions; - double max = previous.max; - double min = previous.min; - - sum += x; - if (insertions == 0) { - max = x; - min = x; - } else { - max = Math.max(x, max); - min = Math.min(x, min); - } - insertions++; - return new Sampling(sum, insertions, max, min, previous.histogram); - } - - /** - * Insert x, do all pertinent operations. (Update histogram, update - * insertion count for calculating mean, etc.) - * - * @return whether this is registered for logging - */ - synchronized boolean put(double x) { - Sampling previous = values; - Histogram histogram = previous.histogram; - if (histogram != null) { - histogram.put(new double[] {x}); - } - values = createSampling(x, previous); - return isRegisteredForLogging; - } - - /** - * Get state and reset it. - */ - synchronized Sampling getAndReset() { - Sampling previous = values; - values = new Sampling(0.0d, 0L, 0.0d, 0.0d, histogramLimits); - setRegisteredForLogging(false); - return previous; - } - - // Setting this state is protected by SampleDirectory.directoryLock. It is - // either set from the logging thread (protected by directoryLock and using - // "this" as a memory barrier through getAndReset()), or from the sampling - // thread protected by directoryLock and without a mem barrier. - void setRegisteredForLogging(boolean isRegisteredForLogging) { - this.isRegisteredForLogging = isRegisteredForLogging; - } - -} diff --git a/statistics/src/main/java/com/yahoo/statistics/Statistics.java b/statistics/src/main/java/com/yahoo/statistics/Statistics.java deleted file mode 100644 index 92d320e0647..00000000000 --- a/statistics/src/main/java/com/yahoo/statistics/Statistics.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - -import com.yahoo.container.StatisticsConfig; - -/** - * Interface used for registering statistics values and counters for logging. - * - * @author steinar - * @author Tony Vaagenes - * @deprecated Will be removed on Vespa 8. If required by a method, there exists an alternative to be used instead. - */ -@Deprecated -public interface Statistics { - /** - * Add a new handle to be scheduled for periodic logging. If a handle - * already exists with the same name, it will be cancelled and removed from - * the internal state of this object. - */ - void register(Handle h); - - /** - * Remove a named handler from the set of working handlers. - */ - void remove(String name); - - /** - * Get current config used. This may be a null reference, depending on how - * the instance was constructed. - */ - StatisticsConfig getConfig(); - - /** - * Purges all cancelled Handles from internal Map and Timer. - * - * @return return value from java.util.Timer.purge() - */ - int purge(); - - /** A null implementation which ignores all calls and returns the default config */ - public static Statistics nullImplementation=new NullImplementation(); - - static class NullImplementation implements Statistics { - - private StatisticsConfig nullConfig=new StatisticsConfig(new StatisticsConfig.Builder()); - - @Override - public void register(Handle h) { } - - @Override - public void remove(String name) { } - - @Override - public StatisticsConfig getConfig() { return nullConfig; } - - @Override - public int purge() { return 0; } - - } - -} diff --git a/statistics/src/main/java/com/yahoo/statistics/StatisticsImpl.java b/statistics/src/main/java/com/yahoo/statistics/StatisticsImpl.java deleted file mode 100644 index f3a0b23a551..00000000000 --- a/statistics/src/main/java/com/yahoo/statistics/StatisticsImpl.java +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.Timer; -import java.util.logging.Level; -import java.util.logging.Logger; - -import com.yahoo.component.AbstractComponent; -import com.yahoo.container.StatisticsConfig; - -/** - * Centralized book keeping for statistics module. Times logging and reads - * config. It is normally obtained through injection, but may be initialized - * manually for testing or when an injection mechanism is unavailable. Logging - * will be disabled by initializing the Statistics class with a null config - * object. - * - * @author Steinar Knutsen - */ -@SuppressWarnings("deprecation") -public final class StatisticsImpl extends AbstractComponent implements Statistics { - - private final Timer worker; - private final StatisticsConfig config; - private static final Logger log = Logger.getLogger(StatisticsImpl.class.getName()); - private final int collectioninterval; - private final int logginginterval; - // default access for testing only - final Map<String, Handle> handles = new HashMap<>(); - - /** - * Build a statistics manager based on the given config values. Use a config - * builder for testing if logging is necessary, set the config argument to - * the constructor to null is logging is not necessary. - * - * @since 5.1.4 - * @param config - * settings for logging interval and configured events. Setting - * it to null disables logging. - * @throws IllegalArgumentException - * if logging interval is smaller than collection interval, or - * collection interval is not a multiplum of logging interval - */ - public StatisticsImpl(StatisticsConfig config) { - int l = (int) config.loggingintervalsec(); - int c = (int) config.collectionintervalsec(); - - if (l != 0 && l < c) { - throw new IllegalArgumentException( - "Logging interval (" + l + ") smaller than collection interval (" + c + ")." - + " New config ignored."); - } - if ((l % c) != 0) { - throw new IllegalArgumentException( - "Collection interval (" + c + ") not multiplum of logging interval (" + l + ")." - + " New config ignored."); - } - this.logginginterval = l; - this.collectioninterval = c; - this.config = config; - this.worker = new Timer(true); - } - - /** - * Cancel internal worker thread and do any other necessary cleanup. The - * internal worker thread is a daemon thread, so not calling this will not - * hamper a clean exit from the VM. - */ - @Override - public void deconstruct() { - worker.cancel(); - } - - private void schedule(Handle h) { - if (logginginterval != 0) { - h.run(); - // We use the rather creative assumption that there is - // exactly 24h pr day+night. - final Date d = new Date(); - final long ms = collectioninterval * 1000L; - final long delay = ms - (d.getTime() % (ms)); - worker.scheduleAtFixedRate(h.makeTask(), delay, ms); - } - } - - /** - * Add a new handle to be scheduled for periodic logging. If a handle - * already exists with the same name, it will be cancelled and removed from - * the internal state of this object. - */ - @Override - public void register(Handle h) { - synchronized (handles) { - Handle oldHandle = handles.get(h.getName()); - if (oldHandle == h) { - log.log(Level.WARNING, "Handle [" + h + "] already registered"); - return; - } - if (oldHandle != null) { - oldHandle.cancel(); - } - handles.put(h.getName(), h); - if (worker != null) { - schedule(h); - } - } - } - - /** - * Remove a named handler from the set of working handlers. - */ - @Override - public void remove(String name) { - synchronized (handles) { - Handle oldHandle = handles.remove(name); - if (oldHandle != null) { - oldHandle.cancel(); - } - } - } - - /** - * Get current config used. This may be a null reference, depending on how - * the instance was constructed. - */ - @Override - public StatisticsConfig getConfig() { - return config; - } - - /** - * Purges all cancelled Handles from internal Map and Timer. - * - * @return return value from java.util.Timer.purge() - */ - @Override - public int purge() { - synchronized (handles) { - Iterator<Handle> it = handles.values().iterator(); - while (it.hasNext()) { - final Handle h = it.next(); - if (h.isCancelled()) { - it.remove(); - } - } - return worker == null ? 0 : worker.purge(); - } - } - -} diff --git a/statistics/src/main/java/com/yahoo/statistics/Sum.java b/statistics/src/main/java/com/yahoo/statistics/Sum.java deleted file mode 100644 index 5661e82cf1b..00000000000 --- a/statistics/src/main/java/com/yahoo/statistics/Sum.java +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - -import java.util.List; - - -/** - * The innermost part of a histogram, a bucket which only contains a - * counter. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -@Deprecated -class Sum implements Bucket { - private long sum = 0L; - private double lower; - private double upper; - - Sum(double lower, double upper) { - this.lower = lower; - this.upper = upper; - } - - /** - * Increment this bucket. - */ - public void put(double[] value, int dim) { - sum += 1; - } - - /** - * Set this bucket's count to 0. - */ - public void reset() { - sum = 0L; - } - - /** - * The lower limit for values counted by this bucket. - */ - public double lowerLimit() { - return lower; - } - - /** - * The upper limit for values counted by this bucket. - */ - public double upperLimit() { - return upper; - } - - public String toString() { - return Long.toString(sum); - } - - @Override - public List<Bucket> getBuckets() { - return null; - } - - @Override - public boolean isLeaf() { - return true; - } - - @Override - public long getSum() { - return sum; - } - - @Override - public void add(long n) { - sum += n; - } -} diff --git a/statistics/src/main/java/com/yahoo/statistics/Value.java b/statistics/src/main/java/com/yahoo/statistics/Value.java deleted file mode 100644 index 9b41f526f13..00000000000 --- a/statistics/src/main/java/com/yahoo/statistics/Value.java +++ /dev/null @@ -1,765 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - -import java.util.HashMap; -import java.util.List; -import java.util.logging.Logger; - -import com.yahoo.container.StatisticsConfig; -import com.yahoo.container.StatisticsConfig.Values.Operations; -import java.util.logging.Level; - -/** - * A statistical variable, typically representing a sampling of an - * arbitrarily changing parameter. - * - * @author Steinar Knutsen - */ -@Deprecated -public class Value extends Handle { - - // For accumulated values, SampleSet instances are mem barriers between {n - // sampling threads} and {logging thread}. - - // lastValue is a memory barrier between {n sampling threads} and {n - // sampling threads, logging thread}. - - // Therefore, the logging thread first locks SampleDirectory.directoryLock, - // then locks each SampleSet, one by one. The sampling threads _either_ lock - // a single SampleSet _or_ lock SampleDirectory.directoryLock. - - // It is necessary to create a memory relationship between the logging - // threads to ensure the newest sample ends up in the log for logRaw = true. - - private final ThreadLocal<SampleSet> sample = new ThreadLocal<>(); - private final SampleDirectory directory = new SampleDirectory(); - - // This must _only_ be touched if logRaw is true - private volatile double lastValue = 0.0d; - - private final boolean logRaw; - private final boolean logMean; - private final boolean logSum; - private final boolean logInsertions; - private final boolean logMax; - private final boolean logMin; - private final boolean logHistogram; - - private final Limits histogram; - final HistogramType histogramId; - - private static final Logger log = Logger.getLogger(Value.class.getName()); - static final String HISTOGRAM_TYPE_WARNING = "Histogram types other than REGULAR currently not supported." - +" Reverting to regular histogram for statistics event"; - - /** - * Parameters for building Value instances. All settings are classes instead - * of primitive types to allow tri-state logic (true, false, unset). - */ - public static class Parameters { - /** - * Log raw values. Raw values are basically the last value logged. - */ - Boolean logRaw; - /** - * Log the sum of all data points for each interval. - */ - Boolean logSum; - /** - * Log the mean value for each interval. - */ - Boolean logMean; - /** - * Log the maximal value observed for each interval. - */ - Boolean logMax; - /** - * Log the minimal value observed for each interval. - */ - Boolean logMin; - /** - * Log the number of observations for each interval. - */ - Boolean logInsertions; - /** - * Whether or not to add an identifying extension (like mean) to event - * names when logging derived values. - * - * It is useful to disable extensions if a only a single dervied value, - * e.g. the mean, is the only thing to be logged. The default is to use - * extensions. - * - * If extensions are disabled, the ability to log more than one of raw - * value, min, max, mean (i.e. the raw value and derived values of the - * same type) is disabled to avoid confusion. Since histograms are not - * Value events, these never have a name extension and are always - * available. - */ - Boolean nameExtension; - /** - * Log a data histogram. - */ - Boolean logHistogram; - /** - * What kind of histogram to log. - * - * @see HistogramType - */ - HistogramType histogramId; - /** - * The limits to use if logging as a histogram. The Limits instance must - * be frozen before using the Parameters instance in a Value constructor - * call. - * - * @see Limits - */ - Limits limits; - /** - * Separator character to use between event name and type of - * nameExtension is set to true. - */ - Character appendChar; - - /** - * This is invoked each time a value is dumped to the log. - * - * @see Handle#runCallback() - */ - Callback callback; - - /** - * Whether to register in the Statistics manager. This is not touched by - * merge and also has no undefined state. In general, a Value should - * always register and not doing so explicitly should not be part of the - * public API. - */ - private boolean register = true; - - /** - * Get a fresh Parameters instance with all features turned/unset. - * Parameters instances may be recycled for construction multiple Value - * instances, but do note any Limits instance added must be frozen. - */ - public Parameters() { - } - - /** - * (De-)Activate logging of raw values. Raw values are basically the - * last value logged. - * - * @return "this" for call chaining - */ - public Parameters setLogRaw(Boolean logRaw) { - this.logRaw = logRaw; - return this; - } - - /** - * (De-)Activate logging the sum of all data points for each interval. - * - * @return "this" for call chaining - */ - public Parameters setLogSum(Boolean logSum) { - this.logSum = logSum; - return this; - } - - /** - * (De)-activate loging the mean value for each interval. - * - * @return "this" for call chaining - */ - public Parameters setLogMean(Boolean logMean) { - this.logMean = logMean; - return this; - } - - /** - * (De-)Activate logging the maximal value observed for each interval. - * - * @return "this" for call chaining - */ - public Parameters setLogMax(Boolean logMax) { - this.logMax = logMax; - return this; - } - - /** - * (De-)Activate logging the minimal value observed for each interval. - * - * @return "this" for call chaining - */ - public Parameters setLogMin(Boolean logMin) { - this.logMin = logMin; - return this; - } - - /** - * (De-)Activate loging the number of observations for each interval. - * - * @return "this" for call chaining - */ - public Parameters setLogInsertions(Boolean logInsertions) { - this.logInsertions = logInsertions; - return this; - } - - /** - * Whether or not to add an identifying extension (like mean) to event - * names when logging derived values. - * - * It is useful to disable extensions if a only a single dervied value, - * e.g. the mean, is the only thing to be logged. The default is to use - * extensions. - * - * If extensions are disabled, the ability to log more than one of raw - * value, min, max, mean (i.e. the raw value and derived values of the - * same type) is disabled to avoid confusion. Since histograms are not - * Value events, these never have a name extension and are always - * available. - * - * @return "this" for call chaining - */ - public Parameters setNameExtension(Boolean nameExtension) { - this.nameExtension = nameExtension; - return this; - } - - /** - * (De-)Activate logging a data histogram. - * - * @return "this" for call chaining - */ - public Parameters setLogHistogram(Boolean logHistogram) { - this.logHistogram = logHistogram; - return this; - } - - /** - * What kind of histogram to log. - * - * @see HistogramType - * - * @return "this" for call chaining - */ - public Parameters setHistogramId(HistogramType histogramId) { - this.histogramId = histogramId; - return this; - } - - /** - * The limits to use if logging as a histogram. The Limits instance must - * be frozen before using the Parameters instance in a Value constructor - * call. - * - * @return "this" for call chaining* - * @see Limits - */ - public Parameters setLimits(Limits limits) { - this.limits = limits; - return this; - } - - /** - * Separator character to use between event name and type of - * nameExtension is set to true. The default is '.'. - * - * @return "this" for call chaining - */ - public Parameters setAppendChar(Character appendChar) { - this.appendChar = appendChar; - return this; - } - - /** - * Set a callback to be invoked each time this Value is written to the - * log. - * - * @param callback - * to be invoked each time the Value is written to the log - * @return "this" for call chaining - */ - public Parameters setCallback(Callback callback) { - this.callback = callback; - return this; - } - - /** - * Set whether to register in the statistics manager. Do note the - * package private access for this method opposed to all the other - * setters. - * - * @param register - * set to false to avoid registering - * @return "this" for call chaining - */ - private Parameters setRegister(boolean register) { - this.register = register; - return this; - } - - /** - * If a member is not set in this, add it from defaults. Do note, this - * applies for both true and false settings, in other words, the default - * may also be used to turn off something if it is undefined in this. - */ - void merge(Parameters defaults) { - if (defaults == null) { - return; - } - this.logRaw = this.logRaw == null ? defaults.logRaw : this.logRaw; - this.logSum = this.logSum == null ? defaults.logSum : this.logSum; - this.logMean = this.logMean == null ? defaults.logMean - : this.logMean; - this.logMax = this.logMax == null ? defaults.logMax : this.logMax; - this.logMin = this.logMin == null ? defaults.logMin : this.logMin; - this.logInsertions = this.logInsertions == null ? defaults.logInsertions - : this.logInsertions; - this.nameExtension = this.nameExtension == null ? defaults.nameExtension - : this.nameExtension; - this.logHistogram = this.logHistogram == null ? defaults.logHistogram - : this.logHistogram; - this.histogramId = this.histogramId == null ? defaults.histogramId - : this.histogramId; - this.limits = this.limits == null ? defaults.limits : this.limits; - this.appendChar = this.appendChar == null ? defaults.appendChar - : this.appendChar; - this.callback = this.callback == null ? defaults.callback - : this.callback; - } - - } - - /** - * Configure a Value instance fully, no raw config access. - * - * @param name - * tag for logging - * @param manager - * the statistics manager acquired by injection - * @param parameters - * all the parameters necessary for initializing a Value - * @throws IllegalStateException - * if Parameters.limits exists and is not frozen - */ - public Value(String name, Statistics manager, Parameters parameters) { - super(name, manager, parameters.callback); - this.logHistogram = isTrue(parameters.logHistogram); - this.logMax = isTrue(parameters.logMax); - this.logMean = isTrue(parameters.logMean); - this.logMin = isTrue(parameters.logMin); - this.logRaw = isTrue(parameters.logRaw); - this.logSum = isTrue(parameters.logSum); - this.logInsertions = isTrue(parameters.logInsertions); - if (logHistogram) { - if (!parameters.limits.isFrozen()) { - throw new IllegalStateException("The Limits instance must be frozen."); - } - if (parameters.histogramId != HistogramType.REGULAR) { - log.log(Level.WARNING, HISTOGRAM_TYPE_WARNING + " '" + name + "'"); - } - this.histogramId = HistogramType.REGULAR; - this.histogram = parameters.limits; - } else { - this.histogram = null; - this.histogramId = HistogramType.REGULAR; - } - - if (parameters.register) { - manager.register(this); - } - } - - private static boolean isTrue(Boolean b) { - return b != null && b; - } - - /** - * Build a Value which should be initialized from config. - * - * @param name - * the name of the event in the log - * @param manager - * the current Statistics manager, acquired by injection - * @param defaults - * defaults for values not defined by config, this may be null - */ - public static Value buildValue(String name, Statistics manager, Parameters defaults) { - return new Value(name, manager, buildParameters(name, manager, defaults)); - } - - /** - * Get a Value instance not registered in the statistics manager. This is - * used by ValueGroup and should not be made public. - * - * @param name - * The name of this counter, for use in logging. - * @param parameters - * setting for the new Value - */ - static Value initializeUnregisteredValue(String name, Parameters parameters) { - return new Value(name, null, parameters.setRegister(false)); - } - - private static Parameters buildParameters(String name, Statistics manager, Parameters defaults) { - Parameters p = null; - StatisticsConfig config = manager.getConfig(); - if (config != null) { - for (int i = 0; i < config.values().size(); i++) { - String configName = config.values(i).name(); - if (configName.equals(name)) { - p = parametersFromConfig(config.values(i).operations()); - break; - } - } - } - if (p == null) { - if (defaults == null) { - p = defaultParameters(); - } else { - p = defaults; - } - } else { - p.merge(defaults); - } - return p; - } - - static Parameters defaultParameters() { - return new Parameters().setLogRaw(true).setNameExtension(true); - } - - private static Parameters parametersFromConfig(List<Operations> o) { - Parameters p = new Parameters().setNameExtension(true); - - for (Operations operation : o) { - Operations.Name.Enum opName = operation.name(); - - HashMap<String, String> args = new HashMap<>(); - for (int j = 0; j < operation.arguments().size(); j++) { - args.put(operation.arguments(j).key(), operation.arguments(j).value()); - } - - if (opName == Operations.Name.MEAN) { - p.setLogMean(true); - } else if (opName == Operations.Name.MAX) { - p.setLogMax(true); - } else if (opName == Operations.Name.MIN) { - p.setLogMin(true); - } else if (opName == Operations.Name.RAW) { - p.setLogRaw(true); - } else if (opName == Operations.Name.SUM) { - p.setLogSum(true); - } else if (opName == Operations.Name.INSERTIONS) { - p.setLogInsertions(true); - } else if (opName == Operations.Name.REGULAR) { - p.setLogHistogram(true); - p.setHistogramId(HistogramType.REGULAR); - p.setLimits(initHistogram(args.get("axes"), args.get("limits"))); - } else if (opName == Operations.Name.CUMULATIVE) { - p.setLogHistogram(true); - p.setHistogramId(HistogramType.CUMULATIVE); - p.setLimits(initHistogram(args.get("axes"), args.get("limits"))); - } else if (opName == Operations.Name.REVERSE_CUMULATIVE) { - p.setLogHistogram(true); - p.setHistogramId(HistogramType.REVERSE_CUMULATIVE); - p.setLimits(initHistogram(args.get("axes"),args.get("limits"))); - } - } - return p; - } - - private static Limits initHistogram(String axes, String limits) { - String[] borders; - double[] vanillaLimits; - Limits l = new Limits(); - int i = 0; - - if (axes != null) { - throw new RuntimeException("Config of multidimensional histograms not yet implemented."); - } - if (limits == null) { - throw new RuntimeException("Config of histograms needs a list of limits."); - } - borders = limits.split(","); - vanillaLimits = new double[borders.length]; - while (i < vanillaLimits.length) { - vanillaLimits[i] = Double.parseDouble(borders[i].trim()); - ++i; - } - l.addAxis(null, vanillaLimits); - l.freeze(); - return l; - } - - private SampleSet getSample() { - SampleSet s = sample.get(); - if (s == null) { - s = new SampleSet(histogram); - sample.set(s); - } - return s; - } - - private void putComposite(double x) { - SampleSet s = getSample(); - boolean isInDir = s.put(x); - if (!isInDir) { - directory.put(s); - } - } - - /** - * Insert x, do all pertinent operations. (Update histogram, update - * insertion count for calculating mean, etc.) - */ - public void put(double x) { - if (logComposite()) { - putComposite(x); - } - if (logRaw) { - lastValue = x; - } - } - - private boolean logComposite() { - return logMean || logMin || logMax || logSum || logInsertions || logHistogram; - } - - /** - * Get mean value since last reset. - */ - public double getMean() { - SampleSet.Sampling[] values = directory.viewValues(); - long insertions = 0L; - double sum = 0.0d; - for (var x : values) { - insertions += x.insertions; - sum += x.sum; - } - if (insertions == 0) { - return 0.0d; - } - return sum/insertions; - } - - /** - * Get minimal value logged since last reset. - */ - public double getMin() { - SampleSet.Sampling[] values = directory.viewValues(); - long insertions = 0L; - double min = 0.0d; - for (var x : values) { - if (x.insertions == 0) { - continue; - } - if (insertions == 0) { - min = x.min; - } else { - min = Math.min(x.min, min); - } - insertions += x.insertions; - } - return min; - } - - /** - * Get maximum value logged since last reset. - */ - public double getMax() { - SampleSet.Sampling[] values = directory.viewValues(); - long insertions = 0L; - double max = 0.0d; - for (var x : values) { - if (x.insertions == 0) { - continue; - } - if (insertions == 0) { - max = x.max; - } else { - max = Math.max(x.max, max); - } - insertions += x.insertions; - } - return max; - } - - private Histogram getHistogram() { - if (histogram == null) { - return null; - } else { - SampleSet.Sampling[] values = directory.viewValues(); - Histogram merged = new Histogram(histogram); - for (var s : values) { - merged.merge(s.histogram); - } - return merged; - } - } - - /** - * Get last value logged, 0 if nothing logged since reset. - */ - public double get() { - return lastValue; - } - - /** - * Set last value logged container to 0, reset histogram and set all - * counters and derived statistics to 0. - */ - public void reset() { - if (logComposite()) { - directory.fetchValues(); - } - if (logRaw) { - lastValue = 0.0d; - } - } - - /** - * Dump state to log and reset. - */ - @Override - public void runHandle() { - getAndSetCurrentState(); - } - - public String toString() { - if (histogram == null) { - return super.toString() + " " + getName(); - } else { - return super.toString() + " " + getName() + " " + getHistogram().toString(); - } - } - - @Override - public boolean equals(Object o) { - if (o.getClass() != this.getClass()) { - return false; - } - Value other = (Value) o; - return getName().equals(other.getName()); - } - - @Override - public int hashCode() { - return getName().hashCode() + 31 * "Value".hashCode(); - } - - static class Snapshot { - double insertions; - double max; - double min; - double mean; - double sum; - double raw; - Histogram histogram = null; - - Snapshot insertions(double lastInsertions) { - this.insertions = lastInsertions; - return this; - } - - Snapshot max(double lastMax) { - this.max = lastMax; - return this; - } - - Snapshot min(double lastMin) { - this.min = lastMin; - return this; - } - - Snapshot mean(double lastMean) { - this.mean = lastMean; - return this; - } - - Snapshot sum(double lastSum) { - this.sum = lastSum; - return this; - } - - Snapshot raw(double lastRaw) { - this.raw = lastRaw; - return this; - } - - Snapshot histogram(Histogram mergedHistogram) { - this.histogram = mergedHistogram; - return this; - } - } - - private Snapshot getAndSetCurrentState() { - double lastInsertions = 0L; - double lastMax = 0.0d; - double lastMin = 0.0d; - double lastMean = 0.0d; - double lastSum = 0.0d; - double lastRaw = 0.0d; - Histogram mergedHistogram = null; - - if (logRaw) { - lastRaw = lastValue; - } - if (logComposite()) { - SampleSet.Sampling[] lastInterval = directory.fetchValues(); - if (histogram != null) { - mergedHistogram = new Histogram(histogram); - } - for (var threadData : lastInterval) { - if (threadData.insertions == 0) { - continue; - } - if (lastInsertions == 0L) { - lastMax = threadData.max; - lastMin = threadData.min; - } else { - lastMax = Math.max(threadData.max, lastMax); - lastMin = Math.min(threadData.min, lastMin); - } - lastSum += threadData.sum; - if (mergedHistogram != null) { - mergedHistogram.merge(threadData.histogram); - } - lastInsertions += threadData.insertions; - } - if (lastInsertions == 0L) { - lastMean = 0.0d; - } else { - lastMean = lastSum / lastInsertions; - } - } - return new Snapshot().insertions(lastInsertions) - .max(lastMax).mean(lastMean).min(lastMin) - .raw(lastRaw).sum(lastSum) - .histogram(mergedHistogram); - } - - ValueProxy getProxyAndReset() { - ValueProxy p = new ValueProxy(getName()); - Snapshot now = getAndSetCurrentState(); - - if (logRaw) { - p.setRaw(now.raw); - } - if (logMean) { - p.setMean(now.mean); - } - if (logMin) { - p.setMin(now.min); - } - if (logMax) { - p.setMax(now.max); - } - if (logHistogram) { - p.setHistogram(now.histogram); - } - - return p; - } -} - diff --git a/statistics/src/main/java/com/yahoo/statistics/ValueGroup.java b/statistics/src/main/java/com/yahoo/statistics/ValueGroup.java deleted file mode 100644 index ea12c1e4c89..00000000000 --- a/statistics/src/main/java/com/yahoo/statistics/ValueGroup.java +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - - -import com.yahoo.log.event.Event; - -import java.util.Iterator; -import java.util.Map; -import java.util.HashMap; - -/** - * A set of related values which should be logged together. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -@SuppressWarnings("removal") // TODO Vespa 8: remove (com.yahoo.log.event) -@Deprecated -public class ValueGroup extends Group { - // A map for names of subevents and Value instances - private Map<String, Value> subEvents = new HashMap<>(); - - /** - * Create a ValueGroup. - * - * @param name - * The symbolic name of this group of values - * @param manager - * the statistics manager acquired by injection - */ - public ValueGroup(String name, Statistics manager) { - this(name, manager, null); - } - - /** - * Create a ValueGroup. - * - * @param name - * The symbolic name of this group of values - * @param manager - * the statistics manager acquired by injection - * @param callback - * will be invoked each time data is written to the log - */ - public ValueGroup(String name, Statistics manager, Callback callback) { - super(name, manager, callback); - manager.register(this); - } - - /** - * Put a value into the named value in the group. - */ - public void put(String name, double x) { - Value v = getValue(name); - v.put(x); - } - - /** - * Get a value with a given name, creates a new value if no - * value with the name given exists. - * - */ - synchronized Value getValue(String name) { - Value v = subEvents.get(name); - if (v == null) { - v = getNewValue(name); - } - return v; - } - - private Value getNewValue(String subName) { - Value v = Value.initializeUnregisteredValue(subName, new Value.Parameters().setLogRaw(true)); - subEvents.put(subName, v); - return v; - } - - /** - * Dump state to log and reset. - */ - @Override - public void runHandle() { - StringBuilder multi = new StringBuilder(); - ValueProxy[] proxies; - int i = 0; - - synchronized (this) { - proxies = new ValueProxy[subEvents.size()]; - i = 0; - for (Iterator<Value> j = subEvents.values().iterator(); j.hasNext();) { - Value v = j.next(); - proxies[i] = v.getProxyAndReset(); - i++; - } - } - - while (i > 0) { - i--; - if (multi.length() > 0) { - multi.append(", "); - } - multi.append(proxies[i].getName()); - multi.append("="); - multi.append(proxies[i].getRaw()); - } - - Event.valueGroup(getName(), multi.toString()); - } - - @Override - public boolean equals(Object o) { - if (o.getClass() != this.getClass()) { - return false; - } - ValueGroup other = (ValueGroup) o; - return getName().equals(other.getName()); - } - - @Override - public int hashCode() { - return getName().hashCode() + 31 * "ValueGroup".hashCode(); - } -} diff --git a/statistics/src/main/java/com/yahoo/statistics/ValueProxy.java b/statistics/src/main/java/com/yahoo/statistics/ValueProxy.java deleted file mode 100644 index 3146dc7d2d7..00000000000 --- a/statistics/src/main/java/com/yahoo/statistics/ValueProxy.java +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - - -/** - * To be able to cache events concerning Values internally, group them - * together and similar. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -@Deprecated -class ValueProxy extends Proxy { - private double raw; - private boolean hasRaw = false; - private double min; - private boolean hasMin = false; - private double mean; - private boolean hasMean = false; - private double max; - private boolean hasMax = false; - private Histogram histogram; - private boolean hasHistogram; - - ValueProxy(String name) { - super(name); - } - - boolean hasRaw() { - return hasRaw; - } - double getRaw() { - return raw; - } - void setRaw(double raw) { - hasRaw = true; - this.raw = raw; - } - - boolean hasMin() { - return hasMin; - } - double getMin() { - return min; - } - void setMin(double min) { - hasMin = true; - this.min = min; - } - - boolean hasMean() { - return hasMean; - } - double getMean() { - return mean; - } - void setMean(double mean) { - hasMean = true; - this.mean = mean; - } - - boolean hasMax() { - return hasMax; - } - double getMax() { - return max; - } - void setMax(double max) { - hasMax = true; - this.max = max; - } - - boolean hasHistogram() { - return hasHistogram; - } - Histogram getHistogram() { - return histogram; - } - void setHistogram(Histogram histogram) { - hasHistogram = true; - this.histogram = histogram; - } - -} - diff --git a/statistics/src/main/java/com/yahoo/statistics/package-info.java b/statistics/src/main/java/com/yahoo/statistics/package-info.java deleted file mode 100644 index bb861889d44..00000000000 --- a/statistics/src/main/java/com/yahoo/statistics/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -@PublicApi -package com.yahoo.statistics; - -import com.yahoo.api.annotations.PublicApi; -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/statistics/src/main/resources/configdefinitions/container.statistics.def b/statistics/src/main/resources/configdefinitions/container.statistics.def deleted file mode 100644 index 5f540241e35..00000000000 --- a/statistics/src/main/resources/configdefinitions/container.statistics.def +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=container - -## Interval between internal sample points measured in seconds -collectionintervalsec double default=300.0 - -## Interval between each dump to log measured in seconds -loggingintervalsec double default=300.0 - -## Configuration for measured values -values[].name string - -## An array of what operations should be performed on gathered data -values[].operations[].name enum { RAW, MEAN, MIN, MAX, REGULAR, CUMULATIVE, REVERSE_CUMULATIVE, SUM, INSERTIONS } default=RAW - -## Arguments for the operations, typically limits for arrays. -values[].operations[].arguments[].key string -values[].operations[].arguments[].value string - -## The counters which should be reset between each collection interval, -## and therefore be logged as values. -counterresets[].name string diff --git a/statistics/src/test/java/com/yahoo/statistics/CounterGroupTestCase.java b/statistics/src/test/java/com/yahoo/statistics/CounterGroupTestCase.java deleted file mode 100644 index c1899aa7e94..00000000000 --- a/statistics/src/test/java/com/yahoo/statistics/CounterGroupTestCase.java +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - - -import java.util.Arrays; -import java.util.logging.Handler; -import java.util.logging.LogRecord; -import java.util.logging.Logger; - -import com.yahoo.container.StatisticsConfig; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; - -/** - * Test set for groups of counters. - * - * @author Steinar Knutsen - */ -@SuppressWarnings({"deprecation","removal"}) // TODO Vespa 8: remove -public class CounterGroupTestCase { - - private volatile boolean gotRecord = false; - - private class CounterGroupHandler extends Handler { - // This is for testing CounterProxy - @Override - public void publish(LogRecord record) { - com.yahoo.log.event.CountGroup msg = (com.yahoo.log.event.CountGroup) record.getParameters()[0]; - assertEquals("test", msg.getValue("name")); - String values = msg.getValue("values"); - assertFalse("Unexpected value for a.", values.indexOf("a=500") == -1); - assertFalse("Unexpected value for b.", values.indexOf("b=1") == -1); - assertFalse("Unexpected value for c.", values.indexOf("c=0") == -1); - gotRecord = true; - - } - - @Override - public void flush() { - } - - @Override - public void close() throws SecurityException { - } - } - - @Test - public void testBasic() { - Logger logger = Logger.getLogger(CounterGroup.class.getName()); - boolean initUseParentHandlers = logger.getUseParentHandlers(); - Handler logChecker = new CounterGroupHandler(); - logger.setUseParentHandlers(false); - CounterGroup c = new CounterGroup("test", Statistics.nullImplementation, false); - Counter n; - c.increment("a"); - c.increment("b"); - c.increment("a", 499); - n = c.getCounter("a"); - assertEquals(500, n.get()); - n = c.getCounter("b"); - assertEquals(1, n.get()); - n = c.getCounter("c"); - assertEquals(0, n.get()); - logger.addHandler(logChecker); - c.run(); - assertFalse("The logging handler did not really run.", gotRecord == false); - // cleanup: - logger.removeHandler(logChecker); - logger.setUseParentHandlers(initUseParentHandlers); - } - - @Test - public void testObjectContracts() { - CounterGroup c = new CounterGroup("test", Statistics.nullImplementation, false); - CounterGroup c2 = new CounterGroup("test", Statistics.nullImplementation, false); - c2.increment("nalle"); - assertEquals(c, c2); - assertEquals(c.hashCode(), c2.hashCode()); - c2 = new CounterGroup("nalle", Statistics.nullImplementation, false); - assertFalse("Different names should lead to different hashcodes", - c.hashCode() == c2.hashCode()); - assertFalse("Different names should lead to equals() return false", - c.equals(c2)); - } - - @Test - public void testConfigStuff() { - Logger logger = Logger.getLogger(CounterGroup.class.getName()); - boolean initUseParentHandlers = logger.getUseParentHandlers(); - logger.setUseParentHandlers(false); - MockStatistics m = new MockStatistics(); - final String joppe = "joppe"; - StatisticsConfig config = new StatisticsConfig( - new StatisticsConfig.Builder().counterresets(Arrays - .asList(new StatisticsConfig.Counterresets.Builder[] { - new StatisticsConfig.Counterresets.Builder().name(joppe) }))); - m.config = config; - CounterGroup c = new CounterGroup("nalle", m); - CounterGroup c2 = new CounterGroup(joppe, m); - final String bamse = "bamse"; - c.increment(bamse); - c2.increment(bamse); - assertEquals(1L, c.getCounter(bamse).get()); - assertEquals(1L, c2.getCounter(bamse).get()); - c2.increment(bamse); - assertEquals(1L, c.getCounter(bamse).get()); - assertEquals(2L, c2.getCounter(bamse).get()); - c.run(); - c2.run(); - assertEquals(1L, c.getCounter(bamse).get()); - assertEquals(0L, c2.getCounter(bamse).get()); - logger.setUseParentHandlers(initUseParentHandlers); - } - - public class MockStatistics implements Statistics { - public StatisticsConfig config = null; - public int registerCount = 0; - - @Override - public void register(Handle h) { - registerCount += 1; - } - - @Override - public void remove(String name) { - } - - @Override - public StatisticsConfig getConfig() { - return config; - } - - @Override - public int purge() { - return 0; - } - } - -} diff --git a/statistics/src/test/java/com/yahoo/statistics/CounterTestCase.java b/statistics/src/test/java/com/yahoo/statistics/CounterTestCase.java deleted file mode 100644 index 8c17fd936a0..00000000000 --- a/statistics/src/test/java/com/yahoo/statistics/CounterTestCase.java +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - -import java.util.Arrays; -import java.util.logging.Logger; - -import com.yahoo.container.StatisticsConfig; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; - -/** - * Check counters work. - * - * @author Steinar Knutsen - */ -@SuppressWarnings("deprecation") -public class CounterTestCase { - - @Test - public void testBasic() { - Counter c = new Counter("test", Statistics.nullImplementation, false); - c.increment(); - assertEquals(1, c.get()); - c.increment(499); - assertEquals(500, c.get()); - c.reset(); - assertEquals(500, c.get()); - c = new Counter("test", Statistics.nullImplementation, false, null, true); - c.increment(); - assertEquals(1, c.get()); - c.increment(499); - assertEquals(500, c.get()); - c.reset(); - assertEquals(0, c.get()); - } - - @Test - public void testObjectContracts() { - final String counterName = "test"; - Counter c = new Counter(counterName, Statistics.nullImplementation, false); - Counter c2 = new Counter(counterName, Statistics.nullImplementation, false); - c2.increment(); - assertEquals(c, c2); - assertEquals(c.hashCode(), c2.hashCode()); - c2 = new Counter("nalle", Statistics.nullImplementation, false); - assertFalse("Different names should lead to different hashcodes", - c.hashCode() == c2.hashCode()); - assertFalse("Different names should lead to equals() return false", - c.equals(c2)); - String prefix = "com.yahoo.statistics.Counter"; - String suffix = counterName + " 0"; - String image = c.toString(); - assertEquals(suffix, image.substring(image.length() - suffix.length())); - assertEquals(prefix, image.substring(0, prefix.length())); - } - - @Test - public void testConfigStuff() { - Logger logger = Logger.getLogger(Counter.class.getName()); - boolean initUseParentHandlers = logger.getUseParentHandlers(); - logger.setUseParentHandlers(false); - MockStatistics m = new MockStatistics(); - final String joppe = "joppe"; - StatisticsConfig config = new StatisticsConfig( - new StatisticsConfig.Builder().counterresets(Arrays - .asList(new StatisticsConfig.Counterresets.Builder[] { new StatisticsConfig.Counterresets.Builder() - .name(joppe) }))); - m.config = config; - Counter c = new Counter("nalle", m, true); - Counter c2 = new Counter(joppe, m, true); - c.increment(); - c2.increment(); - assertEquals(1L, c.get()); - assertEquals(1L, c2.get()); - c.run(); - c2.run(); - assertEquals(1L, c.get()); - assertEquals(0L, c2.get()); - logger.setUseParentHandlers(initUseParentHandlers); - - } - - public class MockStatistics implements Statistics { - public StatisticsConfig config = null; - public int registerCount = 0; - - @Override - public void register(Handle h) { - registerCount += 1; - } - - @Override - public void remove(String name) { - } - - @Override - public StatisticsConfig getConfig() { - return config; - } - - @Override - public int purge() { - return 0; - } - } - -} diff --git a/statistics/src/test/java/com/yahoo/statistics/HistogramTestCase.java b/statistics/src/test/java/com/yahoo/statistics/HistogramTestCase.java deleted file mode 100644 index 517de9b4ef1..00000000000 --- a/statistics/src/test/java/com/yahoo/statistics/HistogramTestCase.java +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -/** - * Some low level checking of the histograms. - * - * @author Steinar Knutsen - */ -@SuppressWarnings("deprecation") -public class HistogramTestCase { - - @Test - public void testFindBucket() { - Limits l = new Limits(); - double[] thresholds = {.5, 1.0, 5.0}; - double[] value = {.5, .5}; - l.addAxis("latency", thresholds); - thresholds = new double[] {500.0, 1000.0, 5000.0}; - l.addAxis("size", thresholds); - Histogram h = new Histogram(l); - assertEquals("latency,size ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 0.5 ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 1.0 ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 5.0 ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0))", - h.toString()); - h.put(value); - assertEquals("latency,size ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 0.5 ((1) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 1.0 ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 5.0 ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0))", - h.toString()); - - } - - @Test - public void testMerge() { - Limits l = new Limits(); - double[] thresholds = {.5, 1.0, 5.0}; - double[] value = {.75}; - l.addAxis("latency", thresholds); - Histogram h = new Histogram(l); - Histogram h2 = new Histogram(l); - h.put(value); - h.put(value); - h2.put(value); - h2.merge(h); - assertEquals("(0) < 0.5 (3) < 1.0 (0) < 5.0 (0)", h2.toString()); - } - - @Test - public void testMultiDimensionalMerge() { - Limits l = new Limits(); - double[] thresholds = {.5, 1.0, 5.0}; - double[] value = {.5, .5}; - l.addAxis("latency", thresholds); - thresholds = new double[] {500.0, 1000.0, 5000.0}; - l.addAxis("size", thresholds); - Histogram h = new Histogram(l); - Histogram h2 = new Histogram(l); - h.put(value); - assertEquals("latency,size ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 0.5 ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 1.0 ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 5.0 ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0))", - h2.toString()); - h2.merge(h); - assertEquals("latency,size ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 0.5 ((1) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 1.0 ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0)) < 5.0 ((0) < 500.0 (0) < 1000.0 (0) < 5000.0 (0))", - h.toString()); - } - - @Test - public void testEmptyHistogram() { - try { - new Histogram(new Limits()); - } catch (IndexOutOfBoundsException e) { - return; - } - fail("Got no exception when trying to create an empty histogram."); - } - -} diff --git a/statistics/src/test/java/com/yahoo/statistics/ProxyTestCase.java b/statistics/src/test/java/com/yahoo/statistics/ProxyTestCase.java deleted file mode 100644 index 8eaee5d8ce8..00000000000 --- a/statistics/src/test/java/com/yahoo/statistics/ProxyTestCase.java +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - -import static org.junit.Assert.*; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * Tabular checking of statistics proxies. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -@SuppressWarnings("deprecation") -public class ProxyTestCase { - private static final double MAX = 2.0d; - private static final double MEAN = 1.0d; - private static final double MIN = -1.0d; - private static final double RAW = 0.5d; - private static final long C_RAW = 3; - ValueProxy vp; - CounterProxy cp; - Histogram h = new Histogram(new Limits(new double[] { 1.0d })); - - @Before - public void setUp() throws Exception { - vp = new ValueProxy("nalle"); - vp.setRaw(RAW); - vp.setMin(MIN); - vp.setMean(MEAN); - vp.setMax(MAX); - vp.setHistogram(h); - cp = new CounterProxy("nalle"); - cp.setRaw(C_RAW); - } - - @After - public void tearDown() throws Exception { - } - - @Test - public final void test() { - assertFalse(vp.hasHistogram() == false); - assertFalse(vp.hasRaw() == false); - assertFalse(vp.hasMin() == false); - assertFalse(vp.hasMean() == false); - assertFalse(vp.hasMax() == false); - assertFalse(cp.hasRaw() == false); - assertEquals(C_RAW, cp.getRaw()); - assertEquals(MAX, vp.getMax(), 1e-9); - assertEquals(MEAN, vp.getMean(), 1e-9); - assertEquals(MIN, vp.getMin(), 1e-9); - assertEquals(RAW, vp.getRaw(), 1e-9); - assertSame(h, vp.getHistogram()); - - final long t = 11L; - Proxy p = new Proxy("nalle", t) { - }; - assertEquals(t, p.getTimestamp()); - } - -} diff --git a/statistics/src/test/java/com/yahoo/statistics/StatisticsImplTestCase.java b/statistics/src/test/java/com/yahoo/statistics/StatisticsImplTestCase.java deleted file mode 100644 index 1ae8725ffe2..00000000000 --- a/statistics/src/test/java/com/yahoo/statistics/StatisticsImplTestCase.java +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertSame; - -import java.util.logging.Logger; - -import org.junit.Test; - -import com.yahoo.container.StatisticsConfig; - -/** - * Check register/remove semantics. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -@SuppressWarnings("deprecation") -public class StatisticsImplTestCase { - private static class TestHandle extends Handle { - - TestHandle(final String name, final Statistics manager, - final Callback parametrizedCallback) { - super(name, manager, parametrizedCallback); - } - - @Override - public void runHandle() { - } - - @Override - public boolean equals(final Object o) { - if (o == this) { - return true; - } else { - return false; - } - } - - @Override - public int hashCode() { - return getName().hashCode(); - } - - } - - @Test - public final void testRegister() { - final StatisticsConfig config = new StatisticsConfig( - new StatisticsConfig.Builder().collectionintervalsec(31e9) - .loggingintervalsec(31e9)); - final StatisticsImpl s = new StatisticsImpl(config); - final Logger logger = Logger.getLogger(TestHandle.class.getName()); - final boolean initUseParentHandlers = logger.getUseParentHandlers(); - logger.setUseParentHandlers(false); - final String firstHandle = "a"; - final Handle a = new TestHandle(firstHandle, s, null); - final Handle a2 = new TestHandle(firstHandle, s, null); - final String secondHandle = "b"; - final Handle b = new TestHandle(secondHandle, s, null); - s.register(a); - s.register(a2); - assertFalse("Old handle should be cancelled.", a.isCancelled() == false); - assertFalse("New handle should not be cancelled.", a2.isCancelled()); - assertEquals("Internal handles map semantics have been changed?", 1, - s.handles.size()); - s.register(b); - s.remove(secondHandle); - assertFalse("Removed handle should be cancelled.", - b.isCancelled() == false); - a2.cancel(); - s.purge(); - assertEquals("Cancelled tasks should be removed.", 0, s.handles.size()); - s.deconstruct(); - assertSame(config, s.getConfig()); - logger.setUseParentHandlers(initUseParentHandlers); - } - - @Test - public void freezeNullImplementationBehavior() { - Statistics s = Statistics.nullImplementation; - assertEquals(0, s.purge()); - // invoke s.register - Handle h = new Handle("nalle", s, null) { - @Override - public void runHandle() { - } - - @Override - public boolean equals(Object o) { - return true; - } - - @Override - public int hashCode() { - return 0; - } - - }; - assertEquals(0, s.purge()); - s.register(h); - s.remove("nalle"); - s.register(h); - h.cancel(); - assertEquals(0, s.purge()); - } - -} diff --git a/statistics/src/test/java/com/yahoo/statistics/ValueGroupTestCase.java b/statistics/src/test/java/com/yahoo/statistics/ValueGroupTestCase.java deleted file mode 100644 index f57fff8c312..00000000000 --- a/statistics/src/test/java/com/yahoo/statistics/ValueGroupTestCase.java +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - - -import java.util.logging.Handler; -import java.util.logging.LogRecord; -import java.util.logging.Logger; - -import com.yahoo.container.StatisticsConfig; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * Test set for groups of values. - * - * @author Steinar Knutsen - */ -@SuppressWarnings({"deprecation","removal"}) // TODO Vespa 8: remove -public class ValueGroupTestCase { - - private volatile boolean gotRecord = false; - - private class ValueGroupHandler extends Handler { - // this is for testing ValueProxy - - @Override - public void publish(LogRecord record) { - com.yahoo.log.event.ValueGroup msg = (com.yahoo.log.event.ValueGroup) record.getParameters()[0]; - assertEquals("test", msg.getValue("name")); - String values = msg.getValue("values"); - assertFalse("Unexpected value for a.", values.indexOf("a=-50.0") == -1); - assertFalse("Unexpected value for b.", values.indexOf("b=40.0") == -1); - assertFalse("Unexpected value for c.", values.indexOf("c=0.0") == -1); - gotRecord = true; - } - - @Override - public void flush() { - } - - @Override - public void close() throws SecurityException { - } - } - - @Test - public void testBasic() { - Logger logger = Logger.getLogger(ValueGroup.class.getName()); - boolean initUseParentHandlers = logger.getUseParentHandlers(); - Handler logChecker = new ValueGroupHandler(); - logger.setUseParentHandlers(false); - final MockStatistics manager = new MockStatistics(); - ValueGroup v = new ValueGroup("test", manager); - v.put("a", 50.0); - v.put("b", 40.0); - v.put("a", -50.0); - assertTrue("Last value inserted to a was -50", - -50.0 == v.getValue("a").get()); - assertTrue("Last value inserted to b was 40.", - 40.0 == v.getValue("b").get()); - assertTrue("c has not been used yet", - 0.0 == v.getValue("c").get()); - logger.addHandler(logChecker); - v.run(); - assertFalse("The logging handler did not really run.", gotRecord == false); - assertEquals(1, manager.registerCount); - // cleanup: - logger.removeHandler(logChecker); - logger.setUseParentHandlers(initUseParentHandlers); - } - - @Test - public void testOverlappingSubnames() { - final MockStatistics manager = new MockStatistics(); - ValueGroup v = new ValueGroup("jappe", manager); - ValueGroup v2 = new ValueGroup("nalle", manager); - final String name = "mobil"; - v.put(name, 50.0); - v2.put(name, 40.0); - assertEquals(50.0, v.getValue(name).get(), 1e-9); - assertEquals(40.0, v2.getValue(name).get(), 1e-9); - assertEquals(2, manager.registerCount); - } - - @Test - public void testObjectContracts() { - ValueGroup v = new ValueGroup("test", new MockStatistics()); - ValueGroup v2 = new ValueGroup("test", new MockStatistics()); - v2.put("nalle", 2.0); - assertEquals(v, v2); - assertEquals(v.hashCode(), v2.hashCode()); - v2 = new ValueGroup("nalle", new MockStatistics()); - assertFalse("Different names should lead to different hashcodes", - v.hashCode() == v2.hashCode()); - assertFalse("Different names should lead to equals() return false", - v.equals(v2)); - } - - public class MockStatistics implements Statistics { - public StatisticsConfig config = null; - public int registerCount = 0; - - @Override - public void register(Handle h) { - registerCount += 1; - } - - @Override - public void remove(String name) { - } - - @Override - public StatisticsConfig getConfig() { - return config; - } - - @Override - public int purge() { - return 0; - } - } - -} diff --git a/statistics/src/test/java/com/yahoo/statistics/ValueTestCase.java b/statistics/src/test/java/com/yahoo/statistics/ValueTestCase.java deleted file mode 100644 index b7f0d9ca8c4..00000000000 --- a/statistics/src/test/java/com/yahoo/statistics/ValueTestCase.java +++ /dev/null @@ -1,268 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.statistics; - -import java.util.Arrays; -import java.util.List; -import java.util.logging.Handler; -import java.util.logging.LogRecord; -import java.util.logging.Logger; - -import com.yahoo.container.StatisticsConfig; -import static com.yahoo.container.StatisticsConfig.Values.Operations; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import org.junit.Test; - -/** - * Check correct statistics are generated for basic values. - * - * @author Steinar Knutsen - */ -@SuppressWarnings({"deprecation","removal"}) // TODO Vespa 8: remove -public class ValueTestCase { - - private static final double delta = 0.0000000001; - - private static final String NALLE = "nalle"; - private static final double SECOND = 43.0d; - private static final double FIRST = 42.0d; - - private static class TrivialCallback implements Callback { - - @Override - public void run(Handle h, boolean firstRun) { - Value v = (Value) h; - if (firstRun) { - v.put(FIRST); - } else { - v.put(SECOND); - } - } - - } - - @Test - public void testMean() { - Value v = new Value("thingie", Statistics.nullImplementation, new Value.Parameters().setLogMean(true)); - v.put(1.0); - v.put(2.0); - v.put(4.0); - v.put(-1.0); - assertTrue("Mean should be 1.5", 1.5 == v.getMean()); - ValueProxy vp = v.getProxyAndReset(); - assertTrue("Proxy mean should be 1.5", 1.5 == vp.getMean()); - assertTrue("Value should have been reset.", 0.0d == v.getMean()); - } - - @Test - public void testMin() { - Value v = new Value("thingie", Statistics.nullImplementation, new Value.Parameters().setLogMin(true)); - v.put(2.0); - assertTrue("Min should be 2.0", 2.0 == v.getMin()); - v.put(1.0); - assertTrue("Min should be 1.0", 1.0 == v.getMin()); - v.put(-1.0); - v.put(4.0); - assertTrue("Min should be -1.0", -1.0 == v.getMin()); - } - - @Test - public void testMax() { - Value v = new Value("thingie", Statistics.nullImplementation, new Value.Parameters().setLogMax(true)); - v.put(-1.0); - assertTrue("Max should be -1.0", -1.0 == v.getMax()); - v.put(1.0); - v.put(2.0); - assertTrue("Max should be 2.0", 2.0 == v.getMax()); - v.put(4.0); - v.put(-1.0); - assertTrue("Max should be 4.0", 4.0 == v.getMax()); - } - - @Test - public void testHistogram() { - Value v = new Value("thingie", Statistics.nullImplementation, new Value.Parameters() - .setLogHistogram(true).setHistogramId(HistogramType.REGULAR) - .setLimits(new Limits(new double[] { 0.0, 1.0, 2.0 }))); - v.put(-1.0); - v.put(0.0); - v.put(1.0); - v.put(2.0); - v.put(3.0); - assertTrue(v.toString().endsWith( - " thingie (1) < 0.0 (1) < 1.0 (1) < 2.0 (2)")); - } - - @Test - public void testCallback() { - Logger logger = Logger.getLogger(Value.class.getName()); - boolean initUseParentHandlers = logger.getUseParentHandlers(); - logger.setUseParentHandlers(false); - Value v = new Value("thingie", Statistics.nullImplementation, new Value.Parameters() - .setLogRaw(true).setCallback(new TrivialCallback())); - v.run(); - assertEquals(FIRST, v.get(), delta); - v.run(); - assertEquals(SECOND, v.get(), delta); - v.run(); - assertEquals(SECOND, v.get(), delta); - logger.setUseParentHandlers(initUseParentHandlers); - } - - @Test - public void testParameter() { - Value.Parameters p = new Value.Parameters().setLogInsertions(true) - .setNameExtension(true).setAppendChar('_'); - Value.Parameters p2 = new Value.Parameters().setLogSum(true); - assertNull(p2.appendChar); - assertNull(p.logSum); - p2.merge(p); - assertEquals(Character.valueOf('_'), p2.appendChar); - assertNull(p2.logMax); - assertEquals(Boolean.TRUE, p2.logSum); - } - - private class CheckHistogram extends Handler { - volatile boolean gotRecord = false; - volatile boolean gotWarning = false; - final String histogram; - final String representation; - - public CheckHistogram(String histogram, String representation) { - this.histogram = histogram; - this.representation = representation; - } - - @Override - public void publish(LogRecord record) { - if (record.getParameters() == null) { - assertEquals(Value.HISTOGRAM_TYPE_WARNING + " '" + NALLE + "'", record.getMessage()); - gotWarning = true; - return; - } - if (!(record.getParameters()[0] instanceof com.yahoo.log.event.Histogram)) { - return; - } - com.yahoo.log.event.Histogram msg = (com.yahoo.log.event.Histogram) record.getParameters()[0]; - assertEquals(NALLE, msg.getValue("name")); - assertEquals(histogram, msg.getValue("counts")); - assertEquals(representation, msg.getValue("representation")); - gotRecord = true; - } - - @Override - public void flush() { - } - - @Override - public void close() throws SecurityException { - } - } - - @Test - public void testParameterFromConfig() { - Logger logger = Logger.getLogger(Value.class.getName()); - boolean initUseParentHandlers = logger.getUseParentHandlers(); - logger.setUseParentHandlers(false); - CheckHistogram h = new CheckHistogram("(0) < 0.0 (0) < 1.0 (0) < 2.0 (1)", "REGULAR"); - logger.addHandler(h); - List<Operations.Arguments.Builder> histogram = Arrays.asList(new Operations.Arguments.Builder[] { - new Operations.Arguments.Builder().key("limits").value("0, 1, 2")}); - List<Operations.Builder> ops = Arrays.asList(new Operations.Builder[] { - new Operations.Builder().name(Operations.Name.Enum.MEAN), - new Operations.Builder().name(Operations.Name.Enum.MIN), - new Operations.Builder().name(Operations.Name.Enum.MAX), - new Operations.Builder().name(Operations.Name.Enum.RAW), - new Operations.Builder().name(Operations.Name.Enum.INSERTIONS), - new Operations.Builder().name(Operations.Name.Enum.REGULAR).arguments(histogram), - new Operations.Builder().name(Operations.Name.Enum.SUM) }); - StatisticsConfig c = new StatisticsConfig( - new StatisticsConfig.Builder() - .values(new StatisticsConfig.Values.Builder().name( - NALLE).operations(ops))); - MockStatistics m = new MockStatistics(); - m.config = c; - Value v = Value.buildValue(NALLE, m, null); - final double x = 79.0d; - v.put(x); - assertEquals(x, v.getMean(), delta); - v.run(); - logger.removeHandler(h); - logger.setUseParentHandlers(initUseParentHandlers); - } - - @Test - public void testReverseHistogram() { - Logger logger = Logger.getLogger(Value.class.getName()); - boolean initUseParentHandlers = logger.getUseParentHandlers(); - logger.setUseParentHandlers(false); - CheckHistogram h = new CheckHistogram("(0) < 0.0 (2) < 1.0 (2) < 2.0 (0)", "REGULAR"); - logger.addHandler(h); - List<Operations.Arguments.Builder> histogram = Arrays.asList(new Operations.Arguments.Builder[] { - new Operations.Arguments.Builder().key("limits").value("0, 1, 2")}); - List<Operations.Builder> ops = Arrays.asList(new Operations.Builder[] { - new Operations.Builder().name(Operations.Name.Enum.REVERSE_CUMULATIVE).arguments(histogram) }); - StatisticsConfig c = new StatisticsConfig( - new StatisticsConfig.Builder() - .values(new StatisticsConfig.Values.Builder().name( - NALLE).operations(ops))); - MockStatistics m = new MockStatistics(); - m.config = c; - Value v = Value.buildValue(NALLE, m, null); - assertEquals(HistogramType.REGULAR.toString(), v.histogramId.toString()); - v.put(.5d); - v.put(.5d); - v.put(1.5d); - v.put(1.5d); - v.run(); - logger.removeHandler(h); - logger.setUseParentHandlers(initUseParentHandlers); - } - - @Test - public void testObjectContracts() { - final String valueName = "test"; - Value v = new Value(valueName, Statistics.nullImplementation, Value.defaultParameters()); - Value v2 = new Value(valueName, Statistics.nullImplementation, Value.defaultParameters()); - v2.put(1.0); - assertEquals(v, v2); - assertEquals(v.hashCode(), v2.hashCode()); - v2 = new Value("nalle", Statistics.nullImplementation, Value.defaultParameters()); - assertFalse("Different names should lead to different hashcodes", - v.hashCode() == v2.hashCode()); - assertFalse("Different names should lead to equals() return false", - v.equals(v2)); - String image = v.toString(); - String prefix = "com.yahoo.statistics.Value"; - assertEquals(prefix, image.substring(0, prefix.length())); - assertEquals(valueName, image.substring(image.length() - valueName.length())); - } - - public class MockStatistics implements Statistics { - public StatisticsConfig config = null; - public int registerCount = 0; - - @Override - public void register(Handle h) { - registerCount += 1; - } - - @Override - public void remove(String name) { - } - - @Override - public StatisticsConfig getConfig() { - return config; - } - - @Override - public int purge() { - return 0; - } - } - -} diff --git a/storage/pom.xml b/storage/pom.xml index 74854da2f3f..3d057b7b928 100644 --- a/storage/pom.xml +++ b/storage/pom.xml @@ -7,11 +7,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>storage</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/storage/src/tests/common/metricstest.cpp b/storage/src/tests/common/metricstest.cpp index cff62542434..a92cf121fab 100644 --- a/storage/src/tests/common/metricstest.cpp +++ b/storage/src/tests/common/metricstest.cpp @@ -133,7 +133,7 @@ void MetricsTest::createFakeLoad() _filestorMetrics->partitionEvents.inc(4); _filestorMetrics->diskEvents.inc(3); { - FileStorDiskMetrics& disk(*_filestorMetrics->disk); + FileStorMetrics& disk(*_filestorMetrics); disk.queueSize.addValue(4 * n); disk.averageQueueWaitingTime.addValue(10 * n); disk.pendingMerges.addValue(4 * n); @@ -207,10 +207,10 @@ TEST_F(MetricsTest, filestor_metrics) { bool retVal = _metricsConsumer->reportStatus(ost, path); ASSERT_TRUE(retVal) << "_metricsConsumer->reportStatus failed"; std::string s = ost.str(); - EXPECT_THAT(s, HasSubstr("vds.filestor.alldisks.allthreads.get.sum.count count=60")); - EXPECT_THAT(s, HasSubstr("vds.filestor.alldisks.allthreads.put.sum.count count=50")); - EXPECT_THAT(s, HasSubstr("vds.filestor.alldisks.allthreads.remove.sum.count count=30")); - EXPECT_THAT(s, HasSubstr("vds.filestor.alldisks.allthreads.remove.sum.not_found count=5")); + EXPECT_THAT(s, HasSubstr("vds.filestor.allthreads.get.count count=60")); + EXPECT_THAT(s, HasSubstr("vds.filestor.allthreads.put.count count=50")); + EXPECT_THAT(s, HasSubstr("vds.filestor.allthreads.remove.count count=30")); + EXPECT_THAT(s, HasSubstr("vds.filestor.allthreads.remove.not_found count=5")); } #define ASSERT_METRIC(interval, metric, count) \ @@ -233,8 +233,7 @@ TEST_F(MetricsTest, filestor_metrics) { } TEST_F(MetricsTest, snapshot_presenting) { - FileStorDiskMetrics& disk0(*_filestorMetrics->disk); - FileStorThreadMetrics& thread0(*disk0.threads[0]); + FileStorThreadMetrics& thread0(*_filestorMetrics->threads[0]); LOG(debug, "Adding to get metric"); @@ -257,20 +256,20 @@ TEST_F(MetricsTest, snapshot_presenting) { thread0.put.count.inc(1); // Verify that active metrics have set put count but not get count - ASSERT_METRIC(-2, "vds.filestor.alldisks.allthreads.put.sum.count", 1); - ASSERT_METRIC(-2, "vds.filestor.alldisks.allthreads.get.sum.count", -1); + ASSERT_METRIC(-2, "vds.filestor.allthreads.put.count", 1); + ASSERT_METRIC(-2, "vds.filestor.allthreads.get.count", -1); // Verify that 5 min metrics have set get count but not put count - ASSERT_METRIC(300, "vds.filestor.alldisks.allthreads.put.sum.count", -1); - ASSERT_METRIC(300, "vds.filestor.alldisks.allthreads.get.sum.count", 1); + ASSERT_METRIC(300, "vds.filestor.allthreads.put.count", -1); + ASSERT_METRIC(300, "vds.filestor.allthreads.get.count", 1); // Verify that the total metrics is equal to 5 minute - ASSERT_METRIC(0, "vds.filestor.alldisks.allthreads.put.sum.count", -1); - ASSERT_METRIC(0, "vds.filestor.alldisks.allthreads.get.sum.count", 1); + ASSERT_METRIC(0, "vds.filestor.allthreads.put.count", -1); + ASSERT_METRIC(0, "vds.filestor.allthreads.get.count", 1); // Verify that total + active have set both - ASSERT_METRIC(-1, "vds.filestor.alldisks.allthreads.put.sum.count", 1); - ASSERT_METRIC(-1, "vds.filestor.alldisks.allthreads.get.sum.count", 1); + ASSERT_METRIC(-1, "vds.filestor.allthreads.put.count", 1); + ASSERT_METRIC(-1, "vds.filestor.allthreads.get.count", 1); } TEST_F(MetricsTest, html_metrics_report) { diff --git a/storage/src/tests/persistence/active_operations_stats_test.cpp b/storage/src/tests/persistence/active_operations_stats_test.cpp index 8caa84977ce..5b5a019f688 100644 --- a/storage/src/tests/persistence/active_operations_stats_test.cpp +++ b/storage/src/tests/persistence/active_operations_stats_test.cpp @@ -115,7 +115,7 @@ ActiveOperationsStatsTest::test_active_operations_stats() EXPECT_EQ(4, stats.get_total_size()); EXPECT_LT(0.0, stats.get_total_latency()); update_metrics(); - auto &ao_metrics = metrics.disk->active_operations; + auto &ao_metrics = metrics.active_operations; EXPECT_DOUBLE_EQ(1.0, ao_metrics.size.getAverage()); EXPECT_DOUBLE_EQ(0.0, ao_metrics.size.getMinimum()); EXPECT_DOUBLE_EQ(2.0, ao_metrics.size.getMaximum()); diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 304720f52b7..4c1b1662f68 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -201,7 +201,7 @@ struct FileStorTestBase : Test { const Metric& metric); auto& thread_metrics_of(FileStorManager& manager) { - return manager.get_metrics().disk->threads[0]; + return manager.get_metrics().threads[0]; } }; @@ -304,7 +304,7 @@ struct PersistenceHandlerComponents : public FileStorHandlerComponents { std::make_unique<PersistenceHandler>(executor, component, cfg, test._node->getPersistenceProvider(), *filestorHandler, bucketOwnershipNotifier, - *metrics.disk->threads[0]); + *metrics.threads[0]); } ~PersistenceHandlerComponents(); std::unique_ptr<DiskThread> make_disk_thread() { @@ -759,11 +759,11 @@ TEST_F(FileStorManagerTest, priority) { BucketOwnershipNotifier bucketOwnershipNotifier(component, c.messageSender); vespa::config::content::StorFilestorConfig cfg; PersistenceHandler persistenceHandler(_node->executor(), component, cfg, _node->getPersistenceProvider(), - filestorHandler, bucketOwnershipNotifier, *metrics.disk->threads[0]); + filestorHandler, bucketOwnershipNotifier, *metrics.threads[0]); std::unique_ptr<DiskThread> thread(createThread(persistenceHandler, filestorHandler, component)); PersistenceHandler persistenceHandler2(_node->executor(), component, cfg, _node->getPersistenceProvider(), - filestorHandler, bucketOwnershipNotifier, *metrics.disk->threads[1]); + filestorHandler, bucketOwnershipNotifier, *metrics.threads[1]); std::unique_ptr<DiskThread> thread2(createThread(persistenceHandler2, filestorHandler, component)); // Creating documents to test with. Different gids, 2 locations. @@ -813,8 +813,8 @@ TEST_F(FileStorManagerTest, priority) { // Verify that thread 1 gets documents over 50 pri EXPECT_EQ(documents.size(), - metrics.disk->threads[0]->operations.getValue() - + metrics.disk->threads[1]->operations.getValue()); + metrics.threads[0]->operations.getValue() + + metrics.threads[1]->operations.getValue()); // Closing file stor handler before threads are deleted, such that // file stor threads getNextMessage calls returns. filestorHandler.close(); diff --git a/storage/src/tests/persistence/persistencetestutils.cpp b/storage/src/tests/persistence/persistencetestutils.cpp index 940436307a1..de9caebbf11 100644 --- a/storage/src/tests/persistence/persistencetestutils.cpp +++ b/storage/src/tests/persistence/persistencetestutils.cpp @@ -58,7 +58,7 @@ PersistenceTestEnvironment::PersistenceTestEnvironment(const std::string & rootO _metrics.initDiskMetrics(1, 1); _handler = std::make_unique<FileStorHandlerImpl>(_messageKeeper, _metrics, _node.getComponentRegister()); _diskEnv = std::make_unique<PersistenceUtil>(_component, *_handler, - *_metrics.disk->threads[0], _node.getPersistenceProvider()); + *_metrics.threads[0], _node.getPersistenceProvider()); } PersistenceTestEnvironment::~PersistenceTestEnvironment() { diff --git a/storage/src/tests/storageserver/statereportertest.cpp b/storage/src/tests/storageserver/statereportertest.cpp index 77192d24786..09df5063989 100644 --- a/storage/src/tests/storageserver/statereportertest.cpp +++ b/storage/src/tests/storageserver/statereportertest.cpp @@ -150,13 +150,11 @@ vespalib::Slime slime; \ for (size_t j=0; j<metricCount; j++) { \ const vespalib::string name = slime.get()["metrics"]["values"][j]["name"] \ .asString().make_string(); \ - if (name.compare("vds.filestor.alldisks.allthreads." \ - "get.sum.count") == 0) \ + if (name.compare("vds.filestor.allthreads.get.count") == 0) \ { \ getCount = slime.get()["metrics"]["values"][j]["values"]["count"] \ .asDouble(); \ - } else if (name.compare("vds.filestor.alldisks.allthreads." \ - "put.sum.count") == 0) \ + } else if (name.compare("vds.filestor.allthreads.put.count") == 0) \ { \ putCount = slime.get()["metrics"]["values"][j]["values"]["count"] \ .asDouble(); \ @@ -217,8 +215,7 @@ TEST_F(StateReporterTest, report_health) { } TEST_F(StateReporterTest, report_metrics) { - FileStorDiskMetrics& disk0(*_filestorMetrics->disk); - FileStorThreadMetrics& thread0(*disk0.threads[0]); + FileStorThreadMetrics& thread0(*_filestorMetrics->threads[0]); LOG(debug, "Adding to get metric"); diff --git a/storage/src/vespa/storage/distributor/distributormetricsset.cpp b/storage/src/vespa/storage/distributor/distributormetricsset.cpp index c3f94c4b281..fad44782dd4 100644 --- a/storage/src/vespa/storage/distributor/distributormetricsset.cpp +++ b/storage/src/vespa/storage/distributor/distributormetricsset.cpp @@ -13,17 +13,16 @@ BucketDbMetrics::BucketDbMetrics(const vespalib::string& db_type, metrics::Metri BucketDbMetrics::~BucketDbMetrics() = default; -//TODO Vespa 8 all metrics with .sum in the name should have that removed. DistributorMetricSet::DistributorMetricSet() : MetricSet("distributor", {{"distributor"}}, ""), - puts("puts.sum", this), + puts("puts", this), updates(this), update_puts("update_puts", this), update_gets("update_gets", this), update_metadata_gets("update_metadata_gets", this), - removes("removes.sum", this), - removelocations("removelocations.sum", this), - gets("gets.sum", this), + removes("removes", this), + removelocations("removelocations", this), + gets("gets", this), stats("stats", this), getbucketlists("getbucketlists", this), visits(this), diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index df1c0de36cc..1c818233746 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -49,7 +49,7 @@ FileStorHandlerImpl::FileStorHandlerImpl(uint32_t numThreads, uint32_t numStripe const vespalib::SharedOperationThrottler::DynamicThrottleParams& dyn_throttle_params) : _component(compReg, "filestorhandlerimpl"), _state(FileStorHandler::AVAILABLE), - _metrics(nullptr), + _metrics(&metrics), _dynamic_operation_throttler(vespalib::SharedOperationThrottler::make_dynamic_throttler(dyn_throttle_params)), _unlimited_operation_throttler(vespalib::SharedOperationThrottler::make_unlimited_throttler()), _active_throttler(_unlimited_operation_throttler.get()), // Will be set by FileStorManager @@ -67,8 +67,6 @@ FileStorHandlerImpl::FileStorHandlerImpl(uint32_t numThreads, uint32_t numStripe _stripes.emplace_back(*this, sender); } - _metrics = metrics.disk.get(); - assert(_metrics != nullptr); uint32_t j(0); for (Stripe & stripe : _stripes) { stripe.setMetrics(_metrics->stripes[j++].get()); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h index a3667e93fff..91b8dbe2f13 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h @@ -301,7 +301,7 @@ public: private: ServiceLayerComponent _component; std::atomic<DiskState> _state; - FileStorDiskMetrics * _metrics; + FileStorMetrics * _metrics; std::unique_ptr<vespalib::SharedOperationThrottler> _dynamic_operation_throttler; std::unique_ptr<vespalib::SharedOperationThrottler> _unlimited_operation_throttler; std::atomic<vespalib::SharedOperationThrottler*> _active_throttler; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index cab6c26f2a0..c7ee0ab97e0 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -187,11 +187,11 @@ FileStorManager::createRegisteredHandler(const ServiceLayerComponent & component { std::lock_guard guard(_lock); size_t index = _persistenceHandlers.size(); - assert(index < _metrics->disk->threads.size()); + assert(index < _metrics->threads.size()); _persistenceHandlers.push_back( std::make_unique<PersistenceHandler>(*_sequencedExecutor, component, *_config, *_provider, *_filestorHandler, - *_bucketOwnershipNotifier, *_metrics->disk->threads[index])); + *_bucketOwnershipNotifier, *_metrics->threads[index])); return *_persistenceHandlers.back(); } diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp index ee375b98a69..fec64765cad 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp @@ -7,7 +7,6 @@ namespace storage { using metrics::MetricSet; -// TODO Vespa 8 all metrics with .sum in the name should have that removed. FileStorThreadMetrics::Op::Op(const std::string& id, const std::string& name, MetricSet* owner) : MetricSet(id, {}, name + " load in filestor thread", owner), _name(name), @@ -108,7 +107,7 @@ FileStorThreadMetrics::OpWithNotFound::clone(std::vector<Metric::UP>& ownerList, } FileStorThreadMetrics::Update::Update(MetricSet* owner) - : OpWithTestAndSetFailed("update.sum", "Update", owner), + : OpWithTestAndSetFailed("update", "Update", owner), latencyRead("latency_read", {}, "Latency of the source read in the request.", this) { } @@ -127,7 +126,7 @@ FileStorThreadMetrics::Update::clone(std::vector<Metric::UP>& ownerList, } FileStorThreadMetrics::Visitor::Visitor(MetricSet* owner) - : Op("visit.sum", "Visit", owner), + : Op("visit", "Visit", owner), documentsPerIterate("docs", {}, "Number of entries read per iterate call", this) { } @@ -149,10 +148,10 @@ FileStorThreadMetrics::FileStorThreadMetrics(const std::string& name, const std: : MetricSet(name, {{"filestor"},{"partofsum"}}, desc), operations("operations", {}, "Number of operations processed.", this), failedOperations("failedoperations", {}, "Number of operations throwing exceptions.", this), - put("put.sum", "Put", this), - get("get.sum", "Get", this), - remove("remove.sum", "Remove", this), - removeLocation("remove_location.sum", "Remove location", this), + put("put", "Put", this), + get("get", "Get", this), + remove("remove", "Remove", this), + removeLocation("remove_location", "Remove location", this), statBucket("stat_bucket", "Stat bucket", this), update(this), revert("revert", "Revert", this), @@ -202,29 +201,33 @@ FileStorStripeMetrics::FileStorStripeMetrics(const std::string& name, const std: FileStorStripeMetrics::~FileStorStripeMetrics() = default; -FileStorDiskMetrics::FileStorDiskMetrics(const std::string& name, const std::string& description, MetricSet* owner) - : MetricSet(name, {{"partofsum"}}, description, owner), +FileStorMetrics::FileStorMetrics() + : MetricSet("filestor", {{"filestor"}}, ""), sumThreads("allthreads", {{"sum"}}, "", this), sumStripes("allstripes", {{"sum"}}, "", this), - averageQueueWaitingTime("averagequeuewait.sum", {}, "Average time an operation spends in input queue.", this), + averageQueueWaitingTime("averagequeuewait", {}, "Average time an operation spends in input queue.", this), queueSize("queuesize", {}, "Size of input message queue.", this), pendingMerges("pendingmerge", {}, "Number of buckets currently being merged.", this), throttle_window_size("throttle_window_size", {}, "Current size of async operation throttler window size", this), throttle_waiting_threads("throttle_waiting_threads", {}, "Number of threads waiting to acquire a throttle token", this), throttle_active_tokens("throttle_active_tokens", {}, "Current number of active throttle tokens", this), waitingForLockHitRate("waitingforlockrate", {}, - "Amount of times a filestor thread has needed to wait for " - "lock to take next message in queue.", this), - active_operations(this) + "Amount of times a filestor thread has needed to wait for " + "lock to take next message in queue.", this), + active_operations(this), + directoryEvents("directoryevents", {}, "Number of directory events received.", this), + partitionEvents("partitionevents", {}, "Number of partition events received.", this), + diskEvents("diskevents", {}, "Number of disk events received.", this), + bucket_db_init_latency("bucket_db_init_latency", {}, "Time taken (in ms) to initialize bucket databases with " + "information from the persistence provider", this) { pendingMerges.unsetOnZeroValue(); waitingForLockHitRate.unsetOnZeroValue(); } -FileStorDiskMetrics::~FileStorDiskMetrics() = default; +FileStorMetrics::~FileStorMetrics() = default; -void -FileStorDiskMetrics::initDiskMetrics(uint32_t numStripes, uint32_t threadsPerDisk) +void FileStorMetrics::initDiskMetrics(uint32_t numStripes, uint32_t threadsPerDisk) { threads.clear(); threads.resize(threadsPerDisk); @@ -250,26 +253,4 @@ FileStorDiskMetrics::initDiskMetrics(uint32_t numStripes, uint32_t threadsPerDis } } -FileStorMetrics::FileStorMetrics() - : MetricSet("filestor", {{"filestor"}}, ""), - sum("alldisks", {{"sum"}}, "", this), - directoryEvents("directoryevents", {}, "Number of directory events received.", this), - partitionEvents("partitionevents", {}, "Number of partition events received.", this), - diskEvents("diskevents", {}, "Number of disk events received.", this), - bucket_db_init_latency("bucket_db_init_latency", {}, "Time taken (in ms) to initialize bucket databases with " - "information from the persistence provider", this) -{ } - -FileStorMetrics::~FileStorMetrics() = default; - -void FileStorMetrics::initDiskMetrics(uint32_t numStripes, uint32_t threadsPerDisk) -{ - assert( ! disk); - // Currently FileStorHandlerImpl expects metrics to exist for - // disks that are not in use too. - disk = std::make_shared<FileStorDiskMetrics>( "disk_0", "Disk 0", this); - sum.addMetricToSum(*disk); - disk->initDiskMetrics(numStripes, threadsPerDisk); -} - } diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h index 24a4d508945..53bf087f036 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h @@ -136,38 +136,24 @@ public: ~FileStorStripeMetrics() override; }; -class FileStorDiskMetrics : public metrics::MetricSet +struct FileStorMetrics : public metrics::MetricSet { -public: - using SP = std::shared_ptr<FileStorDiskMetrics>; - std::vector<FileStorThreadMetrics::SP> threads; std::vector<FileStorStripeMetrics::SP> stripes; metrics::SumMetric<MetricSet> sumThreads; metrics::SumMetric<MetricSet> sumStripes; - metrics::DoubleAverageMetric averageQueueWaitingTime; - metrics::LongAverageMetric queueSize; - metrics::LongAverageMetric pendingMerges; - metrics::LongAverageMetric throttle_window_size; - metrics::LongAverageMetric throttle_waiting_threads; - metrics::LongAverageMetric throttle_active_tokens; - metrics::DoubleAverageMetric waitingForLockHitRate; - ActiveOperationsMetrics active_operations; - - FileStorDiskMetrics(const std::string& name, const std::string& description, MetricSet* owner); - ~FileStorDiskMetrics() override; - - void initDiskMetrics(uint32_t numStripes, uint32_t threadsPerDisk); -}; - -struct FileStorMetrics : public metrics::MetricSet -{ - FileStorDiskMetrics::SP disk; - metrics::SumMetric<MetricSet> sum; - metrics::LongCountMetric directoryEvents; - metrics::LongCountMetric partitionEvents; - metrics::LongCountMetric diskEvents; - metrics::LongAverageMetric bucket_db_init_latency; + metrics::DoubleAverageMetric averageQueueWaitingTime; + metrics::LongAverageMetric queueSize; + metrics::LongAverageMetric pendingMerges; + metrics::LongAverageMetric throttle_window_size; + metrics::LongAverageMetric throttle_waiting_threads; + metrics::LongAverageMetric throttle_active_tokens; + metrics::DoubleAverageMetric waitingForLockHitRate; + ActiveOperationsMetrics active_operations; + metrics::LongCountMetric directoryEvents; + metrics::LongCountMetric partitionEvents; + metrics::LongCountMetric diskEvents; + metrics::LongAverageMetric bucket_db_init_latency; FileStorMetrics(); ~FileStorMetrics() override; diff --git a/storage/src/vespa/storage/visiting/visitorthreadmetrics.cpp b/storage/src/vespa/storage/visiting/visitorthreadmetrics.cpp index af871cc8296..158846d2d5b 100644 --- a/storage/src/vespa/storage/visiting/visitorthreadmetrics.cpp +++ b/storage/src/vespa/storage/visiting/visitorthreadmetrics.cpp @@ -8,15 +8,15 @@ namespace storage { VisitorThreadMetrics::VisitorThreadMetrics(const std::string& name, const std::string& desc) : metrics::MetricSet(name, {{"visitor"},{"partofsum"},{"thread"}}, desc), queueSize("queuesize", {}, "Size of input message queue.", this), - averageQueueWaitingTime("averagequeuewait.sum", {}, "Average time an operation spends in input queue.", this), - averageVisitorLifeTime("averagevisitorlifetime.sum", {}, "Average lifetime of a visitor", this), + averageQueueWaitingTime("averagequeuewait", {}, "Average time an operation spends in input queue.", this), + averageVisitorLifeTime("averagevisitorlifetime", {}, "Average lifetime of a visitor", this), averageVisitorCreationTime("averagevisitorcreationtime", {}, "Average time spent creating a visitor instance", this), - averageMessageSendTime("averagemessagesendtime.sum", {}, "Average time it takes for messages to be sent to their target (and be replied to)", this), - averageProcessingTime("averageprocessingtime.sum", {}, "Average time visitor uses in handleDocuments() call", this), - createdVisitors("created.sum", {}, "Number of visitors created.", this), + averageMessageSendTime("averagemessagesendtime", {}, "Average time it takes for messages to be sent to their target (and be replied to)", this), + averageProcessingTime("averageprocessingtime", {}, "Average time visitor uses in handleDocuments() call", this), + createdVisitors("created", {}, "Number of visitors created.", this), abortedVisitors("aborted", {}, "Number of visitors aborted.", this), - completedVisitors("completed.sum", {}, "Number of visitors completed", this), - failedVisitors("failed.sum", {}, "Number of visitors failed", this), + completedVisitors("completed", {}, "Number of visitors completed", this), + failedVisitors("failed", {}, "Number of visitors failed", this), visitorDestinationFailureReplies("destination_failure_replies", {},"Number of failure replies received from the visitor destination", this) { queueSize.unsetOnZeroValue(); diff --git a/storage/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp b/storage/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp index fa400b565b2..e49eb8842ff 100644 --- a/storage/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp +++ b/storage/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp @@ -538,9 +538,8 @@ ProtocolSerialization5_0::onEncode(GBBuf& buf, const api::CreateVisitorReply& ms buf.putLong(msg.getVisitorStatistics().getBytesVisited()); buf.putLong(msg.getVisitorStatistics().getDocumentsReturned()); buf.putLong(msg.getVisitorStatistics().getBytesReturned()); - // TODO remove second pass concept on Vespa 8 - buf.putLong(msg.getVisitorStatistics().getSecondPassDocumentsReturned()); - buf.putLong(msg.getVisitorStatistics().getSecondPassBytesReturned()); + buf.putLong(0); + buf.putLong(0); } api::StorageReply::UP @@ -555,9 +554,8 @@ ProtocolSerialization5_0::onDecodeCreateVisitorReply(const SCmd& cmd, BBuf& buf) vs.setBytesVisited(SH::getLong(buf)); vs.setDocumentsReturned(SH::getLong(buf)); vs.setBytesReturned(SH::getLong(buf)); - // TODO remove second pass concept on Vespa 8 - vs.setSecondPassDocumentsReturned(SH::getLong(buf)); - vs.setSecondPassBytesReturned(SH::getLong(buf)); + SH::getLong(buf); + SH::getLong(buf); msg->setVisitorStatistics(vs); return msg; diff --git a/storage/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storage/src/vespa/storageapi/mbusprot/protocolserialization7.cpp index 91b5999e34c..dee766b4b2d 100644 --- a/storage/src/vespa/storageapi/mbusprot/protocolserialization7.cpp +++ b/storage/src/vespa/storageapi/mbusprot/protocolserialization7.cpp @@ -1240,8 +1240,6 @@ void ProtocolSerialization7::onEncode(GBBuf& buf, const api::CreateVisitorReply& proto_stats->set_bytes_visited(stats.getBytesVisited()); proto_stats->set_documents_returned(stats.getDocumentsReturned()); proto_stats->set_bytes_returned(stats.getBytesReturned()); - proto_stats->set_second_pass_documents_returned(stats.getSecondPassDocumentsReturned()); // TODO remove on Vespa 8 - proto_stats->set_second_pass_bytes_returned(stats.getSecondPassBytesReturned()); // TODO remove on Vespa 8 }); } @@ -1287,8 +1285,6 @@ api::StorageReply::UP ProtocolSerialization7::onDecodeCreateVisitorReply(const S vs.setBytesVisited(proto_stats.bytes_visited()); vs.setDocumentsReturned(proto_stats.documents_returned()); vs.setBytesReturned(proto_stats.bytes_returned()); - vs.setSecondPassDocumentsReturned(proto_stats.second_pass_documents_returned()); // TODO remove on Vespa 8 - vs.setSecondPassBytesReturned(proto_stats.second_pass_bytes_returned()); // TODO remove on Vespa 8 reply->setVisitorStatistics(vs); return reply; }); diff --git a/streamingvisitors/pom.xml b/streamingvisitors/pom.xml index 2cc777be593..dfc5e7a0b10 100644 --- a/streamingvisitors/pom.xml +++ b/streamingvisitors/pom.xml @@ -7,11 +7,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vsm</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/tenant-base/pom.xml b/tenant-base/pom.xml index 8d54fe53f46..1d0d8b6f426 100644 --- a/tenant-base/pom.xml +++ b/tenant-base/pom.xml @@ -7,7 +7,7 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>tenant-base</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>Hosted Vespa tenant base</name> <description>Parent POM for all hosted Vespa applications.</description> <url>https://github.com/vespa-engine</url> @@ -155,7 +155,7 @@ </goals> <configuration> <includeTypes>jar,test-jar</includeTypes> - <!-- felix is not needed in the fat jar, and felix-main:7.0.1 fails upon unpacking because of bogus timestamps --> + <!-- felix is not needed in the fat jar --> <excludeGroupIds>org.apache.felix</excludeGroupIds> <outputDirectory>target/fat-test-classes</outputDirectory> <!-- WARNING(2018-06-27): bcpkix-jdk15on-1.58.jar and @@ -358,10 +358,10 @@ <configuration> <rules> <requireJavaVersion> - <version>[11, )</version> + <version>[17, )</version> </requireJavaVersion> <requireMavenVersion> - <version>[3.5, )</version> + <version>[3.6, )</version> </requireMavenVersion> </rules> </configuration> diff --git a/tenant-cd-api/pom.xml b/tenant-cd-api/pom.xml index a43c7f96a47..f8411cc74d1 100644 --- a/tenant-cd-api/pom.xml +++ b/tenant-cd-api/pom.xml @@ -15,7 +15,7 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent</relativePath> </parent> diff --git a/tenant-cd-commons/pom.xml b/tenant-cd-commons/pom.xml index 0ada43bca0b..92347a96f31 100644 --- a/tenant-cd-commons/pom.xml +++ b/tenant-cd-commons/pom.xml @@ -12,7 +12,7 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent</relativePath> </parent> diff --git a/testutil/pom.xml b/testutil/pom.xml index 9df445e8d9f..5c6aa1881f7 100644 --- a/testutil/pom.xml +++ b/testutil/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>testutil</artifactId> <packaging>jar</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>${project.artifactId}</name> <description>Library of useful Hamcrest matchers.</description> <dependencies> diff --git a/vdslib/pom.xml b/vdslib/pom.xml index 0a6eae7ec3e..b4825c63ae9 100644 --- a/vdslib/pom.xml +++ b/vdslib/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vdslib</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>vdslib</name> <description> </description> <dependencies> diff --git a/vdslib/src/main/java/com/yahoo/vdslib/VisitorStatistics.java b/vdslib/src/main/java/com/yahoo/vdslib/VisitorStatistics.java index a79f3d5fb6d..fda456ace05 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/VisitorStatistics.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/VisitorStatistics.java @@ -7,8 +7,6 @@ public class VisitorStatistics { long bytesVisited = 0; long documentsReturned = 0; long bytesReturned = 0; - long secondPassDocumentsReturned = 0; - long secondPassBytesReturned = 0; public void add(VisitorStatistics other) { bucketsVisited += other.bucketsVisited; @@ -16,8 +14,6 @@ public class VisitorStatistics { bytesVisited += other.bytesVisited; documentsReturned += other.documentsReturned; bytesReturned += other.bytesReturned; - secondPassDocumentsReturned += other.secondPassDocumentsReturned; - secondPassBytesReturned += other.secondPassBytesReturned; } public int getBucketsVisited() { return bucketsVisited; } @@ -44,37 +40,13 @@ public class VisitorStatistics { public long getBytesReturned() { return bytesReturned; } public void setBytesReturned(long bytesReturned) { this.bytesReturned = bytesReturned; } - /** - * @deprecated Use getDocumentsReturned() instead - */ - @Deprecated(since = "7", forRemoval = true) // TODO: Vespa 8: remove - public long getSecondPassDocumentsReturned() { return secondPassDocumentsReturned; } - /** - * @deprecated only applies for deprecated "orderdoc" ID scheme - */ - @Deprecated(since = "7", forRemoval = true)// TODO: Vespa 8: remove - public void setSecondPassDocumentsReturned(long secondPassDocumentsReturned) { this.secondPassDocumentsReturned = secondPassDocumentsReturned; } - - /** - * @deprecated Use getBytesReturned() instead - */ - @Deprecated(since = "7", forRemoval = true) // TODO: Vespa 8: remove - public long getSecondPassBytesReturned() { return secondPassBytesReturned; } - /** - * @deprecated only applies for deprecated "orderdoc" ID scheme - */ - @Deprecated(since = "7", forRemoval = true) // TODO: Vespa 8: remove - public void setSecondPassBytesReturned(long secondPassBytesReturned) { this.secondPassBytesReturned = secondPassBytesReturned; } - public String toString() { String out = "Buckets visited: " + bucketsVisited + "\n" + "Documents visited: " + documentsVisited + "\n" + "Bytes visited: " + bytesVisited + "\n" + "Documents returned: " + documentsReturned + "\n" + - "Bytes returned: " + bytesReturned + "\n" + - "Documents returned (2nd pass): " + secondPassDocumentsReturned + "\n" + - "Bytes returned (2nd pass): " + secondPassBytesReturned + "\n"; + "Bytes returned: " + bytesReturned + "\n"; return out; } diff --git a/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java b/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java index b6455a7703c..04a577db554 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java @@ -39,7 +39,6 @@ public class Distribution { private final boolean distributorAutoOwnershipTransferOnWholeGroupDown; } - @SuppressWarnings("removal") // TODO Vespa 8: remove private ConfigSubscriber configSub; private final AtomicReference<Config> config = new AtomicReference<>(new Config(null, 1, false)); @@ -61,7 +60,6 @@ public class Distribution { return p; } - @SuppressWarnings("removal") // TODO Vespa 8: remove // NOTE: keep in sync with the below private ConfigSubscriber.SingleSubscriber<StorDistributionConfig> configSubscriber = config -> { try { @@ -148,7 +146,6 @@ public class Distribution { } } - @SuppressWarnings("removal") // TODO Vespa 8: remove public Distribution(String configId) { try { configSub = new ConfigSubscriber(); diff --git a/vdslib/src/vespa/vdslib/container/visitorstatistics.cpp b/vdslib/src/vespa/vdslib/container/visitorstatistics.cpp index 4dc043c4654..2507b2cfaaa 100644 --- a/vdslib/src/vespa/vdslib/container/visitorstatistics.cpp +++ b/vdslib/src/vespa/vdslib/container/visitorstatistics.cpp @@ -10,9 +10,7 @@ VisitorStatistics::VisitorStatistics() _documentsVisited(0), _bytesVisited(0), _documentsReturned(0), - _bytesReturned(0), - _secondPassDocumentsReturned(0), - _secondPassBytesReturned(0) + _bytesReturned(0) { } @@ -24,8 +22,6 @@ VisitorStatistics::operator+(const VisitorStatistics& other) { vs.setBytesVisited(_bytesVisited + other._bytesVisited); vs.setDocumentsReturned(_documentsReturned + other._documentsReturned); vs.setBytesReturned(_bytesReturned + other._bytesReturned); - vs.setSecondPassDocumentsReturned(_secondPassDocumentsReturned + other._secondPassDocumentsReturned); - vs.setSecondPassBytesReturned(_secondPassBytesReturned + other._secondPassBytesReturned); return vs; } @@ -37,8 +33,6 @@ VisitorStatistics::print(std::ostream& out, bool, const std::string& indent) con out << indent << "Bytes visited: " << _bytesVisited << "\n"; out << indent << "Documents returned: " << _documentsReturned << "\n"; out << indent << "Bytes returned: " << _bytesReturned << "\n"; - out << indent << "Documents returned (2nd pass): " << _secondPassDocumentsReturned << "\n"; - out << indent << "Bytes returned (2nd pass): " << _secondPassBytesReturned << "\n"; } } diff --git a/vdslib/src/vespa/vdslib/container/visitorstatistics.h b/vdslib/src/vespa/vdslib/container/visitorstatistics.h index d4912bbf1ba..3f62d7e602b 100644 --- a/vdslib/src/vespa/vdslib/container/visitorstatistics.h +++ b/vdslib/src/vespa/vdslib/container/visitorstatistics.h @@ -27,12 +27,6 @@ public: uint64_t getBytesReturned() const { return _bytesReturned; } void setBytesReturned(uint32_t bytesReturned) { _bytesReturned = bytesReturned; } - uint64_t getSecondPassDocumentsReturned() const { return _secondPassDocumentsReturned; } - void setSecondPassDocumentsReturned(uint32_t documentsReturned) { _secondPassDocumentsReturned = documentsReturned; } - - uint64_t getSecondPassBytesReturned() const { return _secondPassBytesReturned; } - void setSecondPassBytesReturned(uint32_t bytesReturned) { _secondPassBytesReturned = bytesReturned; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; private: uint32_t _bucketsVisited; @@ -40,8 +34,6 @@ private: uint64_t _bytesVisited; uint64_t _documentsReturned; uint64_t _bytesReturned; - uint64_t _secondPassDocumentsReturned; // TODO remove on Vespa 8 - uint64_t _secondPassBytesReturned; // TODO remove on Vespa 8 }; } diff --git a/vespa_jersey2/.gitignore b/vespa-3party-bundles/.gitignore index ea8c4bf7f35..ea8c4bf7f35 100644 --- a/vespa_jersey2/.gitignore +++ b/vespa-3party-bundles/.gitignore diff --git a/vespa_jersey2/CMakeLists.txt b/vespa-3party-bundles/CMakeLists.txt index ced1db89e30..9451c93ca6f 100644 --- a/vespa_jersey2/CMakeLists.txt +++ b/vespa-3party-bundles/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_jar_dependencies(vespa_jersey2) +install_jar_dependencies(vespa-3party-bundles) diff --git a/vespa_jersey2/OWNERS b/vespa-3party-bundles/OWNERS index 3b2ba1ede81..3b2ba1ede81 100644 --- a/vespa_jersey2/OWNERS +++ b/vespa-3party-bundles/OWNERS diff --git a/vespa_jersey2/README b/vespa-3party-bundles/README index d2775883a75..d2775883a75 100644 --- a/vespa_jersey2/README +++ b/vespa-3party-bundles/README diff --git a/vespa-3party-bundles/pom.xml b/vespa-3party-bundles/pom.xml new file mode 100644 index 00000000000..a286f4bedb8 --- /dev/null +++ b/vespa-3party-bundles/pom.xml @@ -0,0 +1,65 @@ +<?xml version="1.0"?> +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>8-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>vespa-3party-bundles</artifactId> + <version>8-SNAPSHOT</version> + <packaging>pom</packaging> + <dependencies> + + <dependency> + <groupId>javax.ws.rs</groupId> + <artifactId>javax.ws.rs-api</artifactId> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.datatype</groupId> + <artifactId>jackson-datatype-jdk8</artifactId> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.datatype</groupId> + <artifactId>jackson-datatype-jsr310</artifactId> + </dependency> + <dependency> + <!-- TODO Vespa 9 Stop providing servlet-api 3.x --> + <groupId>javax.servlet</groupId> + <artifactId>javax.servlet-api</artifactId> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <!-- Explicit for IntelliJ to detect correct language level from parent --> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>copy-dependencies</id> + <phase>package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + </execution> + </executions> + <configuration> + <overWriteReleases>false</overWriteReleases> + <overWriteSnapshots>false</overWriteSnapshots> + <excludeTransitive>false</excludeTransitive> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/vespa-application-maven-plugin/pom.xml b/vespa-application-maven-plugin/pom.xml index 99248fb0907..d2d960f6112 100644 --- a/vespa-application-maven-plugin/pom.xml +++ b/vespa-application-maven-plugin/pom.xml @@ -6,11 +6,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa-application-maven-plugin</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>maven-plugin</packaging> <description>Maven Plugin for assembling a vespa application package</description> <prerequisites> diff --git a/vespa-athenz/pom.xml b/vespa-athenz/pom.xml index 5a7cb97205e..580910799c1 100644 --- a/vespa-athenz/pom.xml +++ b/vespa-athenz/pom.xml @@ -7,12 +7,12 @@ <modelVersion>4.0.0</modelVersion> <artifactId>vespa-athenz</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <dependencies> diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java index 8a587592c71..5b884e3dfb3 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/DefaultIdentityDocumentClient.java @@ -16,7 +16,6 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.util.EntityUtils; -import org.eclipse.jetty.http.HttpStatus; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLContext; @@ -80,7 +79,8 @@ public class DefaultIdentityDocumentClient implements IdentityDocumentClient { .build(); try (CloseableHttpResponse response = client.execute(request)) { String responseContent = EntityUtils.toString(response.getEntity()); - if (HttpStatus.isSuccess(response.getStatusLine().getStatusCode())) { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode >= 200 && statusCode <= 299) { SignedIdentityDocumentEntity entity = objectMapper.readValue(responseContent, SignedIdentityDocumentEntity.class); return EntityBindingsMapper.toSignedIdentityDocument(entity); } else { @@ -88,7 +88,7 @@ public class DefaultIdentityDocumentClient implements IdentityDocumentClient { String.format( "Failed to retrieve identity document for host %s: %d - %s", host, - response.getStatusLine().getStatusCode(), + statusCode, responseContent)); } } diff --git a/vespa-documentgen-plugin/etc/complex/book.sd b/vespa-documentgen-plugin/etc/complex/book.sd index 68cb2e0b2ad..addc556fc9b 100644 --- a/vespa-documentgen-plugin/etc/complex/book.sd +++ b/vespa-documentgen-plugin/etc/complex/book.sd @@ -25,11 +25,11 @@ search book { field mystruct2 type ss2 {} - field mywsfloat type weightedset<float> { + field mywsinteger type weightedset<int> { indexing: attribute } - field mynestedwsfloat type weightedset<float> {} + field mynestedwsinteger type weightedset<int> {} field myarrayint type array<int> { indexing: attribute @@ -80,9 +80,7 @@ search book { } } rank-profile rp1 inherits default { - #static-rank: weight1 } rank-profile rp2 inherits default { - #static-rank: weight2 } } diff --git a/vespa-documentgen-plugin/etc/localapp/book.sd b/vespa-documentgen-plugin/etc/localapp/book.sd index 58a3c121261..a6f1ed9286f 100644 --- a/vespa-documentgen-plugin/etc/localapp/book.sd +++ b/vespa-documentgen-plugin/etc/localapp/book.sd @@ -25,7 +25,7 @@ search book { field mystruct2 type ss2 {} - field mywsfloat type weightedset<float> { + field mywsinteger type weightedset<int> { indexing: attribute } diff --git a/vespa-documentgen-plugin/pom.xml b/vespa-documentgen-plugin/pom.xml index f3d94fa809d..f3357d64e27 100644 --- a/vespa-documentgen-plugin/pom.xml +++ b/vespa-documentgen-plugin/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa-documentgen-plugin</artifactId> <packaging>maven-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>Vespa DocumentGen Plugin</name> <dependencies> <dependency> diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java index 1fcb639589f..0f3531720ae 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -33,7 +33,6 @@ import java.io.FileWriter; import java.io.IOException; import java.io.Writer; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Date; import java.util.HashMap; @@ -58,19 +57,9 @@ public class DocumentGenMojo extends AbstractMojo { private MavenProject project; /** - * Directory containing the searchdefinition files - * @deprecated use {@link #schemasDirectory} instead - */ - // TODO: Remove in Vespa 8 - @Deprecated - @Parameter(defaultValue = ".", required = false) - private File sdDirectory; - - /** * Directory containing the schema files */ - // TODO: Make this required and with defaultValue "." when sdDirectory is removed in Vespa 8 - @Parameter + @Parameter(defaultValue = ".", required = true) private File schemasDirectory; /** @@ -444,10 +433,7 @@ public class DocumentGenMojo extends AbstractMojo { " */\n" + "@com.yahoo.document.Generated\npublic class "+className+" extends "+superType+" {\n\n"+ ind(1)+"/** The doc type of this.*/\n" + - ind(1)+"public static final com.yahoo.document.DocumentType type = getDocumentType();\n\n"+ - ind(1)+"/** Struct type view of the type of the body of this.*/\n" + - ind(1)+"/** Struct type view of the type of the header of this.*/\n" + - ind(1)+"private static final com.yahoo.document.StructDataType headerStructType = getHeaderStructType();\n\n"); + ind(1)+"public static final com.yahoo.document.DocumentType type = getDocumentType();\n\n"); // Constructor out.write( @@ -461,11 +447,6 @@ public class DocumentGenMojo extends AbstractMojo { // isGenerated() out.write(ind(1)+"@Override protected boolean isGenerated() { return true; }\n\n"); - // Mimic header and body to make serialization work. - // This can be improved by generating a method to serialize the document _here_, and use that in serialization. - exportOverriddenStructGetter(docType.allHeader().getFields(), out, 1, "getHeader", className+".headerStructType"); - exportStructTypeGetter(docType.getName()+".header", docType.allHeader().getFields(), out, 1, "getHeaderStructType", "com.yahoo.document.StructDataType"); - Collection<Field> allUniqueFields = getAllUniqueFields(multiExtends, docType.getAllFields()); exportExtendedStructTypeGetter(className, docType.getName(), allUniqueFields, docType.getFieldSets(), docType.getImportedFieldNames(), out, 1, "getDocumentType", "com.yahoo.document.DocumentType"); @@ -477,7 +458,7 @@ public class DocumentGenMojo extends AbstractMojo { exportEquals(className, allUniqueFields, out, 1); Set<DataType> exportedStructs = exportStructTypes(docType.getTypes(), out, 1, null); if (hasAnyPositionField(allUniqueFields)) { - exportedStructs = exportStructTypes(Arrays.asList(PositionDataType.INSTANCE), out, 1, exportedStructs); + exportedStructs = exportStructTypes(List.of(PositionDataType.INSTANCE), out, 1, exportedStructs); } docTypes.put(docType.getName(), packageName+"."+className); for (DataType exportedStruct : exportedStructs) { @@ -590,16 +571,6 @@ public class DocumentGenMojo extends AbstractMojo { ind(ind)+"}\n\n"); } - private static void exportStructTypeGetter(String name, Collection<Field> fields, Writer out, int ind, String methodName, String retType) throws IOException { - out.write(ind(ind)+"private static "+retType+" "+methodName+"() {\n" + - ind(ind+1)+retType+" ret = new "+retType+"(\""+name+"\");\n"); - for (Field f : fields) { - out.write(ind(ind+1)+"ret.addField(new com.yahoo.document.Field(\""+f.getName()+"\", "+toJavaReference(f.getDataType())+"));\n"); - - } - out.write(ind(ind+1)+"return ret;\n"); - out.write(ind(ind)+"}\n\n"); - } private static void addExtendedField(String className, Field f, Writer out, int ind) throws IOException { out.write(ind(ind)+ "ret.addField(new com.yahoo.document.ExtendedField(\""+f.getName()+"\", " + toJavaReference(f.getDataType()) + ",\n"); out.write(ind(ind+1) + "new com.yahoo.document.ExtendedField.Extract() {\n"); @@ -665,17 +636,6 @@ public class DocumentGenMojo extends AbstractMojo { out.write(ind(ind)+"}\n\n"); } - private static void exportOverriddenStructGetter(Collection<Field> fields, Writer out, int ind, String methodName, String structType) throws IOException { - out.write(ind(ind)+"@Override @Deprecated public com.yahoo.document.datatypes.Struct "+methodName+"() {\n" + - ind(ind+1)+"com.yahoo.document.datatypes.Struct ret = new com.yahoo.document.datatypes.Struct("+structType+");\n"); - for (Field f : fields) { - out.write(ind(ind+1)+"ret.setFieldValue(\""+f.getName()+"\", getFieldValue(getField(\""+f.getName()+"\")));\n"); - - } - out.write(ind(ind+1)+"return ret;\n"); - out.write(ind(ind)+"}\n\n"); - } - /** * Exports the necessary overridden methods from Document/StructuredFieldValue */ @@ -990,12 +950,7 @@ public class DocumentGenMojo extends AbstractMojo { @Override public void execute() { - File dir = sdDirectory; - // Prefer schemasDirectory if set - if (this.schemasDirectory != null) - dir = this.schemasDirectory; - - execute(dir, this.outputDirectory, packageName); + execute(this.schemasDirectory, this.outputDirectory, packageName); } Map<String, Schema> getSearches() { diff --git a/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java b/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java index c619a1ff56b..2a7eb261a0b 100644 --- a/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java +++ b/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java @@ -36,8 +36,8 @@ public class DocumentGenTest { assertTrue(searches.get("music3").getDocument("music3").getField("pos").getDataType() instanceof StructDataType); assertEquals(searches.get("music3").getDocument("music3").getField("pos").getDataType().getName(), "position"); assertTrue(searches.get("book").getDocument("book").getField("mystruct").getDataType() instanceof StructDataType); - assertTrue(searches.get("book").getDocument("book").getField("mywsfloat").getDataType() instanceof WeightedSetDataType); - assertTrue(((WeightedSetDataType)(searches.get("book").getDocument("book").getField("mywsfloat").getDataType())).getNestedType() == DataType.FLOAT); + assertTrue(searches.get("book").getDocument("book").getField("mywsinteger").getDataType() instanceof WeightedSetDataType); + assertTrue(((WeightedSetDataType)(searches.get("book").getDocument("book").getField("mywsinteger").getDataType())).getNestedType() == DataType.INT); } @Test @@ -47,8 +47,8 @@ public class DocumentGenTest { Map<String, Schema> searches = mojo.getSearches(); assertEquals(searches.get("video").getDocument("video").getField("weight").getDataType(), DataType.FLOAT); assertTrue(searches.get("book").getDocument("book").getField("mystruct").getDataType() instanceof StructDataType); - assertTrue(searches.get("book").getDocument("book").getField("mywsfloat").getDataType() instanceof WeightedSetDataType); - assertTrue(((WeightedSetDataType)(searches.get("book").getDocument("book").getField("mywsfloat").getDataType())).getNestedType() == DataType.FLOAT); + assertTrue(searches.get("book").getDocument("book").getField("mywsinteger").getDataType() instanceof WeightedSetDataType); + assertTrue(((WeightedSetDataType)(searches.get("book").getDocument("book").getField("mywsinteger").getDataType())).getNestedType() == DataType.INT); } @Test diff --git a/vespa-feed-client-api/pom.xml b/vespa-feed-client-api/pom.xml index 64b71ae6fb1..5509c339eee 100644 --- a/vespa-feed-client-api/pom.xml +++ b/vespa-feed-client-api/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa-feed-client-api</artifactId> <packaging>jar</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <!-- compile scope --> diff --git a/vespa-feed-client-cli/pom.xml b/vespa-feed-client-cli/pom.xml index 04f9e308cfa..46679906fc4 100644 --- a/vespa-feed-client-cli/pom.xml +++ b/vespa-feed-client-cli/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa-feed-client-cli</artifactId> <packaging>jar</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <!-- compile scope --> diff --git a/vespa-feed-client/pom.xml b/vespa-feed-client/pom.xml index a7c4ab0a13c..536637bdce2 100644 --- a/vespa-feed-client/pom.xml +++ b/vespa-feed-client/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa-feed-client</artifactId> <packaging>jar</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <!-- compile scope --> diff --git a/vespa-hadoop/pom.xml b/vespa-hadoop/pom.xml index 5da226f10e6..8060a2d67a1 100644 --- a/vespa-hadoop/pom.xml +++ b/vespa-hadoop/pom.xml @@ -7,11 +7,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa-hadoop</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>${project.artifactId}</name> <description>Integration tools between Vespa and Hadoop</description> @@ -21,6 +21,17 @@ <pig.version>0.14.0</pig.version> </properties> + <dependencyManagement> + <dependencies> + <!-- Force newer version of jetty-util (to match Jetty version of jdisc) --> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-util</artifactId> + <version>${jetty.version}</version> + </dependency> + </dependencies> + </dependencyManagement> + <dependencies> <!-- Hadoop dependencies --> <dependency> @@ -125,11 +136,6 @@ <!-- Vespa feeding dependencies --> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>vespa-http-client</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>vespa-feed-client</artifactId> <version>${project.version}</version> </dependency> diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/LegacyVespaRecordWriter.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/LegacyVespaRecordWriter.java deleted file mode 100644 index 6900c7dc82f..00000000000 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/LegacyVespaRecordWriter.java +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hadoop.mapreduce; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; -import com.yahoo.vespa.hadoop.mapreduce.util.VespaConfiguration; -import com.yahoo.vespa.hadoop.mapreduce.util.VespaCounters; -import com.yahoo.vespa.hadoop.pig.VespaDocumentOperation; -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.config.FeedParams.DataFormat; -import com.yahoo.vespa.http.client.config.SessionParams; -import org.apache.hadoop.mapreduce.RecordWriter; -import org.apache.hadoop.mapreduce.TaskAttemptContext; - -import javax.xml.namespace.QName; -import javax.xml.stream.FactoryConfigurationError; -import javax.xml.stream.XMLEventReader; -import javax.xml.stream.XMLInputFactory; -import javax.xml.stream.XMLStreamException; -import javax.xml.stream.events.StartElement; -import javax.xml.stream.events.XMLEvent; -import java.io.IOException; -import java.io.StringReader; -import java.time.Duration; -import java.util.List; -import java.util.StringTokenizer; -import java.util.concurrent.ThreadLocalRandom; -import java.util.logging.Logger; - -/** - * {@link LegacyVespaRecordWriter} sends the output <key, value> to one or more Vespa endpoints using vespa-http-client. - * - * @author lesters - * @deprecated Replaced by {@link VespaRecordWriter} - */ -@Deprecated -public class LegacyVespaRecordWriter extends RecordWriter<Object, Object> { - - private final Logger log = Logger.getLogger(getClass().getCanonicalName()); - - private boolean initialized = false; - private com.yahoo.vespa.http.client.FeedClient feedClient; - private final VespaCounters counters; - private final int progressInterval; - - final VespaConfiguration configuration; - - LegacyVespaRecordWriter(VespaConfiguration configuration, VespaCounters counters) { - this.counters = counters; - this.configuration = configuration; - this.progressInterval = configuration.progressInterval(); - } - - - @Override - public void write(Object key, Object data) throws IOException, InterruptedException { - if (!initialized) { - initialize(); - } - - String doc = data.toString().trim(); - - // Parse data to find document id - if none found, skip this write - String docId = DataFormat.JSON_UTF8.equals(configuration.dataFormat()) ? findDocId(doc) - : findDocIdFromXml(doc); - if (docId != null && docId.length() >= 0) { - feedClient.stream(docId, doc); - counters.incrementDocumentsSent(1); - } else { - counters.incrementDocumentsSkipped(1); - } - - if (counters.getDocumentsSent() % progressInterval == 0) { - String progress = String.format("Feed progress: %d / %d / %d / %d (sent, ok, failed, skipped)", - counters.getDocumentsSent(), - counters.getDocumentsOk(), - counters.getDocumentsFailed(), - counters.getDocumentsSkipped()); - log.info(progress); - } - - } - - - @Override - public void close(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { - if (feedClient != null) { - feedClient.close(); - } - } - - protected ConnectionParams.Builder configureConnectionParams() { - ConnectionParams.Builder connParamsBuilder = new ConnectionParams.Builder(); - connParamsBuilder.setDryRun(configuration.dryrun()); - connParamsBuilder.setUseCompression(configuration.useCompression()); - connParamsBuilder.setNumPersistentConnectionsPerEndpoint(configuration.numConnections()); - connParamsBuilder.setMaxRetries(configuration.numRetries()); - if (configuration.proxyHost() != null) { - connParamsBuilder.setProxyHost(configuration.proxyHost()); - } - if (configuration.proxyPort() >= 0) { - connParamsBuilder.setProxyPort(configuration.proxyPort()); - } - return connParamsBuilder; - } - - protected FeedParams.Builder configureFeedParams() { - FeedParams.Builder feedParamsBuilder = new FeedParams.Builder(); - feedParamsBuilder.setDataFormat(configuration.dataFormat()); - feedParamsBuilder.setRoute(configuration.route()); - feedParamsBuilder.setMaxSleepTimeMs(configuration.maxSleepTimeMs()); - feedParamsBuilder.setMaxInFlightRequests(configuration.maxInFlightRequests()); - feedParamsBuilder.setLocalQueueTimeOut(Duration.ofMinutes(10).toMillis()); - return feedParamsBuilder; - } - - protected SessionParams.Builder configureSessionParams() { - SessionParams.Builder sessionParamsBuilder = new SessionParams.Builder(); - sessionParamsBuilder.setThrottlerMinSize(configuration.throttlerMinSize()); - sessionParamsBuilder.setClientQueueSize(configuration.maxInFlightRequests()*2); - return sessionParamsBuilder; - } - - private void initialize() { - if (!configuration.dryrun() && configuration.randomStartupSleepMs() > 0) { - int delay = ThreadLocalRandom.current().nextInt(configuration.randomStartupSleepMs()); - log.info("VespaStorage: Delaying startup by " + delay + " ms"); - try { - Thread.sleep(delay); - } catch (Exception e) {} - } - - ConnectionParams.Builder connParamsBuilder = configureConnectionParams(); - FeedParams.Builder feedParamsBuilder = configureFeedParams(); - SessionParams.Builder sessionParams = configureSessionParams(); - - sessionParams.setConnectionParams(connParamsBuilder.build()); - sessionParams.setFeedParams(feedParamsBuilder.build()); - - String endpoints = configuration.endpoint(); - StringTokenizer tokenizer = new StringTokenizer(endpoints, ","); - while (tokenizer.hasMoreTokens()) { - String endpoint = tokenizer.nextToken().trim(); - sessionParams.addCluster(new Cluster.Builder().addEndpoint( - Endpoint.create(endpoint, configuration.defaultPort(), configuration.useSSL().orElse(false)) - ).build()); - } - - ResultCallback resultCallback = new ResultCallback(counters); - feedClient = com.yahoo.vespa.http.client.FeedClientFactory.create(sessionParams.build(), resultCallback); - - initialized = true; - log.info("VespaStorage configuration:\n" + configuration.toString()); - log.info(feedClient.getStatsAsJson()); - } - - private String findDocIdFromXml(String xml) { - try { - XMLEventReader eventReader = XMLInputFactory.newInstance().createXMLEventReader(new StringReader(xml)); - while (eventReader.hasNext()) { - XMLEvent event = eventReader.nextEvent(); - if (event.getEventType() == XMLEvent.START_ELEMENT) { - StartElement element = event.asStartElement(); - String elementName = element.getName().getLocalPart(); - if (VespaDocumentOperation.Operation.valid(elementName)) { - return element.getAttributeByName(QName.valueOf("documentid")).getValue(); - } - } - } - } catch (XMLStreamException | FactoryConfigurationError e) { - // as json dude does - return null; - } - return null; - } - - private String findDocId(String json) throws IOException { - JsonFactory factory = new JsonFactory(); - try(JsonParser parser = factory.createParser(json)) { - if (parser.nextToken() != JsonToken.START_OBJECT) { - return null; - } - while (parser.nextToken() != JsonToken.END_OBJECT) { - String fieldName = parser.getCurrentName(); - parser.nextToken(); - if (VespaDocumentOperation.Operation.valid(fieldName)) { - String docId = parser.getText(); - return docId; - } else { - parser.skipChildren(); - } - } - } catch (JsonParseException ex) { - return null; - } - return null; - } - - - class ResultCallback implements com.yahoo.vespa.http.client.FeedClient.ResultCallback { - final VespaCounters counters; - - public ResultCallback(VespaCounters counters) { - this.counters = counters; - } - - @Override - public void onCompletion(String docId, com.yahoo.vespa.http.client.Result documentResult) { - if (!documentResult.isSuccess()) { - counters.incrementDocumentsFailed(1); - StringBuilder sb = new StringBuilder(); - sb.append("Problems with docid "); - sb.append(docId); - sb.append(": "); - List<com.yahoo.vespa.http.client.Result.Detail> details = documentResult.getDetails(); - for (com.yahoo.vespa.http.client.Result.Detail detail : details) { - sb.append(detail.toString()); - sb.append(" "); - } - log.warning(sb.toString()); - return; - } - counters.incrementDocumentsOk(1); - } - - } - -} diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java index 66ab94574d9..e49a5e17970 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaOutputFormat.java @@ -10,15 +10,12 @@ import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; import java.io.IOException; -import java.util.Objects; import java.util.Properties; import java.util.logging.Logger; -import static com.yahoo.vespa.http.client.config.FeedParams.DataFormat.XML_UTF8; - /** * An output specification for writing to Vespa instances in a Map-Reduce job. - * Mainly returns an instance of a {@link LegacyVespaRecordWriter} that does the + * Mainly returns an instance of a {@link VespaRecordWriter} that does the * actual feeding to Vespa. * * @author lesters @@ -46,14 +43,7 @@ public class VespaOutputFormat extends OutputFormat { public RecordWriter getRecordWriter(TaskAttemptContext context) throws IOException { VespaCounters counters = VespaCounters.get(context); VespaConfiguration configuration = VespaConfiguration.get(context.getConfiguration(), configOverride); - Boolean useLegacyClient = configuration.useLegacyClient().orElse(null); - if (Objects.equals(useLegacyClient, Boolean.TRUE) || configuration.dataFormat() == XML_UTF8) { - log.warning("Feeding with legacy client or XML will no longer be supported on Vespa 8. " + - "See https://docs.vespa.ai/en/vespa8-release-notes.html"); - return new LegacyVespaRecordWriter(configuration, counters); - } else { - return new VespaRecordWriter(configuration, counters); - } + return new VespaRecordWriter(configuration, counters); } diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java index 6d6c3789835..c450d7cdeef 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java @@ -7,7 +7,6 @@ import ai.vespa.feed.client.JsonFeeder; import ai.vespa.feed.client.OperationParseException; import com.yahoo.vespa.hadoop.mapreduce.util.VespaConfiguration; import com.yahoo.vespa.hadoop.mapreduce.util.VespaCounters; -import com.yahoo.vespa.http.client.config.FeedParams; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; @@ -85,18 +84,11 @@ public class VespaRecordWriter extends RecordWriter<Object, Object> { private void initializeOnFirstWrite() { if (initialized) return; - validateConfig(); useRandomizedStartupDelayIfEnabled(); feeder = createJsonStreamFeeder(); initialized = true; } - private void validateConfig() { - if (config.dataFormat() != FeedParams.DataFormat.JSON_UTF8) { - throw new IllegalArgumentException("Only JSON is support by this feed client implementation"); - } - } - private void useRandomizedStartupDelayIfEnabled() { if (!config.dryrun() && config.randomStartupSleepMs() > 0) { int delay = ThreadLocalRandom.current().nextInt(config.randomStartupSleepMs()); diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java index 715546fe6fe..ae0b6a58155 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hadoop.mapreduce.util; -import com.yahoo.vespa.http.client.config.FeedParams; import org.apache.hadoop.conf.Configuration; import java.io.IOException; @@ -19,7 +18,6 @@ public class VespaConfiguration { public static final String PROXY_SCHEME = "vespa.feed.proxy.scheme"; public static final String DRYRUN = "vespa.feed.dryrun"; public static final String USE_COMPRESSION = "vespa.feed.usecompression"; - public static final String DATA_FORMAT = "vespa.feed.data.format"; public static final String PROGRESS_REPORT = "vespa.feed.progress.interval"; public static final String CONNECTIONS = "vespa.feed.connections"; public static final String THROTTLER_MIN_SIZE = "vespa.feed.throttler.min.size"; @@ -29,7 +27,6 @@ public class VespaConfiguration { public static final String MAX_IN_FLIGHT_REQUESTS = "vespa.feed.max.in.flight.requests"; public static final String RANDOM_STARTUP_SLEEP = "vespa.feed.random.startup.sleep.ms"; public static final String NUM_RETRIES = "vespa.feed.num.retries"; - public static final String USE_LEGACY_CLIENT = "vespa.feed.uselegacyclient"; private final Configuration conf; private final Properties override; @@ -129,25 +126,10 @@ public class VespaConfiguration { } - public FeedParams.DataFormat dataFormat() { - String format = getString(DATA_FORMAT); - if ("xml".equalsIgnoreCase(format)) { - return FeedParams.DataFormat.XML_UTF8; - } - return FeedParams.DataFormat.JSON_UTF8; - } - - public int progressInterval() { return getInt(PROGRESS_REPORT, 1000); } - public Optional<Boolean> useLegacyClient() { - String raw = getString(USE_LEGACY_CLIENT); - if (raw == null || raw.trim().isEmpty()) return Optional.empty(); - return Optional.of(Boolean.parseBoolean(raw)); - } - public String getString(String name) { if (override != null && override.containsKey(name)) { return override.getProperty(name); @@ -197,7 +179,6 @@ public class VespaConfiguration { sb.append(PROXY_SCHEME + ": " + proxyScheme() + "\n"); sb.append(DRYRUN + ": " + dryrun() +"\n"); sb.append(USE_COMPRESSION + ": " + useCompression() +"\n"); - sb.append(DATA_FORMAT + ": " + dataFormat() +"\n"); sb.append(PROGRESS_REPORT + ": " + progressInterval() +"\n"); sb.append(CONNECTIONS + ": " + numConnections() +"\n"); sb.append(THROTTLER_MIN_SIZE + ": " + throttlerMinSize() +"\n"); @@ -207,7 +188,6 @@ public class VespaConfiguration { sb.append(MAX_IN_FLIGHT_REQUESTS + ": " + maxInFlightRequests() +"\n"); sb.append(RANDOM_STARTUP_SLEEP + ": " + randomStartupSleepMs() +"\n"); sb.append(NUM_RETRIES + ": " + numRetries() +"\n"); - sb.append(USE_LEGACY_CLIENT + ": " + useLegacyClient().map(Object::toString).orElse("<empty>") +"\n"); return sb.toString(); } diff --git a/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaStorageTest.java b/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaStorageTest.java index f690e767194..3183c770bc7 100644 --- a/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaStorageTest.java +++ b/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaStorageTest.java @@ -24,14 +24,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; public class VespaStorageTest { @Test - public void requireThatPremadeXmlOperationsFeedSucceeds() throws Exception { - Configuration conf = new HdfsConfiguration(); - conf.set(VespaConfiguration.DATA_FORMAT, "xml"); - assertAllDocumentsOk("src/test/pig/feed_operations_xml.pig", conf); - } - - - @Test public void requireThatPremadeOperationsFeedSucceeds() throws Exception { assertAllDocumentsOk("src/test/pig/feed_operations.pig"); } @@ -52,7 +44,6 @@ public class VespaStorageTest { public void requireThatPremadeOperationsWithJsonLoaderFeedAndNonLegacyClientSucceeds() throws Exception { Configuration conf = new HdfsConfiguration(); conf.set(VespaConfiguration.USE_SSL, Boolean.TRUE.toString()); - conf.set(VespaConfiguration.USE_LEGACY_CLIENT, Boolean.FALSE.toString()); assertAllDocumentsOk("src/test/pig/feed_operations_with_json_loader.pig", conf); } diff --git a/vespa-http-client/.gitignore b/vespa-http-client/.gitignore deleted file mode 100644 index 12251442258..00000000000 --- a/vespa-http-client/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/target -/pom.xml.build diff --git a/vespa-http-client/OWNERS b/vespa-http-client/OWNERS deleted file mode 100644 index dbcff24b338..00000000000 --- a/vespa-http-client/OWNERS +++ /dev/null @@ -1 +0,0 @@ -vekterli diff --git a/vespa-http-client/README b/vespa-http-client/README deleted file mode 100644 index 49d375316f6..00000000000 --- a/vespa-http-client/README +++ /dev/null @@ -1,5 +0,0 @@ -Java API for feeding to Vespa from external sources - -Cross-colo etc. - -NOTE: This must be built with JDK 8, because it's used on the grid. diff --git a/vespa-http-client/abi-spec.json b/vespa-http-client/abi-spec.json deleted file mode 100644 index 9e26dfeeb6e..00000000000 --- a/vespa-http-client/abi-spec.json +++ /dev/null @@ -1 +0,0 @@ -{}
\ No newline at end of file diff --git a/vespa-http-client/pom.xml b/vespa-http-client/pom.xml deleted file mode 100644 index 0448b2b862b..00000000000 --- a/vespa-http-client/pom.xml +++ /dev/null @@ -1,219 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> - </parent> - <artifactId>vespa-http-client</artifactId> - <version>7-SNAPSHOT</version> - <name>${project.artifactId}</name> - <description>Independent external feeding API towards Vespa.</description> - - <dependencies> - - <!-- NOTE: Adding dependencies here may break clients because this is used outside an OSGi container with - manually set up classpaths and possibly no access to the with-dependencies jar. - - Vespa dependencies should not be added. - --> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.dataformat</groupId> - <artifactId>jackson-dataformat-xml</artifactId> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.datatype</groupId> - <artifactId>jackson-datatype-jsr310</artifactId> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.datatype</groupId> - <artifactId>jackson-datatype-jdk8</artifactId> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>io.airlift</groupId> - <artifactId>airline</artifactId> - <version>0.6</version> - </dependency> - <dependency> - <!-- Needed for Vespa TLS configuration. Standard jar artifact --> - <groupId>com.yahoo.vespa</groupId> - <artifactId>http-utils</artifactId> - <version>${project.version}</version> - <scope>compile</scope> - <exclusions> - <!-- Optimization: exclude artifacts that are not used by this module --> - <exclusion> - <groupId>io.airlift</groupId> - <artifactId>aircompressor</artifactId> - </exclusion> - <exclusion> - <groupId>net.java.dev.jna</groupId> - <artifactId>jna</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.commons</groupId> - <artifactId>commons-exec</artifactId> - </exclusion> - <exclusion> - <groupId>org.lz4</groupId> - <artifactId>lz4-java</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <!-- Needed for Vespa TLS configuration. --> - <!-- This artifact is packaged as an OSGi bundle - make sure to manually include or exclude transitive dependencies as necessary --> - <!-- Note: includes BouncyCastle to compile scope transitively. --> - <groupId>com.yahoo.vespa</groupId> - <artifactId>security-utils</artifactId> - <version>${project.version}</version> - <scope>compile</scope> - </dependency> - - <!-- Test dependencies --> - <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-core</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>testutil</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-server</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> - <scope>test</scope> - </dependency> - - </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>exec-maven-plugin</artifactId> - <executions> - <execution> - <id>generate-simplified-vtag</id> - <phase>generate-sources</phase> - <goals> - <goal>exec</goal> - </goals> - <configuration> - <executable>src/main/bin/versiontagger.sh</executable> - <arguments> - <argument>${project.basedir}/../dist/vtag.map</argument> - <argument>${project.build.directory}/generated-sources/vtag/com/yahoo/vespa/http/client/core/Vtag.java</argument> - </arguments> - <sourceRoot>${project.build.directory}/generated-sources/vtag</sourceRoot> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <release>${vespaClients.jdk.releaseVersion}</release> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Xlint:-deprecation</arg> - <arg>-Xlint:-serial</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - <configuration> - <finalName>${project.artifactId}-jar-with-dependencies</finalName> - <createDependencyReducedPom>false</createDependencyReducedPom> - <transformers> - <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> - <mainClass>com.yahoo.vespa.http.client.runner.Runner</mainClass> - </transformer> - </transformers> - <shadeSourcesContent>true</shadeSourcesContent> - <!-- Let the user choose any apache lib version they want. --> - <relocations> - <relocation> - <pattern>org.apache.http</pattern> - <shadedPattern>com.yahoo.vespa.feeder.shaded.internal.apache.http</shadedPattern> - </relocation> - <relocation> - <pattern>org.apache.commons</pattern> - <shadedPattern>com.yahoo.vespa.feeder.shaded.internal.apache.commons</shadedPattern> - </relocation> - </relocations> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - <executions> - <execution> - <id>attach-artifacts</id> - <phase>package</phase> - <goals> - <goal>attach-artifact</goal> - </goals> - <configuration> - <artifacts> - <artifact> - <file>target/${project.artifactId}-jar-with-dependencies.jar</file> - <type>jar</type> - <classifier>jar-with-dependencies</classifier> - </artifact> - </artifacts> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> -</project> diff --git a/vespa-http-client/src/main/bin/versiontagger.sh b/vespa-http-client/src/main/bin/versiontagger.sh deleted file mode 100755 index f4e4c23e50a..00000000000 --- a/vespa-http-client/src/main/bin/versiontagger.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -# Extracts the current version number (V_TAG_COMPONENT) from vtag.map and outputs this into a Java class. -# This replaces vespajlib/../VersionTagger.java as this module cannot depend on that, nor the dependencies -# of the resulting class. -# -# Author: bratseth - -source=$1 -destination=$2 -destinationDir=$(dirname $destination) - -mkdir -p $destinationDir - -versionNumber=$(cat $source | grep V_TAG_COMPONENT | awk '{print $2}' ) - -cat > $destination <<- END -package com.yahoo.vespa.http.client.core; - -public class Vtag { - public static final String V_TAG_COMPONENT = "$versionNumber"; -} -END diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java deleted file mode 100644 index d9ff09552ef..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import com.yahoo.vespa.http.client.core.JsonReader; -import com.yahoo.vespa.http.client.core.XmlFeedReader; - -import java.io.InputStream; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * API for feeding document operations (add, removes or updates) to one or many Vespa clusters. - * Use the factory to configure and set up an instance of this. Instances are expensive - create one instance of this - * and use it for all feed operations (from multiple threads, if desired) for the duration of your client runtime. - * The feedclient does automatic error recovery and reconnects to hosts when connections die. - * - * A {@link FeedClientFactory} is provided to instantiate Sessions. - * - * See com.yahoo.text.Text.stripInvalidCharacters(String) to remove invalid characters from string fields before feeding - * - * Instances of this are multithread safe. - * - * @author dybis - * @see FeedClientFactory - * @deprecated Vespa-http-client will be removed in Vespa 8. It's replaced by <a href="https://docs.vespa.ai/en/vespa-feed-client.html">vespa-feed-client</a> - */ -@Deprecated -public interface FeedClient extends AutoCloseable { - - /** - * Issues a document operation to the configured cluster(s). - * If the pipeline and buffers are full, this call will be blocking, ensuring that operations are not - * produced faster than the can be handled. Transient failured are retried internally by this client. - * Exactly one callback will always be received for each (completed) call to this. - * - * @param documentId the document id of the document - * @param documentData the document data as JSON or XML (as specified when using the factory to create the API) - */ - default void stream(String documentId, CharSequence documentData) { - stream(documentId, documentData, null); - } - - /** - * Issues a document operation to the configured cluster(s). - * If the pipeline and buffers are full, this call will be blocking, ensuring that operations are not - * produced faster than the can be handled. Transient failured are retried internally by this client. - * Exactly one callback will always be received for each (completed) call to this. - * - * @param documentId the document id of the document - * @param documentData the document data as JSON or XML (as specified when using the factory to create the API) - * @param context a context object which will be accessible in the result of the callback, or null if none - */ - default void stream(String documentId, CharSequence documentData, Object context) { - stream(documentId, null, documentData, context); - } - - /** - * Issues a document operation to the configured cluster(s). - * If the pipeline and buffers are full, this call will be blocking, ensuring that operations are not - * produced faster than the can be handled. Transient failures are retried internally by this client. - * Exactly one callback will always be received for each (completed) call to this. - * - * @param documentId the document id of the document - * @param operationId the id to use for this operation, or null to let the client decide an operation id. - * This id must be unique for every operation. Passing the operation id allows clients - * to prepare to receive a response for it before issuing the operation to the client. - * @param documentData the document data as JSON or XML (as specified when using the factory to create the API) - * @param context a context object which will be accessible in the result of the callback, or null if none - */ - void stream(String documentId, String operationId, CharSequence documentData, Object context); - - /** - * Waits for all results to arrive and closes the FeedClient. Don't call any other method after calling close(). - * Does not throw any exceptions. - */ - @Override - void close(); - - /** - * Returns stats about the cluster - * - * @return JSON string with information about cluster - */ - String getStatsAsJson(); - - /** - * Utility function that takes an array of JSON documents and calls the FeedClient for each element. - * - * @param inputStream the stream to feed. This can be a very large stream. - * The outer element must be an array of document operations. - * @param feedClient the feed client that will receive the document operations - * @param numSent increased per document sent to API (but not waiting for results) - */ - static void feedJson(InputStream inputStream, FeedClient feedClient, AtomicInteger numSent) { - JsonReader.read(inputStream, feedClient, numSent); - } - - /** - * Utility function that takes an array of XML documents and calls the FeedClient for each element. - * The XML document has to be formatted with line space on each line (like "regular" XML, but stricter - * than the specifications of XML). - * - * @param inputStream the stream to feed. This can be a very large stream. Operations must be enclosed in a - * top-level <vespafeed> tag - * @param feedClient the feed client that will receive the document operations - * @param numSent increased per document sent to API (but not waiting for results) - */ - static void feedXml(InputStream inputStream, FeedClient feedClient, AtomicInteger numSent) { - try { - XmlFeedReader.read(inputStream, feedClient, numSent); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - /** - * This callback is executed when new results are arriving or an error occur. - * Don't do any heavy lifting in this thread (no IO, disk, or heavy CPU usage). - * This call back will run in a different thread than your main program so use e.g. - * AtomicInteger for counters and follow general guides for thread-safe programming. - * There is an example implementation in class SimpleLoggerResultCallback. - */ - interface ResultCallback { - - /** - * This callback is always called exactly once for each feed operation passed to the client - * instance, whether or not it was successful. - */ - void onCompletion(String docId, Result documentResult); - - /** - * Called with an exception whenever an endpoint specific error occurs during feeding. - * The error may or may not be transient - the operation will in both cases be retried until it's successful. - * This callback is intended for application level monitoring (logging, metrics, altering etc). - * Document specific errors will be reported back through {@link #onCompletion(String, Result)}. - * - * @see FeedEndpointException - * @param exception an exception specifying endpoint and cause. See {@link FeedEndpointException} for details. - */ - default void onEndpointException(FeedEndpointException exception) {} - - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClientFactory.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClientFactory.java deleted file mode 100644 index ce2f9e0b140..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClientFactory.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - - -import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.core.api.FeedClientImpl; - -import java.time.Clock; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.ThreadFactory; - -/** - * Factory for creating FeedClient. - * - * @author dybis - * @deprecated Vespa-http-client will be removed in Vespa 8. It's Vespa-http-client will be removed in Vespa 8. It's replaced by <a href="https://docs.vespa.ai/en/vespa-feed-client.html">vespa-feed-client</a> - */ -@Deprecated -public class FeedClientFactory { - - /** - * Creates a FeedClient. Call this sparingly: Feed clients are expensive and should be as long-lived as possible. - * - * @param sessionParams parameters for connection, hosts, cluster configurations and more. - * @param resultCallback on each result, this callback is called. - * @return newly created FeedClient API object. - */ - public static FeedClient create(SessionParams sessionParams, FeedClient.ResultCallback resultCallback) { - return new FeedClientImpl(sessionParams, resultCallback, createTimeoutExecutor(), Clock.systemUTC()); - } - - static ScheduledThreadPoolExecutor createTimeoutExecutor() { - ScheduledThreadPoolExecutor timeoutExecutor; - timeoutExecutor = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("timeout-")); - timeoutExecutor.setRemoveOnCancelPolicy(true); - timeoutExecutor.setContinueExistingPeriodicTasksAfterShutdownPolicy(false); - timeoutExecutor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); - return timeoutExecutor; - } - - private static class DaemonThreadFactory implements ThreadFactory { - - private final ThreadFactory defaultThreadFactory = Executors.defaultThreadFactory(); - private final String prefix; - - private DaemonThreadFactory(String prefix) { - this.prefix = prefix; - } - - @Override - public Thread newThread(Runnable runnable) { - Thread t = defaultThreadFactory.newThread(runnable); - t.setDaemon(true); - t.setName(prefix + t.getName()); - return t; - } - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedConnectException.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedConnectException.java deleted file mode 100644 index 822a670ae08..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedConnectException.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import com.yahoo.vespa.http.client.config.Endpoint; - -/** - * An exception thrown when the client is unable to connect to a feed endpoint. - * - * @author bjorncs - * @deprecated Vespa-http-client will be removed in Vespa 8. It's replaced by <a href="https://docs.vespa.ai/en/vespa-feed-client.html">vespa-feed-client</a> - */ -@Deprecated -public class FeedConnectException extends FeedEndpointException { - - public FeedConnectException(Throwable cause, Endpoint endpoint) { - super(createMessage(cause, endpoint), cause, endpoint); - } - - private static String createMessage(Throwable cause, Endpoint endpoint) { - return String.format("Handshake to endpoint '%s:%d' failed: %s", - endpoint.getHostname(), - endpoint.getPort(), - cause.getMessage()); - } -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedEndpointException.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedEndpointException.java deleted file mode 100644 index 304e2ea321b..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedEndpointException.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import com.yahoo.vespa.http.client.config.Endpoint; - -/** - * An exception type for endpoint specific errors. - * - * @see FeedConnectException - * @see FeedProtocolException - * @author bjorncs - * @deprecated Vespa-http-client will be removed in Vespa 8. It's replaced by <a href="https://docs.vespa.ai/en/vespa-feed-client.html">vespa-feed-client</a> - */ -@Deprecated -public abstract class FeedEndpointException extends RuntimeException { - - private final Endpoint endpoint; - - protected FeedEndpointException(String message, Throwable cause, Endpoint endpoint) { - super(message, cause); - this.endpoint = endpoint; - } - - public Endpoint getEndpoint() { - return endpoint; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedProtocolException.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedProtocolException.java deleted file mode 100644 index 93041cced1c..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedProtocolException.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import com.yahoo.vespa.http.client.config.Endpoint; - -/** - * An exception thrown when a feed endpoint returns an error during feeding. - * - * @author bjorncs - * @deprecated Vespa-http-client will be removed in Vespa 8. It's replaced by <a href="https://docs.vespa.ai/en/vespa-feed-client.html">vespa-feed-client</a> - */ -@Deprecated -public class FeedProtocolException extends FeedEndpointException { - - private final int httpStatusCode; - private final String httpResponseMessage; - - public FeedProtocolException(int httpStatusCode, - String httpResponseMessage, - Throwable cause, - Endpoint endpoint) { - super(createMessage(httpStatusCode, httpResponseMessage, endpoint), cause, endpoint); - this.httpStatusCode = httpStatusCode; - this.httpResponseMessage = httpResponseMessage; - } - - private static String createMessage(int httpStatusCode, - String httpResponseMessage, - Endpoint endpoint) { - return String.format("Endpoint '%s:%d' returned an error on handshake: %d - %s", - endpoint.getHostname(), - endpoint.getPort(), - httpStatusCode, - httpResponseMessage); - } - - public int getHttpStatusCode() { - return httpStatusCode; - } - - public String getHttpResponseMessage() { - return httpResponseMessage; - } -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Result.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Result.java deleted file mode 100644 index 5db592da02f..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Result.java +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.Exceptions; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; - -/** - * The result of a stream operation. A Result refers to a single document, - * but may contain more than one Result.Detail instances, as these pertains to a - * single endpoint, and a Result may wrap data for multiple endpoints. - * - * @author Einar M R Rosenvinge - * @deprecated Vespa-http-client will be removed in Vespa 8. It's replaced by <a href="https://docs.vespa.ai/en/vespa-feed-client.html">vespa-feed-client</a> - */ -@Deprecated -public class Result { - - public enum ResultType { - OPERATION_EXECUTED, - TRANSITIVE_ERROR, - CONDITION_NOT_MET, - FATAL_ERROR - } - - private final Document document; - private final boolean success; - private final List<Detail> details; - private final String localTrace; - - public Result(Document document, Collection<Detail> values, StringBuilder localTrace) { - this.document = document; - this.details = Collections.unmodifiableList(new ArrayList<>(values)); - this.success = details.stream().allMatch(d -> d.getResultType() == ResultType.OPERATION_EXECUTED); - this.localTrace = localTrace == null ? null : localTrace.toString(); - } - - /** Returns the document id that this result is for */ - public String getDocumentId() { - return document.getDocumentId(); - } - - /** Returns the id of the operation this is the result of */ - public String getOperationId() { return document.getOperationId(); } - - /** Returns the document data */ - public CharSequence getDocumentDataAsCharSequence() { - return document.getDataAsString(); - } - - /** Returns the context of the object if any */ - public Object getContext() { - return document.getContext(); - } - - /** - * Returns true if the operation(s) was successful. If at least one {@link Detail} - * in {@link #getDetails()} is unsuccessful, this will return false. - */ - public boolean isSuccess() { - return success; - } - public boolean isSuccessOrConditionNotMet() { - return isSuccess() || - details.stream().allMatch(d -> d.getResultType() == Result.ResultType.OPERATION_EXECUTED || - d.getResultType() == Result.ResultType.CONDITION_NOT_MET); - } - - public List<Detail> getDetails() { return details; } - - /** Returns whether the operation has been set up with local tracing */ - public boolean hasLocalTrace() { - return localTrace != null; - } - - /** Information in a Result for a single operation sent to a single endpoint. */ - public static final class Detail { - - private final ResultType resultType; - private final Endpoint endpoint; - private final Exception exception; - private final String traceMessage; - - public Detail(Endpoint endpoint, ResultType resultType, String traceMessage, Exception e) { - this.endpoint = endpoint; - this.resultType = resultType; - this.exception = e; - this.traceMessage = traceMessage; - } - - public Detail(Endpoint endpoint) { - this.endpoint = endpoint; - this.resultType = ResultType.OPERATION_EXECUTED; - this.exception = null; - this.traceMessage = null; - } - - /** - * Returns the endpoint from which the result was received, - * or null if this failed before being assigned an endpoint - */ - public Endpoint getEndpoint() { - return endpoint; - } - - /** Returns whether the operation was successful */ - public boolean isSuccess() { - return resultType == ResultType.OPERATION_EXECUTED; - } - - /** Returns the result of the operation */ - public ResultType getResultType() { - return resultType; - } - - /** Returns any exception related to this Detail, if unsuccessful. Might be null. */ - public Exception getException() { - return exception; - } - - /** Returns any trace message produces, or null if none */ - public String getTraceMessage() { - return traceMessage; - } - - @Override - public String toString() { - StringBuilder b = new StringBuilder(); - b.append("Detail "); - b.append("resultType=").append(resultType); - if (exception != null) - b.append(" exception='").append(Exceptions.toMessageString(exception)).append("'"); - if (traceMessage != null && ! traceMessage.isEmpty()) - b.append(" trace='").append(traceMessage).append("'"); - if (endpoint != null) - b.append(" endpoint=").append(endpoint); - return b.toString(); - } - - } - - @Override - public String toString() { - return "Result for " + document + " " + (localTrace != null ? localTrace : ""); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Session.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Session.java deleted file mode 100644 index 203f2d3b462..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Session.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import java.io.OutputStream; -import java.util.concurrent.BlockingQueue; - -/** - * A Session is an entity used to feed operations (like documents, removes or updates) to one Vespa - * cluster or several clusters in parallel. Current implementations are fail-fast, i.e. all feeding - * errors are propagated to the user as quickly as possible and with as much detail as possible. - * - * Implementations of this interface are required to be thread safe. - * - * A {@link SessionFactory} is provided to instantiate Sessions. - * - * @author Einar M R Rosenvinge - * @see SessionFactory - * @deprecated Vespa-http-client will be removed in Vespa 8. It's replaced by <a href="https://docs.vespa.ai/en/vespa-feed-client.html">vespa-feed-client</a> - */ -@Deprecated -public interface Session extends AutoCloseable { - - /** - * Returns an OutputStream that can be used to write ONE operation, identified by the - * given document ID. The data format must match the - * {@link com.yahoo.vespa.http.client.config.FeedParams.DataFormat} given when this - * Session was instantiated. Note that most data formats include the document ID in the - * actual buffer, which <em>must</em> match the document ID given as a parameter to this - * method. It is (as always) important to close the OutputStream returned - nothing - * is written to the wire until this is done. Note also that the Session holds a certain, - * dynamically determined maximum number of document operations in memory. - * When this threshold is reached, {@link java.io.OutputStream#close()} will block. - * - * - * @param documentId the unique ID identifying this operation in the system - * @return an OutputStream to write the operation payload into - */ - OutputStream stream(CharSequence documentId); - - /** - * Returns {@link Result}s for all operations enqueued by {@link #stream(CharSequence)}. - * Note that the order of results is non-deterministic, with <em>one</em> exception - results - * for one document ID are returned in the order they were enqueued. In all other cases - * Results may appear out-of-order. - * - * @return a blocking queue for retrieving results - * @see Result - */ - BlockingQueue<Result> results(); - - /** - * Closes this Session. All resources are freed, persistent connections are closed and - * internal threads are stopped. - * - * @throws RuntimeException in cases where underlying resources throw on shutdown/close - */ - void close(); - - /** - * Returns stats about the cluster. - * @return JSON string with information about cluster. - */ - String getStatsAsJson(); - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SessionFactory.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SessionFactory.java deleted file mode 100644 index ef231ccc713..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SessionFactory.java +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.SessionParams; - -import java.time.Clock; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.ThreadFactory; - -/** - * Factory for creating {@link Session} instances. - * - * @author Einar M R Rosenvinge - * @deprecated Vespa-http-client will be removed in Vespa 8. It's replaced by <a href="https://docs.vespa.ai/en/vespa-feed-client.html">vespa-feed-client</a> - */ -@Deprecated -public final class SessionFactory { - - /** - * Creates a {@link Session} with the given parameters. - * - * @param params the parameters to use when creating the Session. - * @return a new Session instance - */ - public static Session create(SessionParams params) { - return createInternal(params); - } - - @SuppressWarnings("deprecation") - static Session createInternal(SessionParams params) { - return new com.yahoo.vespa.http.client.core.api.SessionImpl(params, createTimeoutExecutor(), Clock.systemUTC()); - } - - static ScheduledThreadPoolExecutor createTimeoutExecutor() { - ScheduledThreadPoolExecutor timeoutExecutor; - timeoutExecutor = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("timeout-")); - timeoutExecutor.setRemoveOnCancelPolicy(true); - timeoutExecutor.setContinueExistingPeriodicTasksAfterShutdownPolicy(false); - timeoutExecutor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); - return timeoutExecutor; - } - - /** - * Creates a {@link Session} to a single {@link Endpoint}, with default values for everything. - * For full control of all parameters, or to feed to more than one Endpoint or more than one {@link Cluster}, - * see {@link #create(com.yahoo.vespa.http.client.config.SessionParams)}. - * - * @param endpoint the Endpoint to feed to. - * @return a new Session instance - * @see #create(com.yahoo.vespa.http.client.config.SessionParams) - */ - public static Session create(Endpoint endpoint) { - return createInternal(endpoint); - } - - static Session createInternal(Endpoint endpoint) { - SessionParams params = new SessionParams.Builder().addCluster( - new Cluster.Builder().addEndpoint(endpoint).build()).build(); - return create(params); - } - - private static class DaemonThreadFactory implements ThreadFactory { - private final ThreadFactory defaultThreadFactory = Executors.defaultThreadFactory(); - private final String prefix; - - public DaemonThreadFactory(String prefix) { - this.prefix = prefix; - } - - @Override - public Thread newThread(Runnable runnable) { - Thread t = defaultThreadFactory.newThread(runnable); - t.setDaemon(true); - t.setName(prefix + t.getName()); - return t; - } - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallback.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallback.java deleted file mode 100644 index e03bfd2b816..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallback.java +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import java.time.Duration; -import java.time.Instant; -import java.util.Date; -import java.util.Locale; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Simple implementation of the ResultCallback that logs to std err for every X documents: - * "Result received: 34 (1 failed so far, 2003 sent, success rate 1999.23 docs/sec)." - * On each failure it will print the Result object content. If tracing is enabled, it will print trace messages to - * std err as well. - * - * @author dybis - * @deprecated Vespa-http-client will be removed in Vespa 8. It's replaced by <a href="https://docs.vespa.ai/en/vespa-feed-client.html">vespa-feed-client</a> - */ -@Deprecated -public class SimpleLoggerResultCallback implements FeedClient.ResultCallback { - - private final Object monitor = new Object(); - private int resultCounter = 0; - private int failureCounter = 0; - private final AtomicInteger sentDocumentCounter; - private final int printStatsForEveryXDocument; - private final boolean ignoreConditionNotMet; - private Instant startSampleInstant = Instant.now(); - private int startSampleResultCount = 0; - - protected void println(String output) { - System.err.println(output); - } - - /** - * Constructor - * - * @param sentDocumentCounter a counter that is increased outside this class, but can be nice to print here. - * @param printStatsForEveryXDocument how often to print stats. - */ - public SimpleLoggerResultCallback(AtomicInteger sentDocumentCounter, int printStatsForEveryXDocument, boolean ignoreConditionNotMet) { - this.sentDocumentCounter = sentDocumentCounter; - this.printStatsForEveryXDocument = printStatsForEveryXDocument; - this.ignoreConditionNotMet = ignoreConditionNotMet; - } - - /** - * Prints how many documents that are received, failed and sent. - */ - public void printProgress() { - synchronized (monitor) { - DocumentRate docRate = newSamplingPeriod(Instant.now()); - println(new Date() + " Result received: " + resultCounter - + " (" + failureCounter + " failed so far, " + sentDocumentCounter.get() - + " sent, success rate " + docRate + ")."); - } - } - - static class DocumentRate { - public final double rate; - DocumentRate(double rate) { - this.rate = rate; - } - @Override - public String toString() { - return String.format(Locale.US, "%.2f docs/sec", rate); - } - } - - /* - * Returns success results per second for last interval and resets variables. - */ - protected DocumentRate newSamplingPeriod(Instant now) { - double docsDelta = resultCounter - failureCounter - startSampleResultCount; - Duration duration = Duration.between(startSampleInstant, now); - startSampleInstant = now; - startSampleResultCount = resultCounter - failureCounter; - long durationMilliSecs = duration.toMillis() + 1; // +1 to avoid division by zero - return new DocumentRate(1000. * docsDelta / durationMilliSecs); - } - - int getResultCount() { - synchronized (monitor) { - return resultCounter; - } - } - - int getFailedDocumentCount() { - synchronized (monitor) { - return failureCounter; - } - } - - @Override - public void onCompletion(String docId, Result documentResult) { - synchronized (monitor) { - if (printStatsForEveryXDocument > 0 && (resultCounter % printStatsForEveryXDocument) == 0) { - printProgress(); - } - resultCounter++; - boolean success = ignoreConditionNotMet - ? documentResult.isSuccessOrConditionNotMet() - : documentResult.isSuccess(); - if ( ! success ) { - failureCounter++; - println("Failure: " + documentResult + (documentResult.getDetails().isEmpty() ? "" : ":")); - for (Result.Detail detail : documentResult.getDetails()) - println(" " + detail); - } - } - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SyncFeedClient.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SyncFeedClient.java deleted file mode 100644 index 2fc63bbbcc5..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SyncFeedClient.java +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import com.yahoo.vespa.http.client.config.SessionParams; - -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.ConcurrentModificationException; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.ThreadLocalRandom; - -/** - * A utility wrapper of a FeedClient which feeds a list of documents and blocks until all responses are returned, - * before returning the results. - * - * Not multithread safe: A sync feed client instance can only be used by a single thread - * (but it can and should be reused for multiple subsequent synchronous calls). - * - * @author bratseth - * @deprecated Vespa-http-client will be removed in Vespa 8. It's replaced by <a href="https://docs.vespa.ai/en/vespa-feed-client.html">vespa-feed-client</a> - */ -@Deprecated -public class SyncFeedClient implements AutoCloseable { - - private final FeedClient wrappedClient; - private final Callback callback; - - public SyncFeedClient(SessionParams params) { - callback = new SyncFeedClient.Callback(); - if (params.getFeedParams().getIdlePollFrequency() == null) { - params = params.toBuilder() - .setFeedParams(params.getFeedParams().toBuilder() - .setIdlePollFrequency(500.0) - .build()) - .build(); - } - this.wrappedClient = FeedClientFactory.create(params, callback); - } - - /** - * Calls FeedClient.stream for each entry in the list, blocks until all results are ready and returns them. - * This will block for at most the time it takes to feed these operations + clientTimeout given in the - * sessions params when creating this. - * - * @param operations the Vespa write operations to stream - * @return the result of feeding all these operations - */ - public SyncResult stream(List<SyncOperation> operations) { - callback.expectResultsOf(operations); - for (SyncOperation operation : operations) - wrappedClient.stream(operation.documentId, operation.operationId, operation.documentData, operation.context); - return callback.waitForResults(); - } - - @Override - public void close() { - wrappedClient.close(); - } - - /** Holds the arguments to a single stream operation */ - public static class SyncOperation { - - private final String documentId; - private final CharSequence documentData; - private final Object context; - - /** Operation id passed on to the Document created from this */ - private final String operationId; - - public SyncOperation(String documentId, CharSequence documentData) { - this(documentId, documentData, null); - } - - public SyncOperation(String documentId, CharSequence documentData, Object context) { - this(documentId, documentData, new BigInteger(64, ThreadLocalRandom.current()).toString(32), context); - } - - public SyncOperation(String documentId, CharSequence documentData, String operationId, Object context) { - this.documentId = Objects.requireNonNull(documentId, "documentId"); - this.documentData = Objects.requireNonNull(documentData, "documentData"); - this.context = context; - this.operationId = Objects.requireNonNull(operationId); - } - - } - - /** - * The result of a SyncFeedClient.stream call. This always holds exactly one Result per SyncOperation - * attempted, and the results are guaranteed to be returned in the same order as in the List of SyncOperations. - */ - public static class SyncResult { - - private final Exception exception; - private final List<Result> results; - - private SyncResult(List<Result> results, Exception exception) { - this.results = results; - this.exception = exception; - } - - /** - * Returns the results of this. This has the same size and order as the List of SyncOperations that - * created this. The list returned is modifiable and owned by the client. Multiple calls to this returns the - * same list instance. - */ - public List<Result> results() { return results; } - - /** - * Returns the last exception received when attempting the operations this is the result of, or null if none. - * Even if there is an exception, results() will return one Result per operation attempted. - */ - public Exception exception() { return exception; } - - /** Returns true if all Results in this are successful */ - public boolean isSuccess() { - return results.stream().allMatch(Result::isSuccess); - } - - } - - private static class Callback implements FeedClient.ResultCallback { - - private final Object monitor = new Object(); - - // The rest of the state of this is reset each time we call expectResultsOf - - private int resultsReceived; - private Exception exception = null; - - /** - * A map from operation ids to their results. This is initially populated with null values to keep track of - * which responses we are waiting for. - */ - private LinkedHashMap<String, Result> results = null; - - void expectResultsOf(List<SyncOperation> operations) { - synchronized (monitor) { - if (results != null) - throw new ConcurrentModificationException("A SyncFeedClient instance is used by multiple threads"); - - resultsReceived = 0; - exception = null; - results = new LinkedHashMap<>(operations.size()); - for (SyncOperation operation : operations) - results.put(operation.operationId, null); - } - } - - SyncResult waitForResults() { - try { - synchronized (monitor) { - while ( ! complete()) - monitor.wait(); - - SyncResult syncResult = new SyncResult(new ArrayList<>(results.values()), exception); - results = null; - return syncResult; - } - } - catch (InterruptedException e) { - throw new RuntimeException("Interrupted while waiting for feeding results", e); - } - } - - @Override - public void onCompletion(String docId, Result documentResult) { - synchronized (monitor) { - if ( ! results.containsKey(documentResult.getOperationId())) return; // Stale result - ignore - - Result previousValue = results.put(documentResult.getOperationId(), documentResult); - if (previousValue != null) - throw new IllegalStateException("Received duplicate result for " + docId); - - resultsReceived++; - if (complete()) - monitor.notifyAll(); - } - } - - @Override - public void onEndpointException(FeedEndpointException exception) { - this.exception = exception; // We will still receive one onCompletion per stream invocation done - } - - private boolean complete() { - return resultsReceived == results.size(); - } - - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Cluster.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Cluster.java deleted file mode 100644 index 4f72c380e59..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Cluster.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.config; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; - -/** - * A set of {@link Endpoint} instances. Construct using {@link Cluster.Builder}. - * - * @author Einar M R Rosenvinge - */ -public final class Cluster { - - /** Builder for {@link Cluster}. */ - public final static class Builder { - private final List<Endpoint> endpoints = new LinkedList<>(); - private String route = null; - - /** - * Adds an Endpoint (a HTTP gateway) to this Cluster. - * - * @param endpoint the Endpoint to add - * @return this, for chaining - */ - public Builder addEndpoint(Endpoint endpoint) { - endpoints.add(endpoint); - return this; - } - - /** - * Sets a route specific to this cluster, which overrides the route set in {@link com.yahoo.vespa.http.client.config.FeedParams#getRoute()}. - * - * @param route a route specific to this cluster - * @return this, for chaining - */ - public Builder setRoute(String route) { - this.route = route; - return this; - } - - public Cluster build() { - return new Cluster(endpoints, route); - } - - public String getRoute() { - return route; - } - } - private final List<Endpoint> endpoints; - private final String route; - - private Cluster(List<Endpoint> endpoints, String route) { - this.endpoints = Collections.unmodifiableList(new ArrayList<>(endpoints)); - this.route = route; - } - - public List<Endpoint> getEndpoints() { - return endpoints; - } - - public String getRoute() { - return route; - } - - @Override - public String toString() { - return "cluster with endpoints " + endpoints + " and route '" + route + "'"; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java deleted file mode 100644 index 00cc2512ae3..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java +++ /dev/null @@ -1,486 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.config; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.Multimap; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; - -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.SSLContext; -import java.nio.file.Path; -import java.time.Duration; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.TimeUnit; - -/** - * Connection level parameters. - * This class is immutable - * and has no public constructor - to instantiate one, use a {@link Builder}. - * - * @author Einar M R Rosenvinge - */ -public final class ConnectionParams { - - /** - * Builder for {@link ConnectionParams}. - */ - public static final class Builder { - - private SSLContext sslContext = null; - private HostnameVerifier hostnameVerifier = SSLConnectionSocketFactory.getDefaultHostnameVerifier(); - private final Multimap<String, String> headers = ArrayListMultimap.create(); - private final Map<String, HeaderProvider> headerProviders = new HashMap<>(); - private int numPersistentConnectionsPerEndpoint = 1; - private String proxyHost = null; - private int proxyPort = 8080; - private boolean useCompression = false; - private int maxRetries = 100; - private long minTimeBetweenRetriesMs = 700; - private boolean dryRun = false; - private boolean runThreads = true; - private int traceLevel = 0; - private int traceEveryXOperation = 0; - private boolean printTraceToStdErr = true; - private boolean useTlsConfigFromEnvironment = false; - private Duration connectionTimeToLive = Duration.ofSeconds(30); - private Path privateKey; - private Path certificate; - private Path caCertificates; - - /** - * Use TLS configuration through the standard Vespa environment variables. - * Setting this to 'true' will override any other TLS/HTTPS related configuration. - */ - public Builder setUseTlsConfigFromEnvironment(boolean useTlsConfigFromEnvironment) { - this.useTlsConfigFromEnvironment = useTlsConfigFromEnvironment; - return this; - } - - /** - * Sets the SSLContext for the connection to the gateway when SSL is enabled for Endpoint. - * Default null (no ssl). See also Endpoint configuration. - * - * @param sslContext sslContext for connection to gateway. - * @return pointer to builder. - */ - public Builder setSslContext(SSLContext sslContext) { - this.sslContext = sslContext; - return this; - } - - /** - * Sets the {@link HostnameVerifier} for the connection to the gateway when SSL is enabled for Endpoint. - * Defaults to instance returned by {@link SSLConnectionSocketFactory#getDefaultHostnameVerifier()}. - * - * @param hostnameVerifier hostname verifier for connection to gateway. - * @return pointer to builder. - */ - public Builder setHostnameVerifier(HostnameVerifier hostnameVerifier) { - this.hostnameVerifier = hostnameVerifier; - return this; - } - - /** - * Set path to private key and certificate files. Both the private key and certificate must be PEM-encoded. - */ - public Builder setCertificateAndPrivateKey(Path privateKey, Path certificate) { - this.privateKey = privateKey; - this.certificate = certificate; - return this; - } - - /** - * Set path a PEM file containing the CA certificates. - */ - public Builder setCaCertificates(Path caCertificates) { - this.caCertificates = caCertificates; - return this; - } - - /** - * Set custom headers to be used - * - * @param key header name - * @param value header value - * @return pointer to builder. - */ - public Builder addHeader(String key, String value) { - headers.put(key, value); - return this; - } - - /** - * Adds a header provider for dynamic headers; headers where the value may change during a feeding session - * (e.g. security tokens with limited life time). Only one {@link HeaderProvider} is allowed for a given header name. - * - * @param provider A provider for a dynamic header - * @return pointer to builder. - * @throws IllegalArgumentException if a provider is already registered for the given header name - */ - public Builder addDynamicHeader(String headerName, HeaderProvider provider) { - Objects.requireNonNull(headerName, "Header name cannot be null"); - Objects.requireNonNull(provider, "Header provider cannot be null"); - if (headerProviders.containsKey(headerName)) { - throw new IllegalArgumentException("Provider already registered for name '" + headerName + "'"); - } - headerProviders.put(headerName, provider); - return this; - } - - /** - * The number of connections between the http client and the gateways. A very low number can result - * in the network not fully utilized and the round-trip time can be a limiting factor. A low number - * can cause skew in distribution of load between gateways. A too high number will cause - * many threads to run, more context switching and potential more memory usage. We recommend using about - * 16 connections per gateway. - * - * @param numPersistentConnectionsPerEndpoint number of channels per endpoint - * @return pointer to builder. - */ - public Builder setNumPersistentConnectionsPerEndpoint(int numPersistentConnectionsPerEndpoint) { - this.numPersistentConnectionsPerEndpoint = numPersistentConnectionsPerEndpoint; - return this; - } - - /** - * Sets the HTTP proxy host name to use. - * - * @param proxyHost host name for proxy. - * @return pointer to builder. - */ - public Builder setProxyHost(String proxyHost) { - this.proxyHost = proxyHost; - return this; - } - - /** - * Sets the HTTP proxy host port to use. - * - * @param proxyPort host port for proxy. - * @return pointer to builder. - */ - public Builder setProxyPort(int proxyPort) { - this.proxyPort = proxyPort; - return this; - } - - /** - * Set whether compression of document operations during communication to server should be enabled. - * - * @param useCompression true if compression should be enabled. - * @return pointer to builder. - */ - public Builder setUseCompression(boolean useCompression) { - this.useCompression = useCompression; - return this; - } - - /** - * Set how many times to retry sending an operation to a gateway when encountering transient problems. - * - * @param maxRetries max number of retries - * @return pointer to builder. - */ - public Builder setMaxRetries(int maxRetries) { - this.maxRetries = maxRetries; - return this; - } - - /** - * Set to true to skip making network connections and instead - * let requests complete successfully with no effect. - */ - public Builder setDryRun(boolean dryRun) { - this.dryRun = dryRun; - return this; - } - - /** - * Set to false to skip starting io threads, such that any operation must be driven by a calling thread. - * Useful for testing. - */ - public Builder setRunThreads(boolean runThreads) { - this.runThreads = runThreads; - return this; - } - - /** - * Set the min time between retries when temporarily failing against a gateway. - * - * @param minTimeBetweenRetries the min time value - * @param unit the unit of the min time. - * @return pointer to builder. - */ - public Builder setMinTimeBetweenRetries(long minTimeBetweenRetries, TimeUnit unit) { - this.minTimeBetweenRetriesMs = unit.toMillis(minTimeBetweenRetries); - return this; - } - - public long getMinTimeBetweenRetriesMs() { - return minTimeBetweenRetriesMs; - } - - /** - * Sets the trace level for tracing messagebus. 0 means to tracing. - * - * @param traceLevel tracelevel, larger value means more tracing. - * @return pointer to builder. - */ - public Builder setTraceLevel(int traceLevel) { - this.traceLevel = traceLevel; - return this; - } - - /** - * How often to trace messages in client. Please note that this does not affect tracing with messagebus - * - * @param traceEveryXOperation if zero, no tracing, 1 = every message, and so on. - * @return pointer to builder. - */ - public Builder setTraceEveryXOperation(int traceEveryXOperation) { - this.traceEveryXOperation = traceEveryXOperation; - return this; - } - - /** - * If enabled will write internal trace to stderr. - * - * @param printTraceToStdErr if value is true it is enabled. - * @return pointer to builder. - */ - public Builder setPrintTraceToStdErr(boolean printTraceToStdErr) { - this.printTraceToStdErr = printTraceToStdErr; - return this; - } - - /** - * Set the maximum time to live for persistent connections - */ - public Builder setConnectionTimeToLive(Duration connectionTimeToLive) { - this.connectionTimeToLive = connectionTimeToLive; - return this; - } - - public ConnectionParams build() { - return new ConnectionParams( - sslContext, - privateKey, - certificate, - caCertificates, - hostnameVerifier, - headers, - headerProviders, - numPersistentConnectionsPerEndpoint, - proxyHost, - proxyPort, - useCompression, - maxRetries, - minTimeBetweenRetriesMs, - dryRun, - runThreads, - traceLevel, - traceEveryXOperation, - printTraceToStdErr, - useTlsConfigFromEnvironment, - connectionTimeToLive); - } - - public int getNumPersistentConnectionsPerEndpoint() { - return numPersistentConnectionsPerEndpoint; - } - - public String getProxyHost() { - return proxyHost; - } - - public boolean isDryRun() { - return dryRun; - } - - public boolean runThreads() { return runThreads; } - - public int getMaxRetries() { - return maxRetries; - } - public int getTraceLevel() { - return traceLevel; - } - public int getTraceEveryXOperation() { - return traceEveryXOperation; - } - - public boolean getPrintTraceToStdErr() { - return printTraceToStdErr; - } - - public int getProxyPort() { - return proxyPort; - } - - public SSLContext getSslContext() { - return sslContext; - } - - public HostnameVerifier getHostnameVerifier() { - return hostnameVerifier; - } - - public boolean useTlsConfigFromEnvironment() { - return useTlsConfigFromEnvironment; - } - - public Duration getConnectionTimeToLive() { - return connectionTimeToLive; - } - public Path getPrivateKey() { return privateKey; } - public Path getCertificate() { return certificate; } - public Path getCaCertificates() { return caCertificates; } - } - - private final SSLContext sslContext; - private final Path privateKey; - private final Path certificate; - private final Path caCertificates; - private final HostnameVerifier hostnameVerifier; - private final Multimap<String, String> headers = ArrayListMultimap.create(); - private final Map<String, HeaderProvider> headerProviders = new HashMap<>(); - private final int numPersistentConnectionsPerEndpoint; - private final String proxyHost; - private final int proxyPort; - private final boolean useCompression; - private final int maxRetries; - private final long minTimeBetweenRetriesMs; - private final boolean dryRun; - private final boolean runThreads; - private final int traceLevel; - private final int traceEveryXOperation; - private final boolean printTraceToStdErr; - private final boolean useTlsConfigFromEnvironment; - private final Duration connectionTimeToLive; - - private ConnectionParams( - SSLContext sslContext, - Path privateKey, Path certificate, Path caCertificates, - HostnameVerifier hostnameVerifier, - Multimap<String, String> headers, - Map<String, HeaderProvider> headerProviders, - int numPersistentConnectionsPerEndpoint, - String proxyHost, - int proxyPort, - boolean useCompression, - int maxRetries, - long minTimeBetweenRetriesMs, - boolean dryRun, - boolean runThreads, - int traceLevel, - int traceEveryXOperation, - boolean printTraceToStdErr, - boolean useTlsConfigFromEnvironment, - Duration connectionTimeToLive) { - this.sslContext = sslContext; - this.privateKey = privateKey; - this.certificate = certificate; - this.caCertificates = caCertificates; - this.hostnameVerifier = hostnameVerifier; - this.useTlsConfigFromEnvironment = useTlsConfigFromEnvironment; - this.connectionTimeToLive = connectionTimeToLive; - this.headers.putAll(headers); - this.headerProviders.putAll(headerProviders); - this.numPersistentConnectionsPerEndpoint = numPersistentConnectionsPerEndpoint; - this.proxyHost = proxyHost; - this.proxyPort = proxyPort; - this.useCompression = useCompression; - this.maxRetries = maxRetries; - this.minTimeBetweenRetriesMs = minTimeBetweenRetriesMs; - this.dryRun = dryRun; - this.runThreads = runThreads; - this.traceLevel = traceLevel; - this.traceEveryXOperation = traceEveryXOperation; - this.printTraceToStdErr = printTraceToStdErr; - } - - @JsonIgnore - public SSLContext getSslContext() { - return sslContext; - } - - @JsonIgnore - public HostnameVerifier getHostnameVerifier() { - return hostnameVerifier; - } - - public Collection<Map.Entry<String, String>> getHeaders() { - return Collections.unmodifiableCollection(headers.entries()); - } - @JsonIgnore - public Map<String, HeaderProvider> getDynamicHeaders() { - return Collections.unmodifiableMap(headerProviders); - } - - public int getNumPersistentConnectionsPerEndpoint() { - return numPersistentConnectionsPerEndpoint; - } - - public String getProxyHost() { - return proxyHost; - } - - public int getProxyPort() { - return proxyPort; - } - - public boolean getUseCompression() { - return useCompression; - } - - public int getMaxRetries() { - return maxRetries; - } - - public long getMinTimeBetweenRetriesMs() { - return minTimeBetweenRetriesMs; - } - - public boolean isDryRun() { - return dryRun; - } - - public boolean runThreads() { return runThreads; } - - public int getTraceLevel() { - return traceLevel; - } - - public int getTraceEveryXOperation() { - return traceEveryXOperation; - } - - public boolean getPrintTraceToStdErr() { - return printTraceToStdErr; - } - - public boolean useTlsConfigFromEnvironment() { - return useTlsConfigFromEnvironment; - } - - public Duration getConnectionTimeToLive() { - return connectionTimeToLive; - } - - /** - * A header provider that provides a header value. {@link #getHeaderValue()} is called each time a new HTTP request - * is constructed by {@link com.yahoo.vespa.http.client.FeedClient}. - * - * Important: The implementation of {@link #getHeaderValue()} must be thread-safe! - */ - public interface HeaderProvider { String getHeaderValue(); } - - public Path getPrivateKey() { return privateKey; } - public Path getCertificate() { return certificate; } - public Path getCaCertificates() { return caCertificates; } -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Endpoint.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Endpoint.java deleted file mode 100644 index ae0cf19a3d1..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Endpoint.java +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.config; - -import java.io.Serializable; -import java.net.URL; - -/** - * Represents an endpoint, in most cases a JDisc container - * in a Vespa cluster configured with <code>document-api</code>. - * - * @author Einar M R Rosenvinge - */ -public final class Endpoint implements Serializable { - - private static final int DEFAULT_PORT = 4080; - - private final String hostname; - private final int port; - private final boolean useSsl; - - private Endpoint(String hostname, int port, boolean useSsl) { - if (hostname.startsWith("https://")) { - throw new RuntimeException("Hostname should be name of machine, not prefixed with protocol (https://)"); - } - // A lot of people put http:// before the servername, let us allow that. - if (hostname.startsWith("http://")) { - this.hostname = hostname.replaceFirst("http://", ""); - } else { - this.hostname = hostname; - } - this.port = port; - this.useSsl = useSsl; - } - - public String getHostname() { - return hostname; - } - - public int getPort() { - return port; - } - - public boolean isUseSsl() { - return useSsl; - } - - @Override - public String toString() { - return hostname + ":" + port + " ssl=" + useSsl; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof Endpoint)) return false; - Endpoint endpoint = (Endpoint) o; - return port == endpoint.port && useSsl == endpoint.useSsl && hostname.equals(endpoint.hostname); - } - - @Override - public int hashCode() { - int result = hostname.hashCode(); - result = 31 * result + port; - result = 31 * result + (useSsl ? 1 : 0); - return result; - } - - /** Creates an Endpoint with the default port and without using SSL */ - public static Endpoint create(String hostname) { - return new Endpoint(hostname, DEFAULT_PORT, false); - } - - /** Creates an Endpoint with the given hostname, port and SSL setting. */ - public static Endpoint create(String hostname, int port, boolean useSsl) { - return new Endpoint(hostname, port, useSsl); - } - - public static Endpoint create(URL url) { - return new Endpoint(url.getHost(), url.getPort(), "https".equals(url.getProtocol())); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/FeedParams.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/FeedParams.java deleted file mode 100644 index 01f314a7e36..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/FeedParams.java +++ /dev/null @@ -1,332 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.config; - -import java.util.concurrent.TimeUnit; - -/** - * Feed level parameters. This class is immutable - * and has no public constructor - to instantiate one, use a {@link Builder}. - - * @author Einar M R Rosenvinge - */ -public final class FeedParams { - - public boolean getDenyIfBusyV3() { return denyIfBusyV3; } - - public long getMaxSleepTimeMs() { return maxSleepTimeMs; } - - public boolean getSilentUpgrade() { return silentUpgrade; } - - /** - * Enumeration of data formats that are acceptable by the - * {@link com.yahoo.vespa.http.client.FeedClient} methods. - */ - public enum DataFormat { - /** UTF-8-encoded XML. Preamble is not necessary. */ - XML_UTF8, - JSON_UTF8 - } - /** - * Mutable class used to instantiate a {@link FeedParams}. - */ - public static final class Builder { - - private DataFormat dataFormat = DataFormat.JSON_UTF8; - private long serverTimeout = TimeUnit.SECONDS.toMillis(180); - private long clientTimeout = TimeUnit.SECONDS.toMillis(20); - private String route = null; - private int maxChunkSizeBytes = 50 * 1024; - private int maxInFlightRequests = 5000; - private long localQueueTimeOut = 180000; - private String priority = null; - private boolean denyIfBusyV3 = true; - private long maxSleepTimeMs = 3000; - private boolean silentUpgrade = true; - private Double idlePollFrequency = null; - - /** - * Make server not throw 4xx/5xx for situations that are normal during upgrade as this can esily mask - * other problems. This feature need to be supported on server side to work, but it is still safe - * to enable it, even if server does not yet support it. As of Nov 22 2016 it is not yet implemented on - * the server side. - * @param silentUpgrade true for reducing "false" 4xx/5xx. - * @return this, for chaining - */ - public Builder setSilentUpgrade(boolean silentUpgrade) { - this.silentUpgrade = silentUpgrade; - return this; - } - - /** - * When throttling the load due to transient errors on gateway, what is the most time to wait between - * requests per thread. Only active for V3 protocol. - * @param ms max with time - * @return this, for chaining - */ - public Builder setMaxSleepTimeMs(long ms) { - this.maxSleepTimeMs = ms; - return this; - } - - /** - * If this is set to false, the gateway will block threads until messagebus can send the message. - * If true, the gateway will exit and fail the request early if there are many threads already - * blocked. - * @param value true to reduce number of blocked threads in gateway. - * @return this, for chaining - */ - public Builder setDenyIfBusyV3(boolean value) { - denyIfBusyV3 = value; - return this; - } - - /** - * Sets the data format to be used. - * - * @param dataFormat the data format to be used. - * @see DataFormat - * @return this, for chaining - */ - public Builder setDataFormat(DataFormat dataFormat) { - this.dataFormat = dataFormat; - return this; - } - - /** - * Sets a route to be used for all Clusters, unless overridden on a per-cluster basis - * in {@link com.yahoo.vespa.http.client.config.Cluster#getRoute()}. - * - * @param route a route to be used for all Clusters. - * @return this, for chaining - */ - public Builder setRoute(String route) { - this.route = route; - return this; - } - - /** - * Sets the server-side timeout of each operation - i.e. the timeout used by - * the server endpoint for operations going over the message bus protocol into - * Vespa. - * - * Note that the TOTAL timeout of any one operation in this API would be - * {@link #getServerTimeout(java.util.concurrent.TimeUnit)} + - * {@link #getClientTimeout(java.util.concurrent.TimeUnit)}. - * - * @param serverTimeout timeout value - * @param unit unit of timeout value - * @return this, for chaining - */ - public Builder setServerTimeout(long serverTimeout, TimeUnit unit) { - if (serverTimeout <= 0L) { - throw new IllegalArgumentException("Server timeout cannot be zero or negative."); - } - this.serverTimeout = unit.toMillis(serverTimeout); - return this; - } - - /** - * Sets the client-side timeout for each operation. If BOTH the server-side - * timeout AND this timeout has passed, the {@link com.yahoo.vespa.http.client.FeedClient} - * will synthesize a {@link com.yahoo.vespa.http.client.Result}. - * - * Note that the TOTAL timeout of any one operation in this API would be - * {@link #getServerTimeout(java.util.concurrent.TimeUnit)} + - * {@link #getClientTimeout(java.util.concurrent.TimeUnit)}, - * after which a result callback is guaranteed to be made. - * - * @param clientTimeout timeout value - * @param unit unit of timeout value - * @return this, for chaining - */ - public Builder setClientTimeout(long clientTimeout, TimeUnit unit) { - if (clientTimeout <= 0L) { - throw new IllegalArgumentException("Client timeout cannot be zero or negative."); - } - this.clientTimeout = unit.toMillis(clientTimeout); - return this; - } - - /** - * Sets the maximum number of bytes of document data to send per HTTP request. - * - * @param maxChunkSizeBytes max number of bytes per HTTP request. - * @return this, for chaining - */ - public Builder setMaxChunkSizeBytes(int maxChunkSizeBytes) { - this.maxChunkSizeBytes = maxChunkSizeBytes; - return this; - } - - /** - * Sets the maximum number of operations to be in-flight. - * - * @param maxInFlightRequests max number of operations. - * @return this, for chaining - */ - public Builder setMaxInFlightRequests(int maxInFlightRequests) { - this.maxInFlightRequests = maxInFlightRequests; - return this; - } - - /** - * Sets the number of milliseconds until we respond with a timeout for a document operation - * if we still have not received a response. - */ - public Builder setLocalQueueTimeOut(long timeOutMs) { - this.localQueueTimeOut = timeOutMs; - return this; - } - - /** - * Set what frequency to poll for async responses. Default is 10hz (every 0.1s), but 1000hz when using SyncFeedClient - */ - public Builder setIdlePollFrequency(Double idlePollFrequency) { - this.idlePollFrequency = idlePollFrequency; - return this; - } - - /** - * Sets the messagebus priority. The allowed values are HIGHEST, VERY_HIGH, HIGH_[1-3], - * NORMAL_[1-6], LOW_[1-3], VERY_LOW, and LOWEST.. - * @param priority messagebus priority of this message. - * @return this, for chaining - */ - public Builder setPriority(String priority) { - if (priority == null) { - return this; - } - switch (priority) { - case "HIGHEST": - case "VERY_HIGH": - case "HIGH_1": - case "HIGH_2": - case "HIGH_3": - case "NORMAL_1": - case "NORMAL_2": - case "NORMAL_3": - case "NORMAL_4": - case "NORMAL_5": - case "NORMAL_6": - case "LOW_1": - case "LOW_2": - case "LOW_3": - case "VERY_LOW": - case "LOWEST": - this.priority = priority; - return this; - default: - throw new IllegalArgumentException("Unknown value for priority: " + priority - + " Allowed values are HIGHEST, VERY_HIGH, HIGH_[1-3], " + - "NORMAL_[1-6], LOW_[1-3], VERY_LOW, and LOWEST."); - } - } - - /** - * Instantiates a {@link FeedParams}. - * - * @return a FeedParams object with the parameters of this Builder - */ - public FeedParams build() { - return new FeedParams( - dataFormat, serverTimeout, clientTimeout, route, - maxChunkSizeBytes, maxInFlightRequests, localQueueTimeOut, priority, - denyIfBusyV3, maxSleepTimeMs, silentUpgrade, idlePollFrequency); - } - - public long getClientTimeout(TimeUnit unit) { - return unit.convert(clientTimeout, TimeUnit.MILLISECONDS); - } - - public long getServerTimeout(TimeUnit unit) { - return unit.convert(serverTimeout, TimeUnit.MILLISECONDS); - } - - public String getRoute() { - return route; - } - - public DataFormat getDataFormat() { - return dataFormat; - } - - public int getMaxChunkSizeBytes() { - return maxChunkSizeBytes; - } - - public int getMaxInFlightRequests() { - return maxInFlightRequests; - } - - } - - // NOTE! See toBuilder at the end of this class if you add fields here - - private final DataFormat dataFormat; - private final long serverTimeoutMillis; - private final long clientTimeoutMillis; - private final String route; - private final int maxChunkSizeBytes; - private final int maxInFlightRequests; - private final long localQueueTimeOut; - private final String priority; - private final boolean denyIfBusyV3; - private final long maxSleepTimeMs; - private final boolean silentUpgrade; - private final Double idlePollFrequency; - - private FeedParams(DataFormat dataFormat, long serverTimeout, long clientTimeout, String route, - int maxChunkSizeBytes, final int maxInFlightRequests, - long localQueueTimeOut, String priority, boolean denyIfBusyV3, long maxSleepTimeMs, - boolean silentUpgrade, Double idlePollFrequency) { - this.dataFormat = dataFormat; - this.serverTimeoutMillis = serverTimeout; - this.clientTimeoutMillis = clientTimeout; - this.route = route; - this.maxChunkSizeBytes = maxChunkSizeBytes; - this.maxInFlightRequests = maxInFlightRequests; - this.localQueueTimeOut = localQueueTimeOut; - this.priority = priority; - this.denyIfBusyV3 = denyIfBusyV3; - this.maxSleepTimeMs = maxSleepTimeMs; - this.silentUpgrade = silentUpgrade; - this.idlePollFrequency = idlePollFrequency; - - } - - public DataFormat getDataFormat() { return dataFormat; } - public String getRoute() { return route; } - public long getServerTimeout(TimeUnit unit) { return unit.convert(serverTimeoutMillis, TimeUnit.MILLISECONDS); } - public long getClientTimeout(TimeUnit unit) { return unit.convert(clientTimeoutMillis, TimeUnit.MILLISECONDS); } - - public int getMaxChunkSizeBytes() { return maxChunkSizeBytes; } - public String getPriority() { return priority; } - - public String toUriParameters() { - StringBuilder b = new StringBuilder(); - b.append("&dataformat=").append(dataFormat.name()); //name in dataFormat enum obviously must be ascii - return b.toString(); - } - - public int getMaxInFlightRequests() { return maxInFlightRequests; } - public long getLocalQueueTimeOut() { return localQueueTimeOut; } - public Double getIdlePollFrequency() { return idlePollFrequency; } - - /** Returns a builder initialized to the values of this */ - public FeedParams.Builder toBuilder() { - Builder b = new Builder(); - b.setDataFormat(dataFormat); - b.setServerTimeout(serverTimeoutMillis, TimeUnit.MILLISECONDS); - b.setClientTimeout(clientTimeoutMillis, TimeUnit.MILLISECONDS); - b.setRoute(route); - b.setMaxChunkSizeBytes(maxChunkSizeBytes); - b.setMaxInFlightRequests(maxInFlightRequests); - b.setPriority(priority); - b.setDenyIfBusyV3(denyIfBusyV3); - b.setMaxSleepTimeMs(maxSleepTimeMs); - b.setSilentUpgrade(silentUpgrade); - b.setIdlePollFrequency(idlePollFrequency); - return b; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/SessionParams.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/SessionParams.java deleted file mode 100644 index e8052fa7faa..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/SessionParams.java +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.config; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; - -/** - * Parameters given to a {@link com.yahoo.vespa.http.client.FeedClientFactory} - * when creating {@link com.yahoo.vespa.http.client.FeedClient}s. This class is immutable - * and has no public constructor - to instantiate one, use a {@link Builder}. - * - * @author Einar M R Rosenvinge - * @see com.yahoo.vespa.http.client.FeedClientFactory - * @see Builder - */ -public final class SessionParams { - - /** - * Interface for handling serious errors with connection. - */ - public interface ErrorReporter { - void onSessionError(Endpoint endpoint, String oldSessionID, String newSessionId); - } - - /** - * Mutable class used to instantiate a {@link SessionParams}. A builder - * instance will at the very least contain cluster settings ( - * {@link #addCluster(Cluster)}), for supporting SSL and similar transport - * settings, use {@link #setConnectionParams(ConnectionParams)}. - */ - public static final class Builder { - - private final List<Cluster> clusters = new LinkedList<>(); - private FeedParams feedParams = new FeedParams.Builder().build(); - private ConnectionParams connectionParams = new ConnectionParams.Builder().build(); - private int clientQueueSize = 10000; - private ErrorReporter errorReporter = null; - private int throttlerMinSize = 0; - - /** - * Add a Vespa installation for feeding documents into. - * - * @return this Builder instance, to support chaining - */ - public Builder addCluster(Cluster cluster) { - clusters.add(cluster); - return this; - } - - /** - * Set parameters used for feeding the documents in the receiving - * cluster. Reasonable defaults are supplied, so setting this should not - * be necessary for testing. - * - * @return this builder instance to support chaining - */ - public Builder setFeedParams(FeedParams feedParams) { - this.feedParams = feedParams; - return this; - } - - /** - * Transport parameters, like custom HTTP headers. - * - * @return this Builder instance, to support chaining - */ - public Builder setConnectionParams(ConnectionParams connectionParams) { - this.connectionParams = connectionParams; - return this; - } - - /** - * Sets an error reporter that is invoked in case of serious errors. - * - * @param errorReporter the handler - * @return pointer to builder. - */ - public Builder setErrorReporter(ErrorReporter errorReporter) { - this.errorReporter = errorReporter; - return this; - } - - /** - * Sets the maximum number of document operations to hold in memory, waiting to be - * sent to Vespa. When this threshold is reached, {@link java.io.OutputStream#close()} will block. - * - * @param clientQueueSize the maximum number of document operations to hold in memory. - * @return pointer to builder. - */ - public Builder setClientQueueSize(int clientQueueSize) { - this.clientQueueSize = clientQueueSize; - return this; - } - - /** - * Sets the minimum queue size of the throttler. If this is zero, it means that dynamic throttling is - * not enabled. Otherwise it is the minimum size of the throttler for how many parallel requests that are - * accepted. The max size of the throttler is the clientQueueSize. - * @return the minimum number of requests to be used in throttler or zero if throttler is static. - * - * @param throttlerMinSize the value of the min size. - */ - public Builder setThrottlerMinSize(int throttlerMinSize) { - this.throttlerMinSize = throttlerMinSize; - return this; - } - - /** - * Instantiates a {@link SessionParams} that can be given to a {@link com.yahoo.vespa.http.client.FeedClientFactory}. - * - * @return a SessionParams object with the parameters of this Builder - */ - public SessionParams build() { - return new SessionParams( - clusters, feedParams, connectionParams, clientQueueSize, errorReporter, throttlerMinSize); - } - - public FeedParams getFeedParams() { - return feedParams; - } - public ConnectionParams getConnectionParams() { - return connectionParams; - } - public int getClientQueueSize() { - return clientQueueSize; - } - public int getThrottlerMinSize() { - return throttlerMinSize; - } - } - - // NOTE! See toBuilder at the end of this class if you add fields here - - private final List<Cluster> clusters; - private final FeedParams feedParams; - private final ConnectionParams connectionParams; - private final int clientQueueSize; - private final ErrorReporter errorReport; - private final int throttlerMinSize; - - private SessionParams(Collection<Cluster> clusters, - FeedParams feedParams, - ConnectionParams connectionParams, - int clientQueueSize, - ErrorReporter errorReporter, - int throttlerMinSize) { - this.clusters = Collections.unmodifiableList(new ArrayList<>(clusters)); - this.feedParams = feedParams; - this.connectionParams = connectionParams; - this.clientQueueSize = clientQueueSize; - this.errorReport = errorReporter; - this.throttlerMinSize = throttlerMinSize; - } - - public List<Cluster> getClusters() { - return clusters; - } - - public FeedParams getFeedParams() { - return feedParams; - } - - public ConnectionParams getConnectionParams() { - return connectionParams; - } - - public int getClientQueueSize() { - return clientQueueSize; - } - - public int getThrottlerMinSize() { - return throttlerMinSize; - } - - public ErrorReporter getErrorReport() { - return errorReport; - } - - public Builder toBuilder() { - Builder b = new Builder(); - clusters.forEach(c -> b.addCluster(c)); - b.setFeedParams(feedParams); - b.setConnectionParams(connectionParams); - b.setClientQueueSize(clientQueueSize); - b.setErrorReporter(errorReport); - b.setThrottlerMinSize(throttlerMinSize); - return b; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/package-info.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/package-info.java deleted file mode 100644 index 9c07e819e90..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Settings for creating clients/sessions. - * - * NOTE: This is a PUBLIC API, but not annotated as such because this is not a bundle and - * we don't want to introduce Vespa dependencies. - */ -package com.yahoo.vespa.http.client.config; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Document.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Document.java deleted file mode 100644 index 07262a60dd8..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Document.java +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import java.io.IOException; -import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.CharacterCodingException; -import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.util.Objects; -import java.util.concurrent.ThreadLocalRandom; - -/** - * A document operation - * - * @author Einar M R Rosenvinge - */ -final public class Document { - - private final String documentId; - private final ByteBuffer data; - private final Instant createTime; - // This is initialized lazily to reduce work on calling thread (which is the thread calling the API) - private String operationId = null; - private final Object context; - private Instant queueInsertTime; - - public Document(String documentId, byte[] data, Object context, Instant createTime) { - this(documentId, null, ByteBuffer.wrap(data), context, createTime); - } - - public Document(String documentId, String operationId, CharSequence data, Object context, Instant createTime) { - this(documentId, operationId, encode(data, documentId), context, createTime); - } - - private Document(String documentId, String operationId, ByteBuffer data, Object context, Instant createTime) { - this.documentId = documentId; - this.operationId = operationId; - this.data = data; - this.context = context; - this.createTime = Objects.requireNonNull(createTime, "createTime cannot be null"); - this.queueInsertTime = createTime; - } - - public void setQueueInsertTime(Instant queueInsertTime) { - this.queueInsertTime = queueInsertTime; - } - - public Instant getQueueInsertTime() { return queueInsertTime; } - - public CharSequence getDataAsString() { - return StandardCharsets.UTF_8.decode(data.asReadOnlyBuffer()); - } - - public Object getContext() { return context; } - - public static class DocumentException extends IOException { - private static final long serialVersionUID = 29832833292L; - public DocumentException(String message) - { - super(message); - } - } - - public String getDocumentId() { return documentId; } - - public ByteBuffer getData() { - return data.asReadOnlyBuffer(); - } - - public int size() { - return data.remaining(); - } - - public Instant createTime() { return createTime; } - - public String getOperationId() { - if (operationId == null) { - operationId = new BigInteger(64, ThreadLocalRandom.current()).toString(32); - } - return operationId; - } - - @Override - public String toString() { return "document '" + documentId + "'"; } - - private static ByteBuffer encode(CharSequence data, String documentId) { - try { - return StandardCharsets.UTF_8.newEncoder().encode(CharBuffer.wrap(data)); - } catch (CharacterCodingException e) { - throw new RuntimeException("Error encoding document data into UTF8 " + documentId, e); - } - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/EndpointResult.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/EndpointResult.java deleted file mode 100644 index 94d7237422a..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/EndpointResult.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import com.yahoo.vespa.http.client.Result; - -/** - * Result from a single endpoint. - * - * @author dybis - */ -public class EndpointResult { - - private final String operationId; - private final Result.Detail detail; - - public EndpointResult(String operationId, Result.Detail detail) { - this.operationId = operationId; - this.detail = detail; - } - - public String getOperationId() { - return operationId; - } - - public Result.Detail getDetail() { - return detail; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Exceptions.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Exceptions.java deleted file mode 100644 index 9ff3f793756..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Exceptions.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -/** - * Helper methods for handling exceptions - * - * @author bratseth - */ -public abstract class Exceptions { - - /** - * <p>Returns a use friendly error message string which includes information from all nested exceptions.</p> - * - * <p>The form of this string is - * <code>e.getMessage(): e.getCause().getMessage(): e.getCause().getCause().getMessage()...</code> - * In addition, some heuristics are used to clean up common cases where exception nesting causes bad messages. - * </p> - */ - public static String toMessageString(Throwable t) { - StringBuilder b = new StringBuilder(); - String lastMessage = null; - String message; - for (; t != null; t = t.getCause(), lastMessage = message) { - message = getMessage(t); - if (message == null) continue; - if (message.equals(lastMessage)) continue; - if (b.length() > 0) { - b.append(": "); - } - b.append(message); - } - return b.toString(); - } - - /** Returns a useful message from *this* exception, or null if none */ - private static String getMessage(Throwable t) { - String message = t.getMessage(); - if (t.getCause() == null) { - if (message == null) return t.getClass().getSimpleName(); - } else { - if (message == null) return null; - if (message.equals(t.getCause().getClass().getName() + ": " + t.getCause().getMessage())) return null; - } - return message; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/JsonReader.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/JsonReader.java deleted file mode 100644 index 34b6d8b9144..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/JsonReader.java +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonFactoryBuilder; -import com.fasterxml.jackson.core.JsonParser; -import com.yahoo.vespa.http.client.FeedClient; - -import java.io.EOFException; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Reads a stream of json documents and sends them to feedClient. - * - * @author dybis - */ -public class JsonReader { - - /** - * Max size of documents. As we stream docs in for finding doc id, we buffer the data and later stream them to - * feedclient after doc id has been revealed. - */ - private final static int maxDocumentSizeChars = 50 * 1024 * 1024; - - // Intended to be used as static. - private JsonReader() {} - - /** - * Process one inputstream and send all documents to feedclient. - * - * @param inputStream source of array of json document. - * @param feedClient where data is sent. - * @param numSent counter to be incremented for every document streamed. - */ - public static void read(InputStream inputStream, FeedClient feedClient, AtomicInteger numSent) { - try (InputStreamJsonElementBuffer jsonElementBuffer = new InputStreamJsonElementBuffer(inputStream)) { - JsonFactory jfactory = new JsonFactoryBuilder().disable(JsonFactory.Feature.CANONICALIZE_FIELD_NAMES).build(); - JsonParser jParser = jfactory.createParser(jsonElementBuffer); - while (true) { - int documentStart = (int) jParser.getCurrentLocation().getCharOffset(); - String docId = parseOneDocument(jParser); - if (docId == null) { - int documentEnd = (int) jParser.getCurrentLocation().getCharOffset(); - int documentLength = documentEnd - documentStart; - int maxTruncatedLength = 500; - StringBuilder stringBuilder = new StringBuilder(maxTruncatedLength + 3); - for (int i = 0; i < Math.min(documentLength, maxTruncatedLength); i++) - stringBuilder.append(jsonElementBuffer.circular.get(documentStart + i)); - - if (documentLength > maxTruncatedLength) - stringBuilder.append("..."); - - throw new IllegalArgumentException("Document is missing ID: '" + stringBuilder.toString() + "'"); - } - CharSequence data = jsonElementBuffer.getJsonAsArray(jParser.getCurrentLocation().getCharOffset()); - feedClient.stream(docId, data); - numSent.incrementAndGet(); - } - } catch (EOFException ignored) { - // No more documents - } catch (IOException ioe) { - System.err.println(ioe.getMessage()); - throw new UncheckedIOException(ioe); - } - } - - /** - * This class is intended to be used with a json parser. The data is sent through this intermediate stream - * and to the parser. When the parser is done with a document, it calls postJsonAsArray which will - * stream the document up to the current position of the parser. - */ - private static class InputStreamJsonElementBuffer extends InputStreamReader { - - /** - * Simple class implementing a circular array with some custom function used for finding start and end - * of json object. The reason this is needed is that the json parser reads more than it parses - * from the input stream (seems like about 8k). Using a ByteBuffer and manually moving data - * is an order of magnitude slower than this implementation. - */ - private class CircularCharBuffer { - - int readPointer = 0; - int writePointer = 0; - final char[] data; - final int size; - - public CircularCharBuffer(int chars) { - data = new char[chars]; - size = chars; - } - - /** - * This is for throwing away [ and spaces in front of a json object, and find the position of {. - * Not for parsing much text. - * - * @return position for { - */ - public int findNextObjectStart() { - int readerPos = 0; - while (get(readerPos) != '{') { - readerPos++; - assert(readerPos<=size); - } - return readerPos; - } - - /** - * This is for throwing away comma and or ], and for finding the position of the last }. - * @param fromPos where to start searching - * @return position for } - */ - public int findLastObjectEnd(int fromPos) { - while (get(fromPos-1) != '}') { - fromPos--; - assert(fromPos >=0); - } - return fromPos; - } - - public void put(char dataByte) { - data[writePointer] = dataByte; - writePointer++; - if (writePointer >= size) writePointer = 0; - assert(writePointer != readPointer); - } - - public char get(int pos) { - int readPos = readPointer + pos; - if (readPos >= size) readPos -= size; - assert(readPos != writePointer); - return data[readPos]; - } - - public void advance(int end) { - readPointer += end; - if (readPointer >= size) readPointer -= size; - } - } - - private final CircularCharBuffer circular = new CircularCharBuffer(maxDocumentSizeChars); - private int processedChars = 0; - - public InputStreamJsonElementBuffer(InputStream inputStream) { - super(inputStream, StandardCharsets.UTF_8); - } - - /** - * Removes comma, start/end array tag (last element), spaces etc that might be surrounding a json element. - * Then sends the element to the outputstream. - * @param parserPosition how far the parser has come. Please note that the parser might have processed - * more data from the input source as it is reading chunks of data. - * @throws IOException on errors - */ - public CharSequence getJsonAsArray(long parserPosition) throws IOException { - final int charSize = (int)parserPosition - processedChars; - final int endPosOfJson = circular.findLastObjectEnd(charSize); - final int startPosOfJson = circular.findNextObjectStart(); - processedChars += charSize; - // This can be optimized since we rarely wrap the circular buffer. - StringBuilder dataBuffer = new StringBuilder(endPosOfJson - startPosOfJson); - for (int x = startPosOfJson; x < endPosOfJson; x++) { - dataBuffer.append(circular.get(x)); - } - circular.advance(charSize); - return dataBuffer.toString(); - } - - @Override - public int read(char[] b, int off, int len) throws IOException { - int length = 0; - int value = 0; - while (length < len && value != -1) { - value = read(); - if (value == -1) { - return length == 0 ? -1 : length; - } - b[off + length] = (char) value; - length++; - } - return length; - } - - @Override - public int read() throws IOException { - int value = super.read(); - if (value >= 0) circular.put((char)value); - return value; - } - } - - /** - * Parse one document from the stream and return doc id. - * - * @param jParser parser with stream. - * @return doc id of document or null if no more docs. - * @throws IOException on problems - */ - private static String parseOneDocument(JsonParser jParser) throws IOException { - int objectLevel = 0; - String documentId = null; - boolean foundObject = false; - boolean valueIsDocumentId = false; - while (jParser.nextToken() != null) { - String tokenAsText = jParser.getText(); - if (valueIsDocumentId) { - if (documentId != null) { - throw new RuntimeException("Several document ids"); - } - documentId = tokenAsText; - valueIsDocumentId = false; - } - switch(jParser.getCurrentToken()) { - case START_OBJECT: - foundObject = true; - objectLevel++; - break; - case END_OBJECT: - objectLevel--; - if (objectLevel == 0) { - return documentId; - } - break; - case FIELD_NAME: - if (objectLevel == 1 && - (tokenAsText.equals("put") - || tokenAsText.endsWith("id") - || tokenAsText.endsWith("update") - || tokenAsText.equals("remove"))) { - valueIsDocumentId = true; - } - break; - default: // No operation on all other tags. - } - } - if (!foundObject) - throw new EOFException("No more documents"); - return null; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ServerResponseException.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ServerResponseException.java deleted file mode 100644 index d5a09d2566c..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ServerResponseException.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -/** - * The request was not processed properly on the server. - * - * @author Einar M R Rosenvinge - */ -@SuppressWarnings("serial") -public class ServerResponseException extends Exception { - - private final int responseCode; - private final String responseString; - - public ServerResponseException(int responseCode, String responseString) { - super(responseString); - this.responseCode = responseCode; - this.responseString = responseString; - } - - public ServerResponseException(String responseString) { - super(responseString); - this.responseCode = 0; - this.responseString = responseString; - } - - public int getResponseCode() { - return responseCode; - } - - public String getResponseString() { - return responseString; - } - - @Override - public String toString() { - if (responseCode > 0) { - return responseCode + ": " + responseString; - } - return responseString; - } - -} - diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ThrottlePolicy.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ThrottlePolicy.java deleted file mode 100644 index 101bd001fb8..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ThrottlePolicy.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import static java.lang.Math.abs; -import static java.lang.Math.max; -import static java.lang.Math.min; - -/** - * Class that has a method for finding next maxInFlight. - * - * @author dybis - */ -public class ThrottlePolicy { - - public static final double SMALL_DIFFERENCE_IN_SUCCESSES_RATIO = 0.15; - private static final double MINIMUM_DIFFERENCE = 0.05; - - /** - * Generate nex in-flight value for throttling. - * - * @param maxPerformanceChange This value limit the dynamics of the algorithm. - * @param numOk number of success in last phase - * @param previousNumOk number of success in previous (before last) phase. - * @param previousMaxInFlight number of max-in-flight in previous (before last) phase. - * @param maxInFlightNow number of max-in-flight in last phase. - * @param messagesQueued if any messages where queued. - * @return The new value to be used for max-in-flight (should be cropped externally to fit max/min values). - */ - public int calcNewMaxInFlight(double maxPerformanceChange, int numOk, int previousNumOk, int previousMaxInFlight, - int maxInFlightNow, boolean messagesQueued) { - - double difference = calculateRuleBasedDifference(maxPerformanceChange, numOk, previousNumOk, previousMaxInFlight, maxInFlightNow); - boolean previousRunWasBetter = numOk < previousNumOk; - boolean previousRunHadLessInFlight = previousMaxInFlight < maxInFlightNow; - - - int delta; - if (previousRunWasBetter == previousRunHadLessInFlight) { - delta = (int) (-1.1 * difference * maxInFlightNow); - } else { - delta = (int) (difference * maxInFlightNow); - } - - // We don't want the same size since we need different sizes for algorithm to adjust. - if (abs(delta) < 2) { - delta = -3; - } - // We never used all permits in previous run, no reason to grow more, we should rather reduce permits. - if (!messagesQueued && delta > 0) { - delta = -2; - } - return maxInFlightNow + delta; - } - - private static double calculateRuleBasedDifference(double maxPerformanceChange, double numOk, double previousNumOk, - double previousMaxInFlight, double maxInFlightNow) { - double difference = min( - maxPerformanceChange, - abs((numOk - previousNumOk) / safeDenominator(previousNumOk))); - - if (abs(previousMaxInFlight - maxInFlightNow) / safeDenominator(min(previousMaxInFlight, maxInFlightNow)) - < SMALL_DIFFERENCE_IN_SUCCESSES_RATIO) { - difference = min(difference, 0.2); - } - - // We want some changes so we can track performance as a result of different throttling. - return max(difference, MINIMUM_DIFFERENCE); - } - - private static double safeDenominator(double x) { - return x == 0.0 ? 1.0 : x; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/XmlFeedReader.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/XmlFeedReader.java deleted file mode 100644 index 349959f496e..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/XmlFeedReader.java +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import com.yahoo.vespa.http.client.FeedClient; -import org.xml.sax.Attributes; -import org.xml.sax.InputSource; -import org.xml.sax.ext.DefaultHandler2; - -import javax.xml.parsers.SAXParser; -import javax.xml.parsers.SAXParserFactory; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Reads an input stream of xml, sends these to session. - * - * @author dybis -*/ -public class XmlFeedReader { - - // Static class. - private XmlFeedReader() {} - - public static void read(InputStream inputStream, FeedClient feedClient, AtomicInteger numSent) throws Exception { - SAXParserFactory parserFactory = SAXParserFactory.newInstance(); - // XXE prevention: - parserFactory.setFeature("http://xml.org/sax/features/external-general-entities", false); - parserFactory.setValidating(false); - parserFactory.setNamespaceAware(false); - SAXParser parser = parserFactory.newSAXParser(); - SAXClientFeeder saxClientFeeder = new SAXClientFeeder(feedClient, numSent); - - InputSource inputSource = new InputSource(); - inputSource.setEncoding(StandardCharsets.UTF_8.displayName()); - inputSource.setByteStream(inputStream); - // This is to send events about CDATA to the saxClientFeeder - // (https://docs.oracle.com/javase/tutorial/jaxp/sax/events.html) - parser.setProperty("http://xml.org/sax/properties/lexical-handler", saxClientFeeder); - parser.parse(inputSource, saxClientFeeder); - } -} - -/** - * Streams XML and sends each document operation to feeder. - */ -class SAXClientFeeder extends DefaultHandler2 { - - public static final String CDATA_START = "<![CDATA["; - public static final String CDATA_STOP = "]]>"; - private final FeedClient feedClient; - int vespaIndent = 0; - int documentIndent = 0; - String documentId = null; - StringBuilder content = new StringBuilder(); - final AtomicInteger numSent; - boolean isCData = false; - - public SAXClientFeeder(FeedClient feedClient, AtomicInteger numSent) { - this.feedClient = feedClient; - this.numSent = numSent; - } - - @Override - public void startCDATA() { - content.append(CDATA_START); - isCData = true; - } - - @Override - public void endCDATA() { - content.append(CDATA_STOP); - isCData = false; - } - - @Override - public void comment(char[] ch, int start, int length) { } - - @SuppressWarnings("fallthrough") - @Override - public void startElement(String uri, String localName, String qName, Attributes attributes) { - switch(qName){ - case "vespafeed": - vespaIndent++; - if (vespaIndent == 1 && documentIndent == 0) { - // If this is the first vespafeed tag, it should not be added to content of the first item. - return; - } - case "update": - case "remove": - case "document" : - documentIndent++; - documentId = attributes.getValue("documentid"); - content = new StringBuilder(); - } - content.append("<" + qName); - if (attributes != null) { - for (int i = 0; i < attributes.getLength (); i++) { - content.append(" ") - .append(attributes.getQName(i)) - .append("=\""); - String attributesValue = attributes.getValue(i); - characters(attributesValue.toCharArray(), 0, attributesValue.length()); - content.append("\""); - } - } - content.append(">"); - } - - @Override - public void endElement(String uri, String localName, String qName) { - content.append("</") - .append(qName) - .append(">"); - switch(qName){ - case "vespafeed": - vespaIndent--; - return; - case "update": - case "remove": - case "document" : - documentIndent--; - if (documentIndent == 0) { - if (documentId == null || documentId.isEmpty()) { - throw new IllegalArgumentException("no docid"); - } - feedClient.stream(documentId, content); - numSent.incrementAndGet(); - } - } - } - - @Override - public void characters (char buf [], int offset, int len) { - if (isCData) { - content.append(buf, offset, len); - return; - } - - // This is on the critical loop for performance, otherwise a library would have been used. - // We can do a few shortcuts as well as this data is already decoded by SAX parser. - for (int x = offset ; x < len + offset ; x++) { - switch (buf[x]) { - case '&' : content.append("&"); continue; - case '<' : content.append("<"); continue; - case '>' : content.append(">"); continue; - case '"' : content.append("""); continue; - case '\'' : content.append("'"); continue; - default: content.append(buf[x]); continue; - } - } - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/FeedClientImpl.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/FeedClientImpl.java deleted file mode 100644 index fa214808ae3..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/FeedClientImpl.java +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.api; - -import com.yahoo.vespa.http.client.FeedClient; -import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.ThrottlePolicy; -import com.yahoo.vespa.http.client.core.operationProcessor.IncompleteResultsThrottler; -import com.yahoo.vespa.http.client.core.operationProcessor.OperationProcessor; - -import java.nio.charset.CharsetEncoder; -import java.nio.charset.CodingErrorAction; -import java.nio.charset.StandardCharsets; -import java.time.Clock; -import java.time.Instant; -import java.util.Optional; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.logging.Logger; - -/** - * Implementation of FeedClient. It is a thin layer on top of multiClusterHandler and multiClusterResultAggregator. - * - * @author dybis - */ -public class FeedClientImpl implements FeedClient { - - private static final Logger log = Logger.getLogger(FeedClientImpl.class.getName()); - private static final AtomicBoolean warningPrinted = new AtomicBoolean(false); - - private final Clock clock; - private final OperationProcessor operationProcessor; - private final long closeTimeoutMs; - private final long sleepTimeMs = 500; - - public FeedClientImpl(SessionParams sessionParams, - ResultCallback resultCallback, - ScheduledThreadPoolExecutor timeoutExecutor, - Clock clock) { - this.clock = clock; - this.closeTimeoutMs = (10 + 3 * sessionParams.getConnectionParams().getMaxRetries()) * - (sessionParams.getFeedParams().getServerTimeout(TimeUnit.MILLISECONDS) + - sessionParams.getFeedParams().getClientTimeout(TimeUnit.MILLISECONDS)); - this.operationProcessor = new OperationProcessor( - new IncompleteResultsThrottler(sessionParams.getThrottlerMinSize(), - sessionParams.getClientQueueSize(), - clock, - new ThrottlePolicy()), - resultCallback, - sessionParams, - timeoutExecutor, - clock); - if (warningPrinted.compareAndSet(false, true)) { - log.warning("The vespa-http-client is deprecated and will be removed in Vespa 8. " + - "See https://docs.vespa.ai/en/vespa8-release-notes.html"); - } - } - - @Override - public void stream(String documentId, String operationId, CharSequence documentData, Object context) { - CharsetEncoder charsetEncoder = StandardCharsets.UTF_8.newEncoder(); - charsetEncoder.onMalformedInput(CodingErrorAction.REPORT); - charsetEncoder.onUnmappableCharacter(CodingErrorAction.REPORT); - - Document document = new Document(documentId, operationId, documentData, context, clock.instant()); - operationProcessor.sendDocument(document); - } - - @Override - public void close() { - Instant lastOldestResultReceivedAt = Instant.now(); - Optional<String> oldestIncompleteId = operationProcessor.oldestIncompleteResultId(); - - while (oldestIncompleteId.isPresent() && waitForOperations(lastOldestResultReceivedAt, sleepTimeMs, closeTimeoutMs)) { - Optional<String> oldestIncompleteIdNow = operationProcessor.oldestIncompleteResultId(); - if ( ! oldestIncompleteId.equals(oldestIncompleteIdNow)) - lastOldestResultReceivedAt = Instant.now(); - oldestIncompleteId = oldestIncompleteIdNow; - } - operationProcessor.close(); - } - - @Override - public String getStatsAsJson() { - return operationProcessor.getStatsAsJson(); - } - - // On return value true, wait more. Public for testing. - public static boolean waitForOperations(Instant lastResultReceived, long sleepTimeMs, long closeTimeoutMs) { - if (lastResultReceived.plusMillis(closeTimeoutMs).isBefore(Instant.now())) { - return false; - } - try { - Thread.sleep(sleepTimeMs); - } catch (InterruptedException e) { - return false; - } - return true; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/MultiClusterSessionOutputStream.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/MultiClusterSessionOutputStream.java deleted file mode 100644 index 27fc22b9675..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/MultiClusterSessionOutputStream.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.api; - -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.operationProcessor.OperationProcessor; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.time.Clock; - -/** - * Class for wiring up the Session API. It is the return value of stream() in the Session API. - * - * @author dybis -*/ -class MultiClusterSessionOutputStream extends ByteArrayOutputStream { - - private final CharSequence documentId; - private final OperationProcessor operationProcessor; - private final Object context; - private final Clock clock; - - public MultiClusterSessionOutputStream(CharSequence documentId, - OperationProcessor operationProcessor, - Object context, - Clock clock) { - this.documentId = documentId; - this.context = context; - this.operationProcessor = operationProcessor; - this.clock = clock; - } - - @Override - public void close() throws IOException { - Document document = new Document(documentId.toString(), toByteArray(), context, clock.instant()); - operationProcessor.sendDocument(document); - super.close(); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/SessionImpl.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/SessionImpl.java deleted file mode 100644 index 05b66ac4a46..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/SessionImpl.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.api; - -import com.yahoo.vespa.http.client.FeedClient; -import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.core.ThrottlePolicy; -import com.yahoo.vespa.http.client.core.operationProcessor.IncompleteResultsThrottler; -import com.yahoo.vespa.http.client.core.operationProcessor.OperationProcessor; - -import java.io.OutputStream; -import java.time.Clock; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledThreadPoolExecutor; - -/** - * This class wires up the Session API using MultiClusterHandler and MultiClusterSessionOutputStream. - * - * @deprecated - */ -@Deprecated // TODO: Remove on Vespa 8 -public class SessionImpl implements com.yahoo.vespa.http.client.Session { - - private final OperationProcessor operationProcessor; - private final BlockingQueue<Result> resultQueue = new LinkedBlockingQueue<>(); - private final Clock clock; - - public SessionImpl(SessionParams sessionParams, ScheduledThreadPoolExecutor timeoutExecutor, Clock clock) { - this.clock = clock; - this.operationProcessor = new OperationProcessor( - new IncompleteResultsThrottler( - sessionParams.getThrottlerMinSize(), - sessionParams.getClientQueueSize(), - clock, - new ThrottlePolicy()), - new FeedClient.ResultCallback() { - @Override - public void onCompletion(String docId, Result documentResult) { - resultQueue.offer(documentResult); - } - }, - sessionParams, - timeoutExecutor, - clock); - } - - @Override - public OutputStream stream(CharSequence documentId) { - return new MultiClusterSessionOutputStream(documentId, operationProcessor, null, clock); - } - - @Override - public BlockingQueue<Result> results() { - return resultQueue; - } - - @Override - public void close() { - operationProcessor.close(); - } - - @Override - public String getStatsAsJson() { - return operationProcessor.getStatsAsJson(); - } - - // For testing only (legacy tests). - public int getIncompleteResultQueueSize() { - return operationProcessor.getIncompleteResultQueueSize(); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java deleted file mode 100644 index 6b1078fa393..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java +++ /dev/null @@ -1,507 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import ai.vespa.util.http.hc4.VespaHttpClientBuilder; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.security.SslContextBuilder; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.Encoder; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.ServerResponseException; -import com.yahoo.vespa.http.client.core.Vtag; -import org.apache.http.Header; -import org.apache.http.HttpHost; -import org.apache.http.HttpRequest; -import org.apache.http.HttpResponse; -import org.apache.http.StatusLine; -import org.apache.http.auth.AUTH; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.ChallengeState; -import org.apache.http.auth.Credentials; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.entity.InputStreamEntity; -import org.apache.http.impl.auth.BasicScheme; -import org.apache.http.impl.client.BasicAuthCache; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.message.BasicHeader; -import org.apache.http.protocol.BasicHttpContext; -import org.apache.http.protocol.HttpContext; -import org.apache.http.util.EntityUtils; - -import javax.net.ssl.SSLContext; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.time.Clock; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.zip.GZIPOutputStream; - -/** - * @author Einar M R Rosenvinge - */ -class ApacheGatewayConnection implements GatewayConnection { - - private static final Logger log = Logger.getLogger(ApacheGatewayConnection.class.getName()); - private static final ObjectMapper mapper = new ObjectMapper(); - private static final String PATH = "/reserved-for-internal-use/feedapi?"; - private static final byte[] START_OF_FEED_XML = "<vespafeed>\n".getBytes(StandardCharsets.UTF_8); - private static final byte[] END_OF_FEED_XML = "\n</vespafeed>\n".getBytes(StandardCharsets.UTF_8); - private static final byte[] START_OF_FEED_JSON = "[".getBytes(StandardCharsets.UTF_8); - private static final byte[] END_OF_FEED_JSON = "]".getBytes(StandardCharsets.UTF_8); - - private final List<Integer> supportedVersions = new ArrayList<>(); - private final byte[] startOfFeed; - private final byte[] endOfFeed; - private final Endpoint endpoint; - private final FeedParams feedParams; - private final String clusterSpecificRoute; - private final ConnectionParams connectionParams; - private CloseableHttpClient httpClient; - private Instant connectionTime = null; - private Instant lastPollTime = null; - private String sessionId; - private final String clientId; - private int negotiatedVersion = -1; - private final HttpClientFactory httpClientFactory; - private final String shardingKey = UUID.randomUUID().toString().substring(0, 5); - private final Clock clock; - - ApacheGatewayConnection(Endpoint endpoint, - FeedParams feedParams, - String clusterSpecificRoute, - ConnectionParams connectionParams, - HttpClientFactory httpClientFactory, - String clientId, - Clock clock) { - supportedVersions.add(3); - this.endpoint = endpoint; - this.feedParams = feedParams; - this.clusterSpecificRoute = clusterSpecificRoute; - this.httpClientFactory = httpClientFactory; - this.connectionParams = connectionParams; - this.httpClient = null; - this.clientId = clientId; - this.clock = clock; - - if (feedParams.getDataFormat() == FeedParams.DataFormat.JSON_UTF8) { - startOfFeed = START_OF_FEED_JSON; - endOfFeed = END_OF_FEED_JSON; - } else { - startOfFeed = START_OF_FEED_XML; - endOfFeed = END_OF_FEED_XML; - } - } - - @Override - public InputStream write(List<Document> docs) throws ServerResponseException, IOException { - return write(docs, false, connectionParams.getUseCompression()); - } - - @Override - public InputStream poll() throws ServerResponseException, IOException { - lastPollTime = clock.instant(); - return write(Collections.<Document>emptyList(), false, false); - } - - @Override - public Instant lastPollTime() { return lastPollTime; } - - @Override - public InputStream drain() throws ServerResponseException, IOException { - return write(Collections.<Document>emptyList(), true, false); - } - - @Override - public boolean connect() { - log.fine(() -> "Attempting to connect to " + endpoint); - if (httpClient != null) - log.log(Level.WARNING, "Previous httpClient still exists."); - httpClient = httpClientFactory.createClient(); - connectionTime = clock.instant(); - return httpClient != null; - } - - @Override - public Instant connectionTime() { return connectionTime; } - - // Protected for easier testing only. - protected static InputStreamEntity zipAndCreateEntity(final InputStream inputStream) throws IOException { - byte[] buffer = new byte[4096]; - GZIPOutputStream gzos = null; - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try { - gzos = new GZIPOutputStream(baos); - while (inputStream.available() > 0) { - int length = inputStream.read(buffer); - gzos.write(buffer, 0,length); - } - } finally { - if (gzos != null) { - gzos.close(); - } - } - byte[] fooGzippedBytes = baos.toByteArray(); - return new InputStreamEntity(new ByteArrayInputStream(fooGzippedBytes), -1); - } - - private InputStream write(List<Document> docs, boolean drain, boolean useCompression) - throws ServerResponseException, IOException { - HttpPost httpPost = createPost(drain, useCompression, false); - - ByteBuffer[] buffers = getDataWithStartAndEndOfFeed(docs, negotiatedVersion); - InputStream inputStream = new ByteBufferInputStream(buffers); - InputStreamEntity reqEntity = useCompression ? zipAndCreateEntity(inputStream) - : new InputStreamEntity(inputStream, -1); - reqEntity.setChunked(true); - httpPost.setEntity(reqEntity); - return executePost(httpPost); - } - - private ByteBuffer[] getDataWithStartAndEndOfFeed(List<Document> docs, int version) { - List<ByteBuffer> data = new ArrayList<>(); - if (version == 3) { - for (Document doc : docs) { - int operationSize = doc.size() + startOfFeed.length + endOfFeed.length; - StringBuilder envelope = new StringBuilder(); - Encoder.encode(doc.getOperationId(), envelope); - envelope.append(' '); - envelope.append(Integer.toHexString(operationSize)); - envelope.append('\n'); - data.add(StandardCharsets.US_ASCII.encode(envelope.toString())); - data.add(ByteBuffer.wrap(startOfFeed)); - data.add(doc.getData()); - data.add(ByteBuffer.wrap(endOfFeed)); - } - } else { - throw new IllegalArgumentException("Protocol version " + version + " unsupported by client."); - } - return data.toArray(new ByteBuffer[data.size()]); - } - - private HttpPost createPost(boolean drain, boolean useCompression, boolean isHandshake) { - HttpPost httpPost = new HttpPost(createUri()); - - for (int v : supportedVersions) { - httpPost.addHeader(Headers.VERSION, "" + v); - } - if (sessionId != null) { - httpPost.setHeader(Headers.SESSION_ID, sessionId); - } - if (clientId != null) { - httpPost.setHeader(Headers.CLIENT_ID, clientId); - } - httpPost.setHeader(Headers.SHARDING_KEY, shardingKey); - httpPost.setHeader(Headers.DRAIN, drain ? "true" : "false"); - if (clusterSpecificRoute != null) { - httpPost.setHeader(Headers.ROUTE, feedParams.getRoute()); - } else { - if (feedParams.getRoute() != null) { - httpPost.setHeader(Headers.ROUTE, feedParams.getRoute()); - } - } - if (!isHandshake) { - if (feedParams.getDataFormat() == FeedParams.DataFormat.JSON_UTF8) { - httpPost.setHeader(Headers.DATA_FORMAT, FeedParams.DataFormat.JSON_UTF8.name()); - } else { - httpPost.setHeader(Headers.DATA_FORMAT, FeedParams.DataFormat.XML_UTF8.name()); - } - if (feedParams.getPriority() != null) { - httpPost.setHeader(Headers.PRIORITY, feedParams.getPriority()); - } - if (connectionParams.getTraceLevel() != 0) { - httpPost.setHeader(Headers.TRACE_LEVEL, String.valueOf(connectionParams.getTraceLevel())); - } - if (negotiatedVersion == 3 && feedParams.getDenyIfBusyV3()) { - httpPost.setHeader(Headers.DENY_IF_BUSY, "true"); - } - } - if (feedParams.getSilentUpgrade()) { - httpPost.setHeader(Headers.SILENTUPGRADE, "true"); - } - httpPost.setHeader(Headers.TIMEOUT, "" + feedParams.getServerTimeout(TimeUnit.SECONDS)); - - for (Map.Entry<String, String> extraHeader : connectionParams.getHeaders()) { - httpPost.addHeader(extraHeader.getKey(), extraHeader.getValue()); - } - connectionParams.getDynamicHeaders().forEach((headerName, provider) -> { - String headerValue = Objects.requireNonNull( - provider.getHeaderValue(), - provider.getClass().getName() + ".getHeader() returned null as header value!"); - httpPost.addHeader(headerName, headerValue); - }); - - if (useCompression) { // This causes the apache client to gzip the request content. Weird, huh? - httpPost.setHeader("Content-Encoding", "gzip"); - } - return httpPost; - } - - private InputStream executePost(HttpPost httpPost) throws ServerResponseException, IOException { - if (httpClient == null) - throw new IOException("Trying to executePost while not having a connection/http client"); - String proxyAuthzHeader = getCustomProxyAuthorizationHeader(connectionParams).orElse(null); - HttpResponse response; - if (connectionParams.getProxyHost() != null && proxyAuthzHeader != null) { - HttpContext context = createContextForcingPreemptiveProxyAuth(proxyAuthzHeader); - response = httpClient.execute(httpPost, context); - } else { - response = httpClient.execute(httpPost); - } - try { - verifyServerResponseCode(response); - verifyServerVersion(response.getFirstHeader(Headers.VERSION)); - verifySessionHeader(response.getFirstHeader(Headers.SESSION_ID)); - } catch (ServerResponseException e) { - // Ensure response is consumed to allow connection reuse later on - EntityUtils.consumeQuietly(response.getEntity()); - throw e; - } - // Consume response now to allow connection to be reused immediately - byte[] responseData = EntityUtils.toByteArray(response.getEntity()); - return responseData == null ? null : new ByteArrayInputStream(responseData); - } - - private static Optional<String> getCustomProxyAuthorizationHeader(ConnectionParams params) { - return params.getHeaders().stream() - .filter(h -> h.getKey().equals(AUTH.PROXY_AUTH_RESP)) - .findAny() - .map(Map.Entry::getValue); - } - - private HttpContext createContextForcingPreemptiveProxyAuth(String proxyAuthzHeader) { - BasicAuthCache authCache = new BasicAuthCache(); - HttpHost proxy = new HttpHost(connectionParams.getProxyHost(), connectionParams.getProxyPort()); - authCache.put(proxy, new CustomAuthScheme(proxyAuthzHeader)); - HttpContext context = new BasicHttpContext(); - context.setAttribute(HttpClientContext.AUTH_CACHE, authCache); - BasicCredentialsProvider prov = new BasicCredentialsProvider(); - prov.setCredentials(new AuthScope(proxy), new UsernamePasswordCredentials("", "")); - context.setAttribute(HttpClientContext.CREDS_PROVIDER, prov); - return context; - } - private static class CustomAuthScheme extends BasicScheme { - final String proxyAuthzHeader; - @SuppressWarnings("deprecation") - CustomAuthScheme(String proxyAuthzHeader) { - super(ChallengeState.PROXY); - this.proxyAuthzHeader = proxyAuthzHeader; - } - @Override - public Header authenticate(Credentials credentials, HttpRequest request, HttpContext context) { - return new BasicHeader(AUTH.PROXY_AUTH_RESP, proxyAuthzHeader); - } - } - - - private void verifyServerResponseCode(HttpResponse response) throws ServerResponseException { - StatusLine statusLine = response.getStatusLine(); - int statusCode = statusLine.getStatusCode(); - - // We use code 261-299 to report errors related to internal transitive errors that the tenants should not care - // about to avoid masking more serious errors. - if (statusCode > 199 && statusCode < 260) return; - if (statusCode == 299) throw new ServerResponseException(429, "Too many requests."); - throw new ServerResponseException(statusCode, - tryGetDetailedErrorMessage(response).orElseGet(statusLine::getReasonPhrase)); - } - - private static Optional<String> tryGetDetailedErrorMessage(HttpResponse response) { - Header contentType = response.getEntity().getContentType(); - if (contentType == null || !contentType.getValue().equalsIgnoreCase("application/json")) return Optional.empty(); - try (InputStream in = response.getEntity().getContent()) { - JsonNode jsonNode = mapper.readTree(in); - JsonNode message = jsonNode.get("message"); - if (message == null || message.textValue() == null) return Optional.empty(); - return Optional.of(response.getStatusLine().getReasonPhrase() + " - " + message.textValue()); - } catch (IOException e) { - return Optional.empty(); - } - } - - private void verifySessionHeader(Header serverHeader) throws ServerResponseException { - if (serverHeader == null) { - throw new ServerResponseException("Got no session ID from server."); - } - final String serverHeaderVal = serverHeader.getValue().trim(); - if (negotiatedVersion == 3) { - if (clientId == null || !clientId.equals(serverHeaderVal)) { - String message = "Running using v3. However, server responds with different session " + - "than client has set; " + serverHeaderVal + " vs client code " + clientId; - log.severe(message); - throw new ServerResponseException(message); - } - return; - } - if (sessionId == null) { //this must be the first request - log.finer("Got session ID from server: " + serverHeaderVal); - this.sessionId = serverHeaderVal; - } else { - if (!sessionId.equals(serverHeaderVal)) { - log.info("Request has been routed to a server which does not recognize the client session." + - " Most likely cause is upgrading of cluster, transitive error."); - throw new ServerResponseException("Session ID received from server ('" + serverHeaderVal + - "') does not match cached session ID ('" + sessionId + "')"); - } - } - } - - private void verifyServerVersion(Header serverHeader) throws ServerResponseException { - if (serverHeader == null) { - throw new ServerResponseException("Got bad protocol version from server."); - } - int serverVersion; - try { - serverVersion = Integer.parseInt(serverHeader.getValue()); - } catch (NumberFormatException nfe) { - throw new ServerResponseException("Got bad protocol version from server: " + nfe.getMessage()); - } - if (!supportedVersions.contains(serverVersion)) { - throw new ServerResponseException("Unsupported version: " + serverVersion - + ". Supported versions: " + supportedVersions); - } - if (negotiatedVersion == -1) { - if (log.isLoggable(Level.FINE)) { - log.log(Level.FINE, "Server decided upon protocol version " + serverVersion + "."); - } - } - this.negotiatedVersion = serverVersion; - } - - private String createUri() { - StringBuilder u = new StringBuilder(); - u.append(endpoint.isUseSsl() ? "https://" : "http://"); - u.append(endpoint.getHostname()); - u.append(":").append(endpoint.getPort()); - u.append(PATH); - u.append(feedParams.toUriParameters()); - return u.toString(); - } - - @Override - public Endpoint getEndpoint() { - return endpoint; - } - - @Override - public void handshake() throws ServerResponseException, IOException { - boolean useCompression = false; - boolean drain = false; - boolean handshake = true; - HttpPost httpPost = createPost(drain, useCompression, handshake); - - String oldSessionID = sessionId; - sessionId = null; - try (InputStream stream = executePost(httpPost)) { - if (oldSessionID != null && !oldSessionID.equals(sessionId)) { - throw new ServerResponseException( - "Session ID changed after new handshake, some documents might not be acked to correct thread. " - + getEndpoint() + " old " + oldSessionID + " new " + sessionId); - } - if (stream == null) { - log.fine("Stream is null."); - } - log.fine("Got session ID " + sessionId); - } - } - - @Override - public void close() { - try { - if (httpClient != null) - httpClient.close(); - } - catch (IOException e) { - log.log(Level.WARNING, "Failed closing HTTP client", e); - } - httpClient = null; - } - - /** - * On re-connect we want to recreate the connection, hence we need a factory. - */ - public static class HttpClientFactory { - - private final FeedParams feedParams; - final ConnectionParams connectionParams; - final boolean useSsl; - - public HttpClientFactory(FeedParams feedParams, ConnectionParams connectionParams, boolean useSsl) { - this.feedParams = feedParams; - this.connectionParams = connectionParams; - this.useSsl = useSsl; - } - - public CloseableHttpClient createClient() { - HttpClientBuilder clientBuilder; - if (connectionParams.useTlsConfigFromEnvironment()) { - clientBuilder = VespaHttpClientBuilder.create(); - } else { - clientBuilder = HttpClientBuilder.create(); - if (connectionParams.getSslContext() != null) { - setSslContext(clientBuilder, connectionParams.getSslContext()); - } else { - SslContextBuilder builder = new SslContextBuilder(); - if (connectionParams.getPrivateKey() != null && connectionParams.getCertificate() != null) { - builder.withKeyStore(connectionParams.getPrivateKey(), connectionParams.getCertificate()); - } - if (connectionParams.getCaCertificates() != null) { - builder.withTrustStore(connectionParams.getCaCertificates()); - } - setSslContext(clientBuilder, builder.build()); - } - if (connectionParams.getHostnameVerifier() != null) { - clientBuilder.setSSLHostnameVerifier(connectionParams.getHostnameVerifier()); - } - clientBuilder.setUserTokenHandler(context -> null); // https://stackoverflow.com/a/42112034/1615280 - } - clientBuilder.setMaxConnPerRoute(1); - clientBuilder.setMaxConnTotal(1); - clientBuilder.setUserAgent(String.format("vespa-http-client (%s)", Vtag.V_TAG_COMPONENT)); - clientBuilder.setDefaultHeaders(Collections.singletonList(new BasicHeader(Headers.CLIENT_VERSION, Vtag.V_TAG_COMPONENT))); - int millisTotalTimeout = (int) (feedParams.getClientTimeout(TimeUnit.MILLISECONDS) + feedParams.getServerTimeout(TimeUnit.MILLISECONDS)); - RequestConfig.Builder requestConfigBuilder = RequestConfig.custom() - .setSocketTimeout(millisTotalTimeout) - .setConnectTimeout(millisTotalTimeout); - if (connectionParams.getProxyHost() != null) { - requestConfigBuilder.setProxy(new HttpHost(connectionParams.getProxyHost(), connectionParams.getProxyPort())); - } - clientBuilder.setDefaultRequestConfig(requestConfigBuilder.build()); - - log.fine(() -> "Creating HttpClient:" + - " ConnectionTimeout " + connectionParams.getConnectionTimeToLive().getSeconds() + " seconds" + - " proxyhost (can be null) " + connectionParams.getProxyHost() + ":" + connectionParams.getProxyPort() - + (useSsl ? " using ssl " : " not using ssl") - ); - return clientBuilder.build(); - } - } - - // Note: Using deprecated setSslContext() to allow httpclient 4.4 on classpath (e.g unexpected Maven dependency resolution for test classpath) - @SuppressWarnings("deprecation") - private static void setSslContext(HttpClientBuilder builder, SSLContext sslContext) { - builder.setSslcontext(sslContext); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionFactory.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionFactory.java deleted file mode 100644 index 0b02626d6e8..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionFactory.java +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.FeedParams; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.time.Clock; -import java.util.Objects; - -/** - * @author bratseth - */ -public class ApacheGatewayConnectionFactory implements GatewayConnectionFactory { - - private final Endpoint endpoint; - private final FeedParams feedParams; - private final String clusterSpecificRoute; - private final ConnectionParams connectionParams; - private final ApacheGatewayConnection.HttpClientFactory httpClientFactory; - private final String clientId; - private final Clock clock; - - public ApacheGatewayConnectionFactory(Endpoint endpoint, - FeedParams feedParams, - String clusterSpecificRoute, - ConnectionParams connectionParams, - ApacheGatewayConnection.HttpClientFactory httpClientFactory, - String clientId, - Clock clock) { - this.endpoint = validate(endpoint); - this.feedParams = feedParams; - this.clusterSpecificRoute = clusterSpecificRoute; - this.httpClientFactory = httpClientFactory; - this.connectionParams = connectionParams; - this.clientId = Objects.requireNonNull(clientId, "clientId cannot be null"); - this.clock = clock; - } - - private static Endpoint validate(Endpoint endpoint) { - try { - InetAddress.getByName(endpoint.getHostname()); - return endpoint; - } - catch (UnknownHostException e) { - throw new IllegalArgumentException("Unknown host: " + endpoint); - } - } - - @Override - public GatewayConnection newConnection() { - return new ApacheGatewayConnection(endpoint, - feedParams, - clusterSpecificRoute, - connectionParams, - httpClientFactory, - clientId, - clock); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStream.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStream.java deleted file mode 100644 index f88519c2615..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStream.java +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.util.ArrayDeque; -import java.util.Deque; - -/** - * @author Einar M R Rosenvinge - */ -class ByteBufferInputStream extends InputStream { - private final Deque<ByteBuffer> currentBuffers = new ArrayDeque<>(); - - ByteBufferInputStream(ByteBuffer[] buffers) { - for (int i = buffers.length - 1; i > -1; i--) { - currentBuffers.push(buffers[i]); - } - } - - @Override - public int read() throws IOException { - pop(); - if (currentBuffers.isEmpty()) { - return -1; - } - return currentBuffers.peek().get(); - } - - private void pop() { - if (currentBuffers.isEmpty()) { - return; - } - - while (!currentBuffers.isEmpty() && !currentBuffers.peek().hasRemaining()) { - //it's exhausted, get rid of it - currentBuffers.pop(); - } - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - if (b == null) { - throw new NullPointerException(); - } else if (off < 0 || len < 0 || len > b.length - off) { - throw new IndexOutOfBoundsException(); - } else if (len == 0) { - return 0; - } - pop(); - if (currentBuffers.isEmpty()) { - return -1; - } - int toRead = Math.min(len, currentBuffers.peek().remaining()); - currentBuffers.peek().get(b, off, toRead); - return toRead; - } - - @Override - public long skip(long n) throws IOException { - throw new IOException("skip() not supported."); - } - - @Override - public int available() throws IOException { - if (currentBuffers.isEmpty()) { - return 0; - } - - int size = 0; - for (ByteBuffer b : currentBuffers) { - size += b.remaining(); - } - return size; - } -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java deleted file mode 100644 index ef5b7afd16a..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.Exceptions; -import com.yahoo.vespa.http.client.core.operationProcessor.OperationProcessor; - -import java.io.IOException; -import java.io.StringWriter; -import java.time.Clock; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -/** - * @author Einar M R Rosenvinge - */ -public class ClusterConnection implements AutoCloseable { - - private static ObjectMapper jsonMapper = createMapper(); - - private final List<IOThread> ioThreads = new ArrayList<>(); - private final int clusterId; - private final ThreadGroup ioThreadGroup; - - /** The shared queue of document operations the io threads will take from */ - private final DocumentQueue documentQueue; - - /** The single endpoint this sends to, or null if it will send to multiple endpoints */ - private final Endpoint singleEndpoint; - - public ClusterConnection(OperationProcessor operationProcessor, - FeedParams feedParams, - ConnectionParams connectionParams, - Cluster cluster, - int clusterId, - int clientQueueSizePerCluster, - ScheduledThreadPoolExecutor timeoutExecutor, - Clock clock) { - if (cluster.getEndpoints().isEmpty()) - throw new IllegalArgumentException("At least a single endpoint is required in " + cluster); - - this.clusterId = clusterId; - int totalNumberOfEndpointsInThisCluster = cluster.getEndpoints().size() * connectionParams.getNumPersistentConnectionsPerEndpoint(); - if (totalNumberOfEndpointsInThisCluster == 0) - throw new IllegalArgumentException("At least 1 persistent connection per endpoint is required in " + cluster); - int maxInFlightPerSession = Math.max(1, feedParams.getMaxInFlightRequests() / totalNumberOfEndpointsInThisCluster); - - documentQueue = new DocumentQueue(clientQueueSizePerCluster, clock); - ioThreadGroup = operationProcessor.getIoThreadGroup(); - singleEndpoint = cluster.getEndpoints().size() == 1 ? cluster.getEndpoints().get(0) : null; - Double idlePollFrequency = feedParams.getIdlePollFrequency(); - if (idlePollFrequency == null) - idlePollFrequency = 10.0; - for (Endpoint endpoint : cluster.getEndpoints()) { - EndpointResultQueue endpointResultQueue = new EndpointResultQueue(operationProcessor, - endpoint, - clusterId, - timeoutExecutor, - feedParams.getServerTimeout(TimeUnit.MILLISECONDS) + feedParams.getClientTimeout(TimeUnit.MILLISECONDS)); - for (int i = 0; i < connectionParams.getNumPersistentConnectionsPerEndpoint(); i++) { - GatewayConnectionFactory connectionFactory; - if (connectionParams.isDryRun()) { - connectionFactory = new DryRunGatewayConnectionFactory(endpoint, clock); - } else { - connectionFactory = new ApacheGatewayConnectionFactory(endpoint, - feedParams, - cluster.getRoute(), - connectionParams, - new ApacheGatewayConnection.HttpClientFactory(feedParams, connectionParams, endpoint.isUseSsl()), - operationProcessor.getClientId(), - clock - ); - } - IOThread ioThread = new IOThread(operationProcessor.getIoThreadGroup(), - endpoint, - endpointResultQueue, - connectionFactory, - clusterId, - feedParams.getMaxChunkSizeBytes(), - maxInFlightPerSession, - Duration.ofMillis(feedParams.getLocalQueueTimeOut()), - documentQueue, - feedParams.getMaxSleepTimeMs(), - connectionParams.getConnectionTimeToLive(), - connectionParams.runThreads(), - idlePollFrequency, - clock); - ioThreads.add(ioThread); - } - } - } - - private static ObjectMapper createMapper() { - ObjectMapper mapper = new ObjectMapper(); - mapper.registerModule(new Jdk8Module()); - mapper.registerModule(new JavaTimeModule()); - return mapper; - } - - public int getClusterId() { - return clusterId; - } - - public void post(Document document) throws EndpointIOException { - try { - documentQueue.put(document, Thread.currentThread().getThreadGroup() == ioThreadGroup); - } catch (Throwable t) { // InterruptedException if shutting down, IllegalStateException if already shut down - throw new EndpointIOException(singleEndpoint, "While sending", t); - } - } - - @SuppressWarnings("ThrowableResultOfMethodCallIgnored") - @Override - public void close() { - List<Exception> exceptions = new ArrayList<>(); - for (IOThread ioThread : ioThreads) { - try { - ioThread.close(); - } catch (Exception e) { - exceptions.add(e); - } - } - if (exceptions.isEmpty()) { - return; - } - if (exceptions.size() == 1) { - if (exceptions.get(0) instanceof RuntimeException) { - throw (RuntimeException) exceptions.get(0); - } else { - throw new RuntimeException(exceptions.get(0)); - } - } - StringBuilder b = new StringBuilder(); - b.append("Exception thrown while closing one or more endpoints: "); - for (int i = 0; i < exceptions.size(); i++) { - Exception e = exceptions.get(i); - b.append(Exceptions.toMessageString(e)); - if (i != (exceptions.size() - 1)) { - b.append(", "); - } - } - throw new RuntimeException(b.toString(), exceptions.get(0)); - } - - public String getStatsAsJSon() throws IOException { - StringWriter stringWriter = new StringWriter(); - JsonGenerator jsonGenerator = jsonMapper.createGenerator(stringWriter); - jsonGenerator.writeStartObject(); - jsonGenerator.writeArrayFieldStart("session"); - for (IOThread ioThread : ioThreads) { - jsonGenerator.writeStartObject(); - jsonGenerator.writeObjectFieldStart("endpoint"); - jsonGenerator.writeStringField("host", ioThread.getEndpoint().getHostname()); - jsonGenerator.writeNumberField("port", ioThread.getEndpoint().getPort()); - jsonGenerator.writeEndObject(); - jsonGenerator.writeFieldName("stats"); - IOThread.ConnectionStats connectionStats = ioThread.getConnectionStats(); - jsonMapper.writeValue(jsonGenerator, connectionStats); - jsonGenerator.writeEndObject(); - } - jsonGenerator.writeEndArray(); - jsonGenerator.writeEndObject(); - jsonGenerator.close(); - return stringWriter.toString(); - } - - public List<IOThread> ioThreads() { - return Collections.unmodifiableList(ioThreads); - } - - @Override - public boolean equals(Object o) { - return (this == o) || (o instanceof ClusterConnection && clusterId == ((ClusterConnection) o).clusterId); - } - - @Override - public int hashCode() { - return clusterId; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DocumentQueue.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DocumentQueue.java deleted file mode 100644 index 8164534ca37..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DocumentQueue.java +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.core.Document; - -import java.time.Clock; -import java.time.Duration; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Deque; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.TimeUnit; - -/** - * Shared document queue that gives clients operations on documents which do not have operations already in flight. - * This is multithread safe. - * - * @author dybis - */ -class DocumentQueue { - - private final Deque<Document> queue; - private final int maxSize; - private boolean closed = false; - private final Clock clock; - - DocumentQueue(int maxSize, Clock clock) { - this.maxSize = maxSize; - this.queue = new ArrayDeque<>(maxSize); - this.clock = clock; - } - - List<Document> removeAllDocuments() { - synchronized (queue) { - List<Document> allDocs = new ArrayList<>(); - while (!queue.isEmpty()) { - allDocs.add(queue.poll()); - } - queue.notifyAll(); - return allDocs; - } - } - - void put(Document document, boolean calledFromIoThreadGroup) throws InterruptedException { - document.setQueueInsertTime(clock.instant()); - synchronized (queue) { - while (!closed && (queue.size() >= maxSize) && !calledFromIoThreadGroup) { - queue.wait(); - } - if (closed) { - throw new IllegalStateException("Cannot add elements to closed queue."); - } - queue.add(document); - queue.notifyAll(); - } - } - - Document poll(long timeout, TimeUnit unit) throws InterruptedException { - synchronized (queue) { - long remainingToWait = unit.toMillis(timeout); - while (queue.isEmpty()) { - long startTime = clock.millis(); - queue.wait(remainingToWait); - remainingToWait -= (clock.millis() - startTime); - if (remainingToWait <= 0) { - break; - } - } - Document document = queue.poll(); - queue.notifyAll(); - return document; - } - } - - Document poll() { - synchronized (queue) { - Document document = queue.poll(); - queue.notifyAll(); - return document; - } - } - - boolean isEmpty() { - synchronized (queue) { - return queue.isEmpty(); - } - } - - int size() { - synchronized (queue) { - return queue.size(); - } - } - - void clear() { - synchronized (queue) { - queue.clear(); - queue.notifyAll(); - } - } - - boolean close() { - boolean previousState; - synchronized (queue) { - previousState = closed; - closed = true; - queue.notifyAll(); - } - return previousState; - } - - Optional<Document> pollDocumentIfTimedoutInQueue(Duration localQueueTimeOut) { - synchronized (queue) { - if (queue.isEmpty()) return Optional.empty(); - - Document document = queue.peek(); - if (document.getQueueInsertTime().plus(localQueueTimeOut).isBefore(clock.instant())) - return Optional.ofNullable(queue.poll()); - else - return Optional.empty(); - } - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DryRunGatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DryRunGatewayConnection.java deleted file mode 100644 index a6f6992b238..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DryRunGatewayConnection.java +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.ErrorCode; -import com.yahoo.vespa.http.client.core.OperationStatus; -import com.yahoo.vespa.http.client.core.ServerResponseException; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.time.Clock; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * Dummy implementation. - * - * @author dybis - */ -public class DryRunGatewayConnection implements GatewayConnection { - - private final Endpoint endpoint; - private final Clock clock; - private Instant connectionTime = null; - private Instant lastPollTime = null; - - /** Set to true to hold off responding with a result to any incoming operations until this is set false */ - private boolean hold = false; - private final List<Document> held = new ArrayList<>(); - - /** If this is set, handshake operations will throw this exception */ - private ServerResponseException throwThisOnHandshake = null; - - /** If this is set, all write operations will throw this exception */ - private IOException throwThisOnWrite = null; - - public DryRunGatewayConnection(Endpoint endpoint, Clock clock) { - this.endpoint = endpoint; - this.clock = clock; - } - - @Override - public synchronized InputStream write(List<Document> docs) throws IOException { - if (throwThisOnWrite != null) - throw throwThisOnWrite; - - if (hold) { - held.addAll(docs); - return new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8)); - } - else { - StringBuilder result = new StringBuilder(); - for (Document doc : held) - result.append(okResponse(doc).render()); - held.clear(); - for (Document doc : docs) - result.append(okResponse(doc).render()); - return new ByteArrayInputStream(result.toString().getBytes(StandardCharsets.UTF_8)); - } - } - - @Override - public synchronized InputStream poll() throws IOException { - lastPollTime = clock.instant(); - return write(new ArrayList<>()); - } - - @Override - public synchronized Instant lastPollTime() { return lastPollTime; } - - @Override - public synchronized InputStream drain() throws IOException { - return write(new ArrayList<>()); - } - - @Override - public synchronized boolean connect() { - connectionTime = clock.instant(); - return true; - } - - @Override - public synchronized Instant connectionTime() { return connectionTime; } - - @Override - public Endpoint getEndpoint() { - return endpoint; - } - - @Override - public synchronized void handshake() throws ServerResponseException { - if (throwThisOnHandshake != null) - throw throwThisOnHandshake; - } - - @Override - public synchronized void close() { } - - public synchronized void hold(boolean hold) { - this.hold = hold; - } - - /** Returns the document currently held in this */ - public synchronized List<Document> held() { return Collections.unmodifiableList(held); } - - public synchronized void throwOnWrite(IOException throwThisOnWrite) { - this.throwThisOnWrite = throwThisOnWrite; - } - - public synchronized void throwOnHandshake(ServerResponseException throwThisOnHandshake) { - this.throwThisOnHandshake = throwThisOnHandshake; - } - - private OperationStatus okResponse(Document document) { - return new OperationStatus("ok", document.getOperationId(), ErrorCode.OK, false, ""); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DryRunGatewayConnectionFactory.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DryRunGatewayConnectionFactory.java deleted file mode 100644 index 01bec563889..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DryRunGatewayConnectionFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.config.Endpoint; - -import java.time.Clock; - -/** - * @author bratseth - */ -public class DryRunGatewayConnectionFactory implements GatewayConnectionFactory { - - private final Endpoint endpoint; - private final Clock clock; - - public DryRunGatewayConnectionFactory(Endpoint endpoint, Clock clock) { - this.endpoint = endpoint; - this.clock = clock; - } - - @Override - public GatewayConnection newConnection() { - return new DryRunGatewayConnection(endpoint, clock); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointIOException.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointIOException.java deleted file mode 100644 index 34b8ef76068..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointIOException.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.config.Endpoint; - -import java.io.IOException; - -/** - * Class for throwing exception from endpoint. - * - * @author dybis -*/ -public class EndpointIOException extends IOException { - - private final Endpoint endpoint; - private static final long serialVersionUID = 29335813211L; - - public EndpointIOException(Endpoint endpoint, String message, Throwable cause) { - super(message, cause); - this.endpoint = endpoint; - } - - /** Returns the endpoint, or null if the failure occurred before this was assigned to a unique endpoint */ - public Endpoint getEndpoint() { return endpoint; } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java deleted file mode 100644 index 98b6aee1e33..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.FeedEndpointException; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.core.EndpointResult; -import com.yahoo.vespa.http.client.core.operationProcessor.EndPointResultFactory; -import com.yahoo.vespa.http.client.core.operationProcessor.OperationProcessor; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; - -/** - * The shared queue of operation results. - * This is multithread safe. - * - * @author Einar M R Rosenvinge - */ -class EndpointResultQueue { - - private static final Logger log = Logger.getLogger(EndpointResultQueue.class.getName()); - private final OperationProcessor operationProcessor; - - private final Map<String, InflightOperation> inflightOperations = new HashMap<>(); - - private final Endpoint endpoint; - private final int clusterId; - private final ScheduledThreadPoolExecutor timer; - private final long totalTimeoutMs; - - EndpointResultQueue(OperationProcessor operationProcessor, - Endpoint endpoint, - int clusterId, - ScheduledThreadPoolExecutor timer, - long totalTimeoutMs) { - this.operationProcessor = operationProcessor; - this.endpoint = endpoint; - this.clusterId = clusterId; - this.timer = timer; - this.totalTimeoutMs = totalTimeoutMs; - } - - public synchronized void operationSent(String operationId, GatewayConnection connection) { - DocumentTimerTask task = new DocumentTimerTask(operationId); - ScheduledFuture<?> future = timer.schedule(task, totalTimeoutMs, TimeUnit.MILLISECONDS); - inflightOperations.put(operationId, new InflightOperation(future, connection)); - } - - public synchronized void failOperation(EndpointResult result, int clusterId) { - resultReceived(result, clusterId, false); - } - - public synchronized void resultReceived(EndpointResult result, int clusterId) { - resultReceived(result, clusterId, true); - } - - void onEndpointError(FeedEndpointException e) { - operationProcessor.onEndpointError(e); - } - - private synchronized void resultReceived(EndpointResult result, int clusterId, boolean duplicateGivesWarning) { - operationProcessor.resultReceived(result, clusterId); - InflightOperation operation = inflightOperations.remove(result.getOperationId()); - if (operation == null) { - if (duplicateGivesWarning) { - log.info("Result for ID '" + result.getOperationId() + "' received from '" + endpoint + - "', but we have no record of a sent operation. This may happen if an operation is " + - "initiated, but also retried, due to HTTP failure. Otherwise, something is wrong on " + - "the server side (bad VIP usage?), or operation was received _after_ client-side timeout."); - } - return; - } - operation.future.cancel(false); - } - - /** Called only from ScheduledThreadPoolExecutor thread in DocumentTimerTask.run(), see below */ - private synchronized void timeout(String operationId) { - InflightOperation operation = inflightOperations.remove(operationId); - if (operation == null) { - log.finer("Timeout of operation '" + operationId + "', but operation " + - "not found in map. Result was probably received just-in-time from server, while timeout " + - "task could not be cancelled."); - return; - } - EndpointResult endpointResult = EndPointResultFactory.createTransientError( - endpoint, operationId, new RuntimeException("Timed out waiting for reply from server.")); - operationProcessor.resultReceived(endpointResult, clusterId); - } - - public synchronized int getPendingSize() { - return inflightOperations.values().size(); - } - - public synchronized void failPending(Exception exception) { - inflightOperations.forEach((operationId, operation) -> { - operation.future.cancel(false); - EndpointResult result = EndPointResultFactory.createError(endpoint, operationId, exception); - operationProcessor.resultReceived(result, clusterId); - }); - inflightOperations.clear(); - } - - public synchronized boolean hasInflightOperations(GatewayConnection connection) { - return inflightOperations.entrySet().stream() - .anyMatch(entry -> entry.getValue().connection.equals(connection)); - } - - private class DocumentTimerTask implements Runnable { - - private final String operationId; - - private DocumentTimerTask(String operationId) { - this.operationId = operationId; - } - - @Override - public void run() { - timeout(operationId); - } - - } - - private static class InflightOperation { - final ScheduledFuture<?> future; - final GatewayConnection connection; - - InflightOperation(ScheduledFuture<?> future, GatewayConnection connection) { - this.future = future; - this.connection = connection; - } - } -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayConnection.java deleted file mode 100644 index 25057a1dead..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayConnection.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.ServerResponseException; -import java.io.IOException; -import java.io.InputStream; -import java.time.Instant; -import java.util.List; - -public interface GatewayConnection { - - /** Returns the time this connected over the network, or null if not connected yet */ - Instant connectionTime(); - - /** Returns the last time poll was called on this, or null if never */ - Instant lastPollTime(); - - InputStream write(List<Document> docs) throws ServerResponseException, IOException; - - /** Returns any operation results that are ready now */ - InputStream poll() throws ServerResponseException, IOException; - - /** Attempt to drain all outstanding operations, even if this leads to blocking */ - InputStream drain() throws ServerResponseException, IOException; - - boolean connect(); - - Endpoint getEndpoint(); - - void handshake() throws ServerResponseException, IOException; - - void close(); - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayConnectionFactory.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayConnectionFactory.java deleted file mode 100644 index 4988b73510f..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayConnectionFactory.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -/** - * Creates gateway connections on request - * - * @author bratseth - */ -public interface GatewayConnectionFactory { - - GatewayConnection newConnection(); - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottler.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottler.java deleted file mode 100644 index 41a1b6a7e87..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottler.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import java.util.concurrent.ThreadLocalRandom; - -/** - * When the gateways says it can not handle more load, we should send less load. That is the responsibility - * of this component - * - * @author dybis - */ -public class GatewayThrottler { - - private long backOffTimeMs = 0; - private final long maxSleepTimeMs; - - public GatewayThrottler(long maxSleepTimeMs) { - this.maxSleepTimeMs = maxSleepTimeMs; - } - - public void handleCall(int transientErrors) { - if (transientErrors > 0) { - backOffTimeMs = Math.min(maxSleepTimeMs, backOffTimeMs + distribute(100)); - } else { - backOffTimeMs = Math.max(0, backOffTimeMs - distribute(10)); - } - sleepMs(backOffTimeMs); - } - - protected void sleepMs(long sleepTime) { - try { - if (backOffTimeMs > 0L) { - Thread.sleep(backOffTimeMs); - } - } catch (InterruptedException e) { - // Do nothing - } - } - - public int distribute(int expected) { - double factor = 0.5 + ThreadLocalRandom.current().nextDouble(); - Double result = expected * factor; - return result.intValue(); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java deleted file mode 100644 index aeb164227f1..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java +++ /dev/null @@ -1,649 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.FeedConnectException; -import com.yahoo.vespa.http.client.FeedProtocolException; -import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.EndpointResult; -import com.yahoo.vespa.http.client.core.Exceptions; -import com.yahoo.vespa.http.client.core.ServerResponseException; -import com.yahoo.vespa.http.client.core.operationProcessor.EndPointResultFactory; - -import java.io.IOException; -import java.io.InputStream; -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.Random; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Thread which feeds document operations asynchronously and processes the results. - * - * @author Einar M R Rosenvinge - */ -public class IOThread implements Runnable, AutoCloseable { - - private static final Logger log = Logger.getLogger(IOThread.class.getName()); - - private final Endpoint endpoint; - private final GatewayConnectionFactory connectionFactory; - private final DocumentQueue documentQueue; - private final EndpointResultQueue resultQueue; - - /** The thread running this, or null if it does not run a thread (meaning tick() must be called from the outside) */ - private final Thread thread; - private final int clusterId; - private final CountDownLatch running = new CountDownLatch(1); - private final CountDownLatch stopSignal = new CountDownLatch(1); - private final int maxChunkSizeBytes; - private final int maxInFlightRequests; - private final Duration localQueueTimeOut; - private final GatewayThrottler gatewayThrottler; - private final Duration connectionTimeToLive; - private final long pollIntervalUS; - private final Clock clock; - private final Random random = new Random(); - private final OldConnectionsDrainer oldConnectionsDrainer; - - private volatile GatewayConnection currentConnection; - private volatile ConnectionState connectionState = ConnectionState.DISCONNECTED; - - private enum ConnectionState { DISCONNECTED, CONNECTED, SESSION_SYNCED }; - private final AtomicInteger wrongSessionDetectedCounter = new AtomicInteger(0); - private final AtomicInteger wrongVersionDetectedCounter = new AtomicInteger(0); - private final AtomicInteger problemStatusCodeFromServerCounter = new AtomicInteger(0); - private final AtomicInteger executeProblemsCounter = new AtomicInteger(0); - private final AtomicInteger docsReceivedCounter = new AtomicInteger(0); - private final AtomicInteger statusReceivedCounter = new AtomicInteger(0); - private final AtomicInteger pendingDocumentStatusCount = new AtomicInteger(0); - private final AtomicInteger successfulHandshakes = new AtomicInteger(0); - private final AtomicInteger lastGatewayProcessTimeMillis = new AtomicInteger(0); - - IOThread(ThreadGroup ioThreadGroup, - Endpoint endpoint, - EndpointResultQueue endpointResultQueue, - GatewayConnectionFactory connectionFactory, - int clusterId, - int maxChunkSizeBytes, - int maxInFlightRequests, - Duration localQueueTimeOut, - DocumentQueue documentQueue, - long maxSleepTimeMs, - Duration connectionTimeToLive, - boolean runThreads, - double idlePollFrequency, - Clock clock) { - this.endpoint = endpoint; - this.documentQueue = documentQueue; - this.connectionFactory = connectionFactory; - this.currentConnection = connectionFactory.newConnection(); - this.resultQueue = endpointResultQueue; - this.clusterId = clusterId; - this.maxChunkSizeBytes = maxChunkSizeBytes; - this.maxInFlightRequests = maxInFlightRequests; - this.connectionTimeToLive = connectionTimeToLive; - this.gatewayThrottler = new GatewayThrottler(maxSleepTimeMs); - this.pollIntervalUS = Math.max(1000, (long)(1000000.0/Math.max(0.1, idlePollFrequency))); // ensure range [1ms, 10s] - this.clock = clock; - this.localQueueTimeOut = localQueueTimeOut; - this.oldConnectionsDrainer = new OldConnectionsDrainer(endpoint, - clusterId, - Duration.ofMillis(pollIntervalUS/1000), - connectionTimeToLive, - localQueueTimeOut, - statusReceivedCounter, - resultQueue, - stopSignal, - clock); - if (runThreads) { - this.thread = new Thread(ioThreadGroup, this, "IOThread " + endpoint); - thread.setDaemon(true); - thread.start(); - Thread thread = new Thread(ioThreadGroup, oldConnectionsDrainer, "IOThread " + endpoint + " drainer"); - thread.setDaemon(true); - thread.start(); - } - else { - this.thread = null; - } - } - - public Endpoint getEndpoint() { - return endpoint; - } - - /** - * Returns a snapshot of counters. Threadsafe. - */ - public ConnectionStats getConnectionStats() { - return new ConnectionStats( - wrongSessionDetectedCounter.get(), - wrongVersionDetectedCounter.get(), - problemStatusCodeFromServerCounter.get(), - executeProblemsCounter.get(), - docsReceivedCounter.get(), - statusReceivedCounter.get(), - pendingDocumentStatusCount.get(), - successfulHandshakes.get(), - lastGatewayProcessTimeMillis.get()); - } - - @Override - public void close() { - documentQueue.close(); - if (stopSignal.getCount() == 0) return; - - stopSignal.countDown(); - log.finer("Closed called."); - - oldConnectionsDrainer.close(); - - // Make a last attempt to get results from previous operations, we have already waited quite a bit before getting here. - int size = resultQueue.getPendingSize(); - if (size > 0) { - log.info("We have outstanding operations (" + size + ") , trying to fetch responses."); - try { - processResponse(currentConnection.drain()); - } catch (Throwable e) { - log.log(Level.SEVERE, "Some failures while trying to get latest responses from vespa.", e); - } - } - - try { - currentConnection.close(); - } finally { - // If there is still documents in the queue, fail them. - drainDocumentQueueWhenFailingPermanently(new Exception("Closed call, did not manage to process everything so failing this document.")); - } - - log.fine("Session to " + endpoint + " closed."); - } - - /** For testing only */ - public void post(Document document) throws InterruptedException { - documentQueue.put(document, true); - } - - @Override - public String toString() { - return "I/O thread (for " + endpoint + ")"; - } - - List<Document> getNextDocsForFeeding(long maxWaitUnits, TimeUnit timeUnit) { - List<Document> docsForSendChunk = new ArrayList<>(); - int chunkSizeBytes = 0; - try { - drainFirstDocumentsInQueueIfOld(); - Document doc = thread != null ? documentQueue.poll(maxWaitUnits, timeUnit) : documentQueue.poll(); - if (doc != null) { - docsForSendChunk.add(doc); - chunkSizeBytes = doc.size(); - } - } catch (InterruptedException ie) { - log.fine("Got break signal while waiting for new documents to feed"); - return docsForSendChunk; - } - int pendingSize = 1 + resultQueue.getPendingSize(); - - // see if we can get more documents without blocking - // slightly randomize how much is taken to avoid harmonic interactions leading - // to some threads consistently taking more than others - int thisMaxChunkSizeBytes = randomize(maxChunkSizeBytes); - int thisMaxInFlightRequests = randomize(maxInFlightRequests); - while (chunkSizeBytes < thisMaxChunkSizeBytes && pendingSize < thisMaxInFlightRequests) { - drainFirstDocumentsInQueueIfOld(); - Document document = documentQueue.poll(); - if (document == null) break; - docsForSendChunk.add(document); - chunkSizeBytes += document.size(); - pendingSize++; - } - if (log.isLoggable(Level.FINEST)) - log.finest("Chunk has " + docsForSendChunk.size() + " docs with a size " + chunkSizeBytes + " bytes"); - docsReceivedCounter.addAndGet(docsForSendChunk.size()); - return docsForSendChunk; - } - - private int randomize(int limit) { - double multiplier = 0.75 + 0.25 * random.nextDouble(); - return Math.max(1, (int)(limit * multiplier)); - } - - private void addDocumentsToResultQueue(List<Document> docs) { - for (Document doc : docs) { - resultQueue.operationSent(doc.getOperationId(), currentConnection); - } - } - - private void markDocumentAsFailed(List<Document> docs, ServerResponseException servletException) { - for (Document doc : docs) { - resultQueue.failOperation(EndPointResultFactory.createTransientError(endpoint, - doc.getOperationId(), - servletException), - clusterId); - } - } - - private InputStream sendAndReceive(List<Document> docs) throws IOException, ServerResponseException { - try { - // Post the new docs and get async responses for other posts. - return currentConnection.write(docs); - } catch (ServerResponseException ser) { - markDocumentAsFailed(docs, ser); - throw ser; - } catch (Exception e) { - markDocumentAsFailed(docs, new ServerResponseException(Exceptions.toMessageString(e))); - throw e; - } - } - - private static class ProcessResponse { - - private final int transitiveErrorCount; - private final int processResultsCount; - - ProcessResponse(int transitiveErrorCount, int processResultsCount) { - this.transitiveErrorCount = transitiveErrorCount; - this.processResultsCount = processResultsCount; - } - - } - - private ProcessResponse processResponse(InputStream serverResponse) throws IOException { - return processResponse(serverResponse, endpoint, clusterId, statusReceivedCounter, resultQueue); - } - - private static ProcessResponse processResponse(InputStream serverResponse, - Endpoint endpoint, - int clusterId, - AtomicInteger statusReceivedCounter, - EndpointResultQueue resultQueue) throws IOException { - Collection<EndpointResult> endpointResults = EndPointResultFactory.createResult(endpoint, serverResponse); - statusReceivedCounter.addAndGet(endpointResults.size()); - int transientErrors = 0; - for (EndpointResult endpointResult : endpointResults) { - if (endpointResult.getDetail().getResultType() == Result.ResultType.TRANSITIVE_ERROR) { - transientErrors++; - } - resultQueue.resultReceived(endpointResult, clusterId); - } - return new ProcessResponse(transientErrors, endpointResults.size()); - } - - private ProcessResponse feedDocumentAndProcessResults(List<Document> docs) - throws ServerResponseException, IOException { - addDocumentsToResultQueue(docs); - long startTime = clock.millis(); - InputStream serverResponse = sendAndReceive(docs); - - ProcessResponse processResponse = processResponse(serverResponse); - lastGatewayProcessTimeMillis.set((int) (clock.millis() - startTime)); - return processResponse; - } - - private ProcessResponse pullAndProcessData(long maxWaitTimeUS) throws ServerResponseException, IOException { - int pendingResultQueueSize = resultQueue.getPendingSize(); - pendingDocumentStatusCount.set(pendingResultQueueSize); - - List<Document> nextDocsForFeeding = (pendingResultQueueSize > maxInFlightRequests) - ? new ArrayList<>() // The queue is full, will not send more documents - : getNextDocsForFeeding(maxWaitTimeUS, TimeUnit.MICROSECONDS); - - if (nextDocsForFeeding.isEmpty() && pendingResultQueueSize == 0) { - //we have no unfinished business with the server now. - log.finest("No document awaiting feeding, not waiting for results."); - return new ProcessResponse(0, 0); - } - log.finest("Awaiting " + pendingResultQueueSize + " results."); - ProcessResponse processResponse = feedDocumentAndProcessResults(nextDocsForFeeding); - - if (pendingResultQueueSize > maxInFlightRequests && processResponse.processResultsCount == 0) { - try { - // Max outstanding document operations, no more results on server side, wait a bit before asking again - Thread.sleep(300); - } catch (InterruptedException e) { - // Ignore - } - } - return processResponse; - } - - /** Given a current connection state, take the appropriate action and return the resulting new connection state */ - private ConnectionState cycle(ConnectionState connectionState) { - switch(connectionState) { - case DISCONNECTED: - try { - if (! currentConnection.connect()) { - log.log(Level.WARNING, "Could not connect to endpoint: '" + endpoint + "'. Will re-try."); - drainFirstDocumentsInQueueIfOld(); - return ConnectionState.DISCONNECTED; - } - return ConnectionState.CONNECTED; - } catch (Throwable throwable1) { - drainFirstDocumentsInQueueIfOld(); - - log.log(Level.INFO, "Failed connecting to endpoint: '" + endpoint + "'. Will re-try connecting.", - throwable1); - executeProblemsCounter.incrementAndGet(); - return ConnectionState.DISCONNECTED; - } - case CONNECTED: - try { - if (isStale(currentConnection)) - return refreshConnection(connectionState); - currentConnection.handshake(); - successfulHandshakes.getAndIncrement(); - } catch (ServerResponseException ser) { - int code = ser.getResponseCode(); - if (code == 401 || code == 403) { - drainDocumentQueueWhenFailingPermanently(new Exception("Denied access by endpoint:" + ser.getResponseString())); - log.log(Level.SEVERE, "Failed authentication or authorization with '" + endpoint + "': " + Exceptions.toMessageString(ser)); - return ConnectionState.CONNECTED; // Should ideally exit immediately, instead of doing this per X documents :/ - } - - executeProblemsCounter.incrementAndGet(); - log.log(Level.INFO, "Failed talking to endpoint. Handshake with server endpoint '" + endpoint + - "' failed -- will re-try handshake: " + Exceptions.toMessageString(ser)); - - drainFirstDocumentsInQueueIfOld(); - resultQueue.onEndpointError(new FeedProtocolException(ser.getResponseCode(), ser.getResponseString(), ser, endpoint)); - return ConnectionState.CONNECTED; - } catch (Throwable throwable) { // This cover IOException as well - executeProblemsCounter.incrementAndGet(); - resultQueue.onEndpointError(new FeedConnectException(throwable, endpoint)); - log.log(Level.INFO, "Failed talking to endpoint. Handshake with server endpoint '" + - endpoint + "' failed. Will re-try handshake.", - throwable); - drainFirstDocumentsInQueueIfOld(); - currentConnection.close(); - return ConnectionState.DISCONNECTED; - } - return ConnectionState.SESSION_SYNCED; - case SESSION_SYNCED: - try { - if (isStale(currentConnection)) - return refreshConnection(connectionState); - ProcessResponse processResponse = pullAndProcessData(pollIntervalUS); - gatewayThrottler.handleCall(processResponse.transitiveErrorCount); - } - catch (ServerResponseException ser) { - log.log(Level.INFO, "Problems while handing data over to endpoint '" + endpoint + - "'. Will re-try. Endpoint responded with an unexpected HTTP response code.", - ser); - return ConnectionState.CONNECTED; - } - catch (Throwable e) { - log.log(Level.INFO, - "Connection level error handing data over to endpoint '" + endpoint + "'. Will re-try.", - e); - currentConnection.close(); - return ConnectionState.DISCONNECTED; - } - return ConnectionState.SESSION_SYNCED; - default: { - log.severe("Should never get here."); - currentConnection.close(); - return ConnectionState.DISCONNECTED; - } - } - } - - private void sleepIfProblemsGettingSyncedConnection(ConnectionState newState, ConnectionState oldState) { - if (newState == ConnectionState.SESSION_SYNCED) return; - if (newState == ConnectionState.CONNECTED && oldState == ConnectionState.DISCONNECTED) return; - try { - // Take it easy we have problems getting a connection up. - if (stopSignal.getCount() > 0 || !documentQueue.isEmpty()) { - Thread.sleep(gatewayThrottler.distribute(3000)); - } - } catch (InterruptedException e) { - } - } - - @Override - public void run() { - while (stopSignal.getCount() > 0 || !documentQueue.isEmpty()) - tick(); - log.finer(toString() + " exiting, documentQueue.size()=" + documentQueue.size()); - running.countDown(); - } - - /** Do one iteration of work. Should be called from the single worker thread of this. */ - public void tick() { - ConnectionState oldState = connectionState; - connectionState = cycle(connectionState); - if (thread == null) - oldConnectionsDrainer.checkOldConnections(); - if (thread != null) - sleepIfProblemsGettingSyncedConnection(connectionState, oldState); - } - - private void drainFirstDocumentsInQueueIfOld() { - while (true) { - Optional<Document> document = documentQueue.pollDocumentIfTimedoutInQueue(localQueueTimeOut); - if ( ! document.isPresent()) return; - - EndpointResult endpointResult = EndPointResultFactory.createTransientError( - endpoint, document.get().getOperationId(), - new Exception("Not sending document operation, timed out in queue after " + - (clock.millis() - document.get().getQueueInsertTime().toEpochMilli()) + " ms.")); - resultQueue.failOperation(endpointResult, clusterId); - } - } - - private void drainDocumentQueueWhenFailingPermanently(Exception exception) { - // first, clear sentOperations: - resultQueue.failPending(exception); - - for (Document document : documentQueue.removeAllDocuments()) { - EndpointResult endpointResult= - EndPointResultFactory.createError(endpoint, document.getOperationId(), exception); - resultQueue.failOperation(endpointResult, clusterId); - } - } - - private boolean isStale(GatewayConnection connection) { - return connection.connectionTime() != null - && connection.connectionTime().plus(connectionTimeToLive).isBefore(clock.instant()); - } - - private ConnectionState refreshConnection(ConnectionState currentConnectionState) { - if (currentConnectionState == ConnectionState.SESSION_SYNCED) - oldConnectionsDrainer.add(currentConnection); - currentConnection = connectionFactory.newConnection(); - return ConnectionState.DISCONNECTED; - } - - public static class ConnectionStats { - - // NOTE: These fields are accessed by reflection in JSON serialization - - public final int wrongSessionDetectedCounter; - public final int wrongVersionDetectedCounter; - public final int problemStatusCodeFromServerCounter; - public final int executeProblemsCounter; - public final int docsReceivedCounter; - public final int statusReceivedCounter; - public final int pendingDocumentStatusCount; - public final int successfullHandshakes; - public final int lastGatewayProcessTimeMillis; - - ConnectionStats(int wrongSessionDetectedCounter, - int wrongVersionDetectedCounter, - int problemStatusCodeFromServerCounter, - int executeProblemsCounter, - int docsReceivedCounter, - int statusReceivedCounter, - int pendingDocumentStatusCount, - int successfullHandshakes, - int lastGatewayProcessTimeMillis) { - this.wrongSessionDetectedCounter = wrongSessionDetectedCounter; - this.wrongVersionDetectedCounter = wrongVersionDetectedCounter; - this.problemStatusCodeFromServerCounter = problemStatusCodeFromServerCounter; - this.executeProblemsCounter = executeProblemsCounter; - this.docsReceivedCounter = docsReceivedCounter; - this.statusReceivedCounter = statusReceivedCounter; - this.pendingDocumentStatusCount = pendingDocumentStatusCount; - this.successfullHandshakes = successfullHandshakes; - this.lastGatewayProcessTimeMillis = lastGatewayProcessTimeMillis; - } - } - - /** For testing. Returns the current connection of this. Not thread safe. */ - public GatewayConnection currentConnection() { return currentConnection; } - - /** For testing. Returns a snapshot of the old connections of this. */ - public List<GatewayConnection> oldConnections() { return oldConnectionsDrainer.connections(); } - - /** For testing */ - public EndpointResultQueue resultQueue() { return resultQueue; } - - /** - * We need to drain results on the connection where they were sent to make sure we request results on - * the node which received the operation also when going through a VIP. - */ - private static class OldConnectionsDrainer implements Runnable { - - private static final Logger log = Logger.getLogger(OldConnectionsDrainer.class.getName()); - - private final Endpoint endpoint; - private final int clusterId; - private final Duration pollInterval; - private final Duration connectionTimeToLive; - private final Duration localQueueTimeOut; - private final AtomicInteger statusReceivedCounter; - private final EndpointResultQueue resultQueue; - private final CountDownLatch stopSignal; - private final Clock clock; - - /** - * Previous connections on which we may have sent operations and are still waiting for the results - * All connections in this are in state SESSION_SYNCED. - */ - private final List<GatewayConnection> connections = new CopyOnWriteArrayList<>(); - - OldConnectionsDrainer(Endpoint endpoint, - int clusterId, - Duration pollInterval, - Duration connectionTimeToLive, - Duration localQueueTimeOut, - AtomicInteger statusReceivedCounter, - EndpointResultQueue resultQueue, - CountDownLatch stopSignal, - Clock clock) { - this.endpoint = endpoint; - this.clusterId = clusterId; - this.pollInterval = pollInterval; - this.connectionTimeToLive = connectionTimeToLive; - this.localQueueTimeOut = localQueueTimeOut; - this.statusReceivedCounter = statusReceivedCounter; - this.resultQueue = resultQueue; - this.stopSignal = stopSignal; - this.clock = clock; - } - - /** Add another old connection to this for draining */ - public void add(GatewayConnection connection) { - connections.add(connection); - } - - @Override - public void run() { - while (stopSignal.getCount() > 0) { - try { - checkOldConnections(); - Thread.sleep(pollInterval.toMillis()); - } - catch (InterruptedException e) { - log.log(Level.WARNING, "Close thread was interrupted: " + e.getMessage(), e); - Thread.currentThread().interrupt(); - return; - } catch (Exception e) { - log.log(Level.WARNING, "Connection draining failed: " + e.getMessage(), e); - } - } - } - - public void checkOldConnections() { - for (GatewayConnection connection : connections) { - if (!resultQueue.hasInflightOperations(connection)) { - log.fine(() -> connection + " no longer has inflight operations"); - closeConnection(connection); - } else if (closingTime(connection).isBefore(clock.instant())) { - log.fine(() -> connection + " still has inflight operations, but drain period is over"); - tryPollAndDrainInflightOperations(connection); - closeConnection(connection); - } else if (timeToPoll(connection)) { - tryPollAndDrainInflightOperations(connection); - } - } - } - - private void closeConnection(GatewayConnection connection) { - log.fine(() -> "Closing " + connection); - connection.close(); - connections.remove(connection); // Safe as CopyOnWriteArrayList allows removal during iteration - } - - private void tryPollAndDrainInflightOperations(GatewayConnection connection) { - try { - log.fine(() -> "Polling and draining inflight operations for " + connection); - IOThread.processResponse(connection.poll(), endpoint, clusterId, statusReceivedCounter, resultQueue); - } catch (Exception e) { - // Old connection; best effort - log.log(Level.FINE, e, () -> "Polling status of inflight operations failed: " + e.getMessage()); - } - } - - private boolean timeToPoll(GatewayConnection connection) { - // connectionEndOfLife < connectionLastPolled < now - Instant now = clock.instant(); - Instant endOfLife = connection.connectionTime().plus(connectionTimeToLive); - if (connection.lastPollTime() == null) return endOfLife.plus(pollInterval).isBefore(now); - if (connection.lastPollTime().plus(pollInterval).isAfter(now)) return false; - - // Exponential (2^x) dropoff: - double connectionEndOfLife = endOfLife.toEpochMilli(); - double connectionLastPolled = connection.lastPollTime().toEpochMilli(); - return now.toEpochMilli() - connectionEndOfLife > 2 * (connectionLastPolled - connectionEndOfLife); - } - - private Instant closingTime(GatewayConnection connection) { - return connection.connectionTime().plus(connectionTimeToLive).plus(localQueueTimeOut); - } - - private void close() { - int size = resultQueue.getPendingSize(); - if (size > 0) { - log.info("We have outstanding operations (" + size + ") , trying to fetch responses."); - for (GatewayConnection connection : connections) { - try { - IOThread.processResponse(connection.poll(), endpoint, clusterId, statusReceivedCounter, resultQueue); - } catch (Throwable e) { - log.log(Level.SEVERE, "Some failures while trying to get latest responses from vespa.", e); - } - } - } - for (GatewayConnection oldConnection : connections) - oldConnection.close(); - } - - /** For testing. Returns the old connections of this. */ - public List<GatewayConnection> connections() { return Collections.unmodifiableList(connections); } - - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlocker.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlocker.java deleted file mode 100644 index 79a04d8f043..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlocker.java +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.operationProcessor; - -import java.util.concurrent.Semaphore; - -/** - * A semaphore that can be re-sized. - * - * @author dybis - */ -final public class ConcurrentDocumentOperationBlocker { - - private static final int INITIAL_SIZE = 0; - private final ReducableSemaphore semaphore = new ReducableSemaphore(); - private int maxConcurrency = INITIAL_SIZE; - private final Object monitor = new Object(); - - /* - * Resizes the semaphore. It does not wait for threads that are in the queue when downsizing. - */ - void setMaxConcurrency(int maxConcurrency) { - synchronized (monitor) { - int deltaConcurrency = maxConcurrency - this.maxConcurrency; - - if (deltaConcurrency > 0) { - semaphore.release(deltaConcurrency); - } - if (deltaConcurrency < 0) { - semaphore.reducePermits(-1 * deltaConcurrency); - } - this.maxConcurrency = maxConcurrency; - } - } - - /** - * Release a permit. - */ - void operationDone() { - semaphore.release(); - } - - /** - * Acquire a permit. Blocking if no permits available. - */ - void startOperation() throws InterruptedException { - semaphore.acquire(); - } - - int availablePermits() { - return semaphore.availablePermits(); - } - - /** - * We need to extend Semaphore to get access to protected reducePermit() method. - */ - @SuppressWarnings("serial") - private static final class ReducableSemaphore extends Semaphore { - - ReducableSemaphore() { - super(INITIAL_SIZE, true /* FIFO */); - } - - @Override - protected void reducePermits(int reduction) { - super.reducePermits(reduction); - } - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/DocumentSendInfo.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/DocumentSendInfo.java deleted file mode 100644 index b72a6c67398..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/DocumentSendInfo.java +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.operationProcessor; - -import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.core.Document; - -import java.time.Clock; -import java.util.HashMap; -import java.util.Map; - -/** - * Keeps an overview of what is sent and what is received for an operation. - * This class is not thread-safe. - */ -class DocumentSendInfo { - - private final Document document; - private final Map<Integer, Result.Detail> detailByClusterId = new HashMap<>(); - // This is lazily populated as normal cases does not require retries. - private Map<Integer, Integer> attemptedRetriesByClusterId = null; - private final StringBuilder localTrace; - private final Clock clock; - - DocumentSendInfo(Document document, boolean traceThisDoc, Clock clock) { - this.document = document; - localTrace = traceThisDoc ? new StringBuilder("\n" + document.createTime() + " Trace starting " + "\n") - : null; - this.clock = clock; - } - - boolean addIfNotAlreadyThere(Result.Detail detail, int clusterId) { - if (detailByClusterId.containsKey(clusterId)) { - if (localTrace != null) { - localTrace.append(clock.millis() + " Got duplicate detail, ignoring this: " + - detail.toString() + "\n"); - } - return false; - } - if (localTrace != null) { - localTrace.append(clock.millis() + " Got detail: " + detail.toString() + "\n"); - } - detailByClusterId.put(clusterId, detail); - return true; - } - - int detailCount() { - return detailByClusterId.size(); - } - - public Result createResult() { - return new Result(document, detailByClusterId.values(), localTrace); - } - - int incRetries(int clusterId, Result.Detail detail) { - if (attemptedRetriesByClusterId == null) { - attemptedRetriesByClusterId = new HashMap<>(); - } - int retries = 0; - if (attemptedRetriesByClusterId.containsKey(clusterId)) { - retries = attemptedRetriesByClusterId.get(clusterId); - } - retries++; - attemptedRetriesByClusterId.put(clusterId, retries); - if (localTrace != null) { - localTrace.append(clock.millis() + " Asked about retrying for cluster ID " - + clusterId + ", number of retries is " + retries + " Detail:\n" + detail.toString()); - } - return retries; - } - - Document getDocument() { - return document; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/EndPointResultFactory.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/EndPointResultFactory.java deleted file mode 100644 index 9af63f6637a..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/EndPointResultFactory.java +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.operationProcessor; - -import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.core.EndpointResult; -import com.yahoo.vespa.http.client.core.OperationStatus; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.logging.Logger; - -/** - * @author Einar M R Rosenvinge - */ -public final class EndPointResultFactory { - - private static final Logger log = Logger.getLogger(EndPointResultFactory.class.getName()); - private static final String EMPTY_MESSAGE = "-"; - - public static Collection<EndpointResult> createResult(Endpoint endpoint, - InputStream inputStream) throws IOException { - List<EndpointResult> results = new ArrayList<>(); - try (BufferedReader reader = new BufferedReader( - new InputStreamReader(inputStream, StandardCharsets.US_ASCII))) { - String line; - while ((line = reader.readLine()) != null) { - results.add(parseResult(line, endpoint)); - } - } - return results; - } - - public static EndpointResult createError(Endpoint endpoint, String operationId, Exception exception) { - return new EndpointResult(operationId, new Result.Detail(endpoint, - Result.ResultType.FATAL_ERROR, - null, - exception)); - } - - public static EndpointResult createTransientError(Endpoint endpoint, String operationId, Exception exception) { - return new EndpointResult(operationId, new Result.Detail(endpoint, - Result.ResultType.TRANSITIVE_ERROR, - null, - exception)); - } - - private static Result.ResultType replyToResultType(OperationStatus reply) { - // The ordering below is important, e.g. if success, it is never a transient error even if isTransient is true. - if (reply.errorCode.isSuccess()) - return Result.ResultType.OPERATION_EXECUTED; - if (reply.isConditionNotMet) - return Result.ResultType.CONDITION_NOT_MET; - if (reply.errorCode.isTransient()) - return Result.ResultType.TRANSITIVE_ERROR; - return Result.ResultType.FATAL_ERROR; - } - - private static EndpointResult parseResult(String line, Endpoint endpoint) { - try { - OperationStatus reply = OperationStatus.parse(line); - String message; - if (EMPTY_MESSAGE.equals(reply.message)) { - message = null; - } else { - message = reply.message; - } - Exception exception = null; - if (!reply.errorCode.isSuccess() && message != null) { - exception = new RuntimeException(message); - } - if (reply.traceMessage != null && !reply.traceMessage.isEmpty()) { - log.fine("Got trace message: " + reply.traceMessage); - } - return new EndpointResult( - reply.operationId, - new Result.Detail(endpoint, - replyToResultType(reply), - reply.traceMessage, - exception)); - } catch (Throwable t) { - throw new IllegalArgumentException("Bad result line from server: '" + line + "'", t); - } - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottler.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottler.java deleted file mode 100644 index a88d5af8094..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottler.java +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.operationProcessor; - -import com.yahoo.vespa.http.client.core.ThrottlePolicy; - -import java.time.Clock; -import java.util.concurrent.ThreadLocalRandom; - -/** - * Adjusts in-flight operations based on throughput. It will walk the graph and try to find - * local optimum. - * - * It looks at the throughput, adjust max in-flight based on the previous throughput and settings. - * - * In the beginning it moves faster, and then stabilizes. - * - * It will wait a bit after adjusting before it starts to sample, since there is a latency between adjustment - * and result. - * - * There are several mechanisms to reduce impact of several clients running in parallel. The window size has a - * random part, and the wait time before sampling after adjustment has a random part as well. - * - * To avoid running wild with large values of max-in flight, it is tuned to stay on the smaller part, and - * rather reduce max-in flight than to have a too large value. - * - * In case the where the queue is moved to minimum size, it will now and then increase queue size to get - * more sample data and possibly grow size. - * - * Class is fully thread safe, i.e. all public methods are thread safe. - * - * @author dybis - */ -public class IncompleteResultsThrottler { - - private final ConcurrentDocumentOperationBlocker blocker = new ConcurrentDocumentOperationBlocker(); - private final int maxInFlightValue; - private final int minInFlightValue; - private final ThrottlePolicy policy; - - // 9-11 seconds with some randomness to avoid fully synchronous feeders. - public final long phaseSizeMs = 9000 + (ThreadLocalRandom.current().nextInt() % 2000); - private final Clock clock; - - private final Object monitor = new Object(); - private long sampleStartTimeMs = 0; - private int previousNumOk = 0; - private int previousMaxInFlight = 0; - private int stabilizingPhasesLeft = 0; - private int adjustCycleCount = 0; - private int maxInFlightNow; - private int numOk = 0; - private int minWindowSizeCounter = 0; - private int minPermitsAvailable = 0; - - protected static int INITIAL_MAX_IN_FLIGHT_VALUE = 200; - protected static int SECOND_MAX_IN_FLIGHT_VALUE = 270; - private StringBuilder debugMessage = new StringBuilder(); - - /** - * Creates the throttler. - * - * @param minInFlightValue the throttler will never throttle beyond this limit. - * @param maxInFlightValue the throttler will never throttle above this limit. If zero, no limit. - * @param clock use to calculate window size. Can be null if minWindowSize and maxInFlightValue are equal. - * @param policy is the algorithm for finding next value of the number of in-flight documents operations. - */ - public IncompleteResultsThrottler(int minInFlightValue, int maxInFlightValue, Clock clock, ThrottlePolicy policy) { - this.maxInFlightValue = maxInFlightValue == 0 ? Integer.MAX_VALUE : maxInFlightValue; - this.minInFlightValue = minInFlightValue == 0 ? this.maxInFlightValue : minInFlightValue; - this.policy = policy; - this.clock = clock; - if (minInFlightValue != maxInFlightValue) { - this.sampleStartTimeMs = clock.millis(); - } - setNewSemaphoreSize(INITIAL_MAX_IN_FLIGHT_VALUE); - } - - public int availableCapacity() { - return blocker.availablePermits(); - } - - public void operationStart() { - try { - blocker.startOperation(); - } catch (InterruptedException e) { - // Ignore - } - if (maxInFlightValue != minInFlightValue) { - synchronized (monitor) { - adjustThrottling(); - } - } - } - - public String getDebugMessage() { - synchronized (monitor) { - return debugMessage.toString(); - } - } - - public void resultReady(boolean success) { - blocker.operationDone(); - if (!success) { - return; - } - synchronized (monitor) { - numOk++; - minPermitsAvailable = Math.min(minPermitsAvailable, blocker.availablePermits()); - } - } - - // Only for testing - protected int waitingThreads() { - synchronized (monitor) { - return maxInFlightNow - blocker.availablePermits(); - } - } - - private double getCeilingDifferencePerformance(int adjustCycle) { - // We want larger adjustments in the early phase. - if (adjustCycle > 10) { - return 0.7; - } - return 1.2; - } - - private void adjustCycle() { - adjustCycleCount++; - stabilizingPhasesLeft = adjustCycleCount < 5 ? 1 : 2 + ThreadLocalRandom.current().nextInt() % 2; - - double maxPerformanceChange = getCeilingDifferencePerformance(adjustCycleCount); - boolean messagesQueued = minPermitsAvailable < 2; - - int newMaxInFlight = policy.calcNewMaxInFlight( - maxPerformanceChange, numOk, previousNumOk, previousMaxInFlight, maxInFlightNow, messagesQueued); - debugMessage = new StringBuilder(); - debugMessage.append("previousMaxInFlight: " + previousMaxInFlight - + " maxInFlightNow: " + maxInFlightNow - + " numOk: " + numOk + " " + " previousOk: " + previousNumOk - + " new size is: " + newMaxInFlight); - previousMaxInFlight = maxInFlightNow; - previousNumOk = numOk; - - setNewSemaphoreSize(adjustCycleCount == 1 ? SECOND_MAX_IN_FLIGHT_VALUE : newMaxInFlight); - } - - private void adjustThrottling() { - if (clock.millis() < sampleStartTimeMs + phaseSizeMs) return; - - sampleStartTimeMs += phaseSizeMs; - - if (stabilizingPhasesLeft-- == 0) { - adjustCycle(); - } - numOk = 0; - this.minPermitsAvailable = maxInFlightNow; - } - - private int tryBoostingSizeIfMinValueOverSeveralCycles(final int size) { - if (size <= minInFlightValue) { - minWindowSizeCounter++; - } else { - minWindowSizeCounter = 0; - } - if (minWindowSizeCounter == 4) { - debugMessage.append(" (inc max in flight to get more data)"); - minWindowSizeCounter = 0; - return size + 10; - } - return size; - - } - - private void setNewSemaphoreSize(final int size) { - maxInFlightNow = - Math.max(minInFlightValue, Math.min( - tryBoostingSizeIfMinValueOverSeveralCycles(size), maxInFlightValue)); - blocker.setMaxConcurrency(maxInFlightNow); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java deleted file mode 100644 index 5d0e063efdf..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java +++ /dev/null @@ -1,307 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.operationProcessor; - -import com.google.common.collect.ArrayListMultimap; -import com.yahoo.vespa.http.client.FeedClient; -import com.yahoo.vespa.http.client.FeedEndpointException; -import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.communication.EndpointIOException; -import com.yahoo.vespa.http.client.core.EndpointResult; -import com.yahoo.vespa.http.client.core.Exceptions; -import com.yahoo.vespa.http.client.core.communication.ClusterConnection; - -import java.math.BigInteger; -import java.security.SecureRandom; -import java.time.Clock; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Merges several endpointResult into one Result and does the callback. - * - * @author dybis - */ -public class OperationProcessor { - - private static final Logger log = Logger.getLogger(OperationProcessor.class.getName()); - private final Map<String, DocumentSendInfo> docSendInfoByOperationId = new LinkedHashMap<>(); - private final ArrayListMultimap<String, Document> blockedDocumentsByDocumentId = ArrayListMultimap.create(); - private final Set<String> inflightDocumentIds = new HashSet<>(); - private final int numDestinations; - private final FeedClient.ResultCallback resultCallback; - private final Object monitor = new Object(); - private final IncompleteResultsThrottler incompleteResultsThrottler; - // Position in the array is cluster ID. - private final List<ClusterConnection> clusters = new ArrayList<>(); - private final ScheduledThreadPoolExecutor timeoutExecutor; - private final OperationStats operationStats; - private final int maxRetries; - private final long minTimeBetweenRetriesMs; - private final Random random = new SecureRandom(); - private final int traceEveryXOperation; - private int traceCounter = 0; - private final boolean traceToStderr; - private final ThreadGroup ioThreadGroup; - private final String clientId = new BigInteger(130, random).toString(32); - private final Clock clock; - - public OperationProcessor(IncompleteResultsThrottler incompleteResultsThrottler, - FeedClient.ResultCallback resultCallback, - SessionParams sessionParams, - ScheduledThreadPoolExecutor timeoutExecutor, - Clock clock) { - this.numDestinations = sessionParams.getClusters().size(); - this.resultCallback = resultCallback; - this.incompleteResultsThrottler = incompleteResultsThrottler; - this.timeoutExecutor = timeoutExecutor; - this.ioThreadGroup = new ThreadGroup("operationprocessor"); - this.clock = clock; - - if (sessionParams.getClusters().isEmpty()) - throw new IllegalArgumentException("Cannot feed to 0 clusters."); - - for (Cluster cluster : sessionParams.getClusters()) { - if (cluster.getEndpoints().isEmpty()) - throw new IllegalArgumentException("Cannot feed to empty cluster."); - } - - for (int i = 0; i < sessionParams.getClusters().size(); i++) { - Cluster cluster = sessionParams.getClusters().get(i); - clusters.add(new ClusterConnection(this, - sessionParams.getFeedParams(), - sessionParams.getConnectionParams(), - cluster, - i, - sessionParams.getClientQueueSize() / sessionParams.getClusters().size(), - timeoutExecutor, - clock)); - } - operationStats = new OperationStats(sessionParams, clusters, incompleteResultsThrottler); - maxRetries = sessionParams.getConnectionParams().getMaxRetries(); - minTimeBetweenRetriesMs = sessionParams.getConnectionParams().getMinTimeBetweenRetriesMs(); - traceEveryXOperation = sessionParams.getConnectionParams().getTraceEveryXOperation(); - traceToStderr = sessionParams.getConnectionParams().getPrintTraceToStdErr(); - } - - public ThreadGroup getIoThreadGroup() { - return ioThreadGroup; - } - - public int getIncompleteResultQueueSize() { - synchronized (monitor) { - return docSendInfoByOperationId.size(); - } - } - - /** Returns the id of the oldest operation to be sent. */ - public Optional<String> oldestIncompleteResultId() { - synchronized (monitor) { - return docSendInfoByOperationId.isEmpty() - ? Optional.empty() - : Optional.of(docSendInfoByOperationId.keySet().iterator().next()); - } - } - - public String getClientId() { - return clientId; - } - - private boolean retriedThis(EndpointResult endpointResult, DocumentSendInfo documentSendInfo, int clusterId) { - Result.Detail detail = endpointResult.getDetail(); - if (detail.getResultType() == Result.ResultType.OPERATION_EXECUTED) return false; // Success: No retries - - int retries = documentSendInfo.incRetries(clusterId, detail); - if (retries > maxRetries) return false; - - String exceptionMessage = detail.getException() == null ? "" : detail.getException().getMessage(); - if (exceptionMessage == null) - exceptionMessage = ""; - - // TODO: Return proper error code in structured data in next version of internal API. - // Error codes from messagebus/src/cpp/messagebus/errorcode.h - boolean retryThisOperation = - detail.getResultType() == Result.ResultType.TRANSITIVE_ERROR || - exceptionMessage.contains("SEND_QUEUE_CLOSED") || - exceptionMessage.contains("ILLEGAL_ROUTE") || - exceptionMessage.contains("NO_SERVICES_FOR_ROUTE") || - exceptionMessage.contains("NETWORK_ERROR") || - exceptionMessage.contains("SEQUENCE_ERROR") || - exceptionMessage.contains("NETWORK_SHUTDOWN") || - exceptionMessage.contains("TIMEOUT"); - - if (retryThisOperation) { - int waitTime = (int) (minTimeBetweenRetriesMs * (1 + random.nextDouble() / 3)); - log.finest("Retrying due to " + detail + " attempt " + retries + " in " + waitTime + " ms."); - timeoutExecutor.schedule(() -> postToCluster(clusters.get(clusterId), documentSendInfo.getDocument()), - waitTime, - TimeUnit.MILLISECONDS); - return true; - } - - return false; - } - - private Result process(EndpointResult endpointResult, int clusterId) { - Result result; - Document blockedDocumentToSend = null; - synchronized (monitor) { - if (!docSendInfoByOperationId.containsKey(endpointResult.getOperationId())) { - log.finer("Received out-of-order or too late result, discarding: " + endpointResult); - return null; - } - DocumentSendInfo documentSendInfo = docSendInfoByOperationId.get(endpointResult.getOperationId()); - - if (retriedThis(endpointResult, documentSendInfo, clusterId)) return null; - - // Duplicate message - if ( ! documentSendInfo.addIfNotAlreadyThere(endpointResult.getDetail(), clusterId)) return null; - - // Is this the last operation we are waiting for? - if (documentSendInfo.detailCount() != numDestinations) return null; - - result = documentSendInfo.createResult(); - docSendInfoByOperationId.remove(endpointResult.getOperationId()); - - String documentId = documentSendInfo.getDocument().getDocumentId(); - // If we got a pending operation against this document - // dont't remove it from inflightDocuments and send blocked document operation - List<Document> blockedDocuments = blockedDocumentsByDocumentId.get(documentId); - if (blockedDocuments.isEmpty()) { - inflightDocumentIds.remove(documentId); - } else { - blockedDocumentToSend = blockedDocuments.remove(0); - } - } - if (blockedDocumentToSend != null) { - sendToClusters(blockedDocumentToSend, clock); - } - return result; - } - - public void resultReceived(EndpointResult endpointResult, int clusterId) { - Result result = process(endpointResult, clusterId); - if (result != null) { - incompleteResultsThrottler.resultReady(result.isSuccess()); - resultCallback.onCompletion(result.getDocumentId(), result); - if (traceToStderr && result.hasLocalTrace()) { - System.err.println(result.toString()); - } - } - } - - public void onEndpointError(FeedEndpointException e) { - resultCallback.onEndpointException(e); - } - - public List<Exception> closeClusters() { - List<Exception> exceptions = new ArrayList<>(); - // first, close cluster sessions and allow connections to drain normally - for (ClusterConnection cluster : clusters) { - try { - cluster.close(); - } catch (Exception e) { - exceptions.add(e); - } - } - return exceptions; - } - - public void sendDocument(Document document) { - incompleteResultsThrottler.operationStart(); - - synchronized (monitor) { - if (inflightDocumentIds.contains(document.getDocumentId())) { - blockedDocumentsByDocumentId.put(document.getDocumentId(), document); - return; - } - inflightDocumentIds.add(document.getDocumentId()); - } - - sendToClusters(document, clock); - } - - private void sendToClusters(Document document, Clock clock) { - synchronized (monitor) { - boolean traceThisDoc = traceEveryXOperation > 0 && traceCounter++ % traceEveryXOperation == 0; - docSendInfoByOperationId.put(document.getOperationId(), new DocumentSendInfo(document, traceThisDoc, clock)); - } - - for (ClusterConnection clusterConnection : clusters) { - postToCluster(clusterConnection, document); - } - } - - private void postToCluster(ClusterConnection clusterConnection, Document document) { - try { - clusterConnection.post(document); - } catch (EndpointIOException eio) { - resultReceived(EndPointResultFactory.createError(eio.getEndpoint(), - document.getOperationId(), - eio), - clusterConnection.getClusterId()); - } - } - - public List<ClusterConnection> clusters() { return Collections.unmodifiableList(clusters); } - - public String getStatsAsJson() { - return operationStats.getStatsAsJson(); - } - - public void close() { - List<Exception> exceptions = closeClusters(); - try { - closeExecutor(); - } catch (InterruptedException e) { - exceptions.add(e); - } - - if (exceptions.isEmpty()) { - return; - } - if (exceptions.size() == 1) { - if (exceptions.get(0) instanceof RuntimeException) { - throw (RuntimeException) exceptions.get(0); - } else { - throw new RuntimeException(exceptions.get(0)); - } - } - - StringBuilder b = new StringBuilder(); - b.append("Exception thrown while closing one or more clusters: "); - for (int i = 0; i < exceptions.size(); i++) { - Exception e = exceptions.get(i); - b.append(Exceptions.toMessageString(e)); - if (i != (exceptions.size() - 1)) { - b.append(", "); - } - } - throw new RuntimeException(b.toString(), exceptions.get(0)); - } - - private void closeExecutor() throws InterruptedException { - log.log(Level.FINE, "Shutting down timeout executor."); - timeoutExecutor.shutdownNow(); - - log.log(Level.FINE, "Awaiting termination of already running timeout tasks."); - if (! timeoutExecutor.awaitTermination(300, TimeUnit.SECONDS)) { - log.severe("Did not manage to shut down the executors within 300 secs, system stuck?"); - throw new RuntimeException("Did not manage to shut down retry threads. Please report problem."); - } - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationStats.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationStats.java deleted file mode 100644 index 1eebe593062..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationStats.java +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.operationProcessor; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.core.communication.ClusterConnection; - -import java.io.IOException; -import java.io.StringWriter; -import java.util.List; - -public class OperationStats { - - private static ObjectMapper jsonMapper = createMapper(); - - private final String sessionParamsAsXmlString; - private List<ClusterConnection> clusters; - private IncompleteResultsThrottler throttler; - - public OperationStats( - SessionParams sessionParams, - List<ClusterConnection> clusters, - IncompleteResultsThrottler throttler) { - this.sessionParamsAsXmlString = generateSessionParamsAsXmlString(sessionParams); - this.clusters = clusters; - this.throttler = throttler; - } - - private static ObjectMapper createMapper() { - ObjectMapper mapper = new ObjectMapper(); - mapper.registerModule(new Jdk8Module()); - mapper.registerModule(new JavaTimeModule()); - return mapper; - } - - private String generateSessionParamsAsXmlString(final SessionParams sessionParams) { - StringWriter stringWriter = new StringWriter(); - try { - JsonGenerator jsonGenerator = jsonMapper.createGenerator(stringWriter); - jsonMapper.writeValue(jsonGenerator, sessionParams); // TODO SessionParams should not be blindly serialized. This may serialize objects that are not really serializable. - return stringWriter.toString(); - } catch (IOException e) { - return e.getMessage(); - } - } - - public String getStatsAsJson() { - try { - StringWriter stringWriter = new StringWriter(); - JsonGenerator jsonGenerator = jsonMapper.createGenerator(stringWriter); - jsonGenerator.writeStartObject(); - jsonGenerator.writeArrayFieldStart("clusters"); - for (ClusterConnection cluster : clusters) { - jsonGenerator.writeStartObject(); - jsonGenerator.writeNumberField("clusterid", cluster.getClusterId()); - jsonGenerator.writeFieldName("stats"); - jsonGenerator.writeRawValue(cluster.getStatsAsJSon()); - jsonGenerator.writeEndObject(); - } - jsonGenerator.writeEndArray(); - jsonGenerator.writeFieldName("sessionParams"); - jsonGenerator.writeRawValue(sessionParamsAsXmlString); - jsonGenerator.writeFieldName("throttleDebugMessage"); - jsonGenerator.writeRawValue("\"" + throttler.getDebugMessage() + "\""); - jsonGenerator.writeEndObject(); - jsonGenerator.close(); - return stringWriter.toString(); - } catch (IOException e) { - return "{ \"Error\" : \""+ e.getMessage() + "\"}"; - } - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/package-info.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/package-info.java deleted file mode 100644 index 13d4e768daf..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/package-info.java +++ /dev/null @@ -1,2 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/package-info.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/package-info.java deleted file mode 100644 index 958d3793875..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/package-info.java +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Programmatic API for feeding to Vespa clusters independently of the - * cluster configuration. - * - * NOTE: This is a PUBLIC API, but not annotated as such because this is not a bundle and - * we don't want to introduce Vespa dependencies. - */ -package com.yahoo.vespa.http.client; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java deleted file mode 100644 index 7ccdf3ebd43..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java +++ /dev/null @@ -1,323 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.runner; - -import com.google.common.base.Splitter; -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.config.SessionParams; -import io.airlift.command.Command; -import io.airlift.command.HelpOption; -import io.airlift.command.Option; -import io.airlift.command.SingleCommand; -import org.apache.http.Header; -import org.apache.http.ParseException; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.message.BasicLineParser; - -import javax.inject.Inject; -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.TimeUnit; - -/** - * Commandline interface for the binary. - * - * @author dybis - */ -@Command(name = "vespa-http-client", - description = "This is a tool for feeding xml or json data to a Vespa application.") -public class CommandLineArguments { - - /** - * Creates a CommandLineArguments instance and populates it with data. - * - * @param args array of arguments. - * @return null on failure or if help option is set to true. - */ - static CommandLineArguments build(String[] args) { - CommandLineArguments cmdArgs; - try { - cmdArgs = SingleCommand.singleCommand(CommandLineArguments.class).parse(args); - } catch (Exception e) { - System.err.println(e.getMessage()); - System.err.println("Use --help to show usage.\n"); - return null; - } - if (cmdArgs.helpOption.showHelpIfRequested()) { - return null; - } - if (cmdArgs.endpointArg != null) { - if (cmdArgs.hostArg != null) { - System.err.println("Cannot set both '--host' and '--endpoint' "); - return null; - } - try { - URL url = new URL(cmdArgs.endpointArg); - } catch (MalformedURLException e) { - e.printStackTrace(System.err); - return null; - } - } else { - if (cmdArgs.hostArg == null) { - System.err.println("'--host' or '--endpoint' not set."); - return null; - } - } - if (cmdArgs.priorityArg != null && ! checkPriorityFlag(cmdArgs.priorityArg)) { - return null; - } - - for (String header : cmdArgs.headers) { - try { - cmdArgs.parsedHeaders.add(BasicLineParser.parseHeader(header, null)); - } catch (ParseException e) { - System.err.printf("Invalid header: '%s' (%s)%n", header, e.getMessage()); - return null; - } - } - - if (cmdArgs.privateKeyPath == null && cmdArgs.certificatePath != null || - cmdArgs.privateKeyPath != null && cmdArgs.certificatePath == null) { - System.err.println("Both '--privateKey' and '--certificate' must be set"); - return null; - } - - return cmdArgs; - } - - private static boolean checkPriorityFlag(String priorityArg) { - switch (priorityArg) { - case "HIGHEST": - case "VERY_HIGH": - case "HIGH_1": - case "HIGH_2": - case "HIGH_3": - case "NORMAL_1": - case "NORMAL_2": - case "NORMAL_3": - case "NORMAL_4": - case "NORMAL_5": - case "NORMAL_6": - case "LOW_1": - case "LOW_2": - case "LOW_3": - case "VERY_LOW": - case "LOWEST": - return true; - default: - System.err.println("Not valid value for priority. Allowed values are HIGHEST, VERY_HIGH, HIGH_[1-3], " + - "NORMAL_[1-6], LOW_[1-3], VERY_LOW, and LOWEST."); - return false; - } - } - - // TODO Don't duplicate default values from ConnectionParams.Builder. Some defaults are already inconsistent. - - @Inject - private HelpOption helpOption; - - @Option(name = {"--useV3Protocol"}, description = "Use V3 protocol to gateway. This is the default protocol.") - private boolean enableV3Protocol = true; - - @Option(name = {"--file"}, - description = "The name of the input file to read.") - private String fileArg = null; - - @Option(name = {"--add-root-element-to-xml"}, - description = "Add <vespafeed> tag to XML document, makes it easier to feed raw data.") - private boolean addRootElementToXml = false; - - @Option(name = {"--route"}, - description = "(=default)The route to send the data to.") - private String routeArg = "default"; - - @Option(name = {"--endpoint"}, - description = "Vespa endpoint.") - private String endpointArg; - - @Option(name = {"--host"}, - description = "The host(s) for the gateway. If using several, use comma to separate them.") - private String hostArg; - - @Option(name = {"--port"}, - description = "The port for the host of the gateway.") - private int portArg = 4080; - - @Option(name = {"--timeout"}, - description = "(=180) The time (in seconds) allowed for sending operations.") - private long timeoutArg = 180; - - @Option(name = {"--useCompression"}, - description = "Use compression over network.") - private boolean useCompressionArg = false; - - @Option(name = {"--useDynamicThrottling"}, - description = "Try to maximize throughput by using dynamic throttling.") - private boolean useDynamicThrottlingArg = false; - - @Option(name = {"--maxpending"}, - description = "The maximum number of operations that are allowed " + - "to be pending at any given time.") - private int maxPendingOperationCountArg = 10000; - - @Option(name = {"-v", "--verbose"}, - description = "Enable verbose output of progress.") - private boolean verboseArg = false; - - @Option(name = {"--noretry"}, - description = "Turns off retries of recoverable failures..") - private boolean noRetryArg = false; - - @Option(name = {"--retrydelay"}, - description = "The time (in seconds) to wait between retries of a failed operation.") - private int retrydelayArg = 1; - - @Option(name = {"--trace"}, - description = "(=0 (=off)) The trace level of network traffic.") - private int traceArg = 0; - - @Option(name = {"--printTraceEveryXOperation"}, - description = "(=1) How often to to tracing.") - private int traceEveryXOperation = 1; - - @Option(name = {"--validate"}, - description = "Run validation tool on input files instead of feeding them.") - private boolean validateArg = false; - - @Option(name = {"--ignoreConditionNotMet"}, - description = "Ignore condition not met failures.") - private boolean ignoreConditionNotMet = false; - - @Option(name = {"--priority"}, - description = "Specify priority of sent messages, see documentation ") - private String priorityArg = null; - - @Option(name = {"--numPersistentConnectionsPerEndpoint"}, - description = "How many tcp connections to establish per endoint.)") - private int numPersistentConnectionsPerEndpoint = 4; - - @Option(name = {"--maxChunkSizeBytes"}, - description = "How much data to send to gateway in each message.") - private int maxChunkSizeBytes = 20 * 1024; - - @Option(name = {"--whenVerboseEnabledPrintMessageForEveryXDocuments"}, - description = "How often to print verbose message.)") - private int whenVerboseEnabledPrintMessageForEveryXDocuments = 1000; - - @Option(name = {"--useTls"}, - description = "Use TLS when connecting to endpoint") - private boolean useTls = false; - - @Option(name = {"--insecure", "--disable-hostname-verification"}, - description = "Skip hostname verification when using TLS") - private boolean insecure = false; - - @Option(name = {"--header"}, - description = "Add http header to every request. Header must have the format '<Name>: <Value>'. Use this parameter multiple times for multiple headers") - private List<String> headers = new ArrayList<>(); - - @Option(name = {"--vespaTls"}, - description = "BETA! Use Vespa TLS configuration from environment if available. Other HTTPS/TLS configuration will be ignored if this is set.") - private boolean useTlsConfigFromEnvironment = false; - - @Option(name = {"--connectionTimeToLive"}, - description = "Maximum time to live for persistent connections. Specified as integer, in seconds.") - private long connectionTimeToLive = 15; - - @Option(name = {"--certificate"}, - description = "Path to a file containing a PEM encoded x509 certificate") - private String certificatePath; - - @Option(name = {"--privateKey"}, - description = "Path to a file containing a PEM encoded private key") - private String privateKeyPath; - - @Option(name = "--caCertificates", - description = "Path to a file containing a PEM encoded CA certificates") - private String caCertificatesPath; - - private final List<Header> parsedHeaders = new ArrayList<>(); - - int getWhenVerboseEnabledPrintMessageForEveryXDocuments() { - return whenVerboseEnabledPrintMessageForEveryXDocuments; - } - - public String getFile() { return fileArg; }; - - public boolean getVerbose() { return verboseArg; } - - public boolean getIgnoreConditionNotMet() { return ignoreConditionNotMet; } - - public boolean getAddRootElementToXml() { return addRootElementToXml; } - - SessionParams createSessionParams(boolean useJson) { - int minThrottleValue = useDynamicThrottlingArg ? 10 : 0; - Path privateKeyPath = Optional.ofNullable(this.privateKeyPath).map(Paths::get).orElse(null); - Path certificatePath = Optional.ofNullable(this.certificatePath).map(Paths::get).orElse(null); - Path caCertificatesPath = Optional.ofNullable(this.caCertificatesPath).map(Paths::get).orElse(null); - ConnectionParams.Builder connectionParamsBuilder = new ConnectionParams.Builder(); - parsedHeaders.forEach(header -> connectionParamsBuilder.addHeader(header.getName(), header.getValue())); - SessionParams.Builder builder = new SessionParams.Builder() - .setFeedParams( - new FeedParams.Builder() - .setDataFormat(useJson - ? FeedParams.DataFormat.JSON_UTF8 - : FeedParams.DataFormat.XML_UTF8) - .setRoute(routeArg) - .setMaxInFlightRequests(maxPendingOperationCountArg) - .setClientTimeout(timeoutArg, TimeUnit.SECONDS) - .setServerTimeout(timeoutArg, TimeUnit.SECONDS) - .setLocalQueueTimeOut(timeoutArg * 1000) - .setPriority(priorityArg) - .setMaxChunkSizeBytes(maxChunkSizeBytes) - .build() - ) - .setConnectionParams( - connectionParamsBuilder - .setHostnameVerifier(insecure ? NoopHostnameVerifier.INSTANCE : - SSLConnectionSocketFactory.getDefaultHostnameVerifier()) - .setUseCompression(useCompressionArg) - .setMaxRetries(noRetryArg ? 0 : 100) - .setMinTimeBetweenRetries(retrydelayArg, TimeUnit.SECONDS) - .setDryRun(validateArg) - .setTraceLevel(traceArg) - .setTraceEveryXOperation(traceEveryXOperation) - .setPrintTraceToStdErr(traceArg > 0) - .setNumPersistentConnectionsPerEndpoint(numPersistentConnectionsPerEndpoint) - .setCertificateAndPrivateKey(privateKeyPath, certificatePath) - .setCaCertificates(caCertificatesPath) - .setUseTlsConfigFromEnvironment(useTlsConfigFromEnvironment) - .setConnectionTimeToLive(Duration.ofSeconds(connectionTimeToLive)) - .build() - ) - // Enable dynamic throttling. - .setThrottlerMinSize(minThrottleValue) - .setClientQueueSize(maxPendingOperationCountArg); - if (endpointArg != null) { - try { - builder.addCluster(new Cluster.Builder() - .addEndpoint(Endpoint.create(new URL(endpointArg))) - .build()); - } - catch (MalformedURLException e) {} // already checked when parsing arguments - } - else { - Iterable<String> hosts = Splitter.on(',').trimResults().split(hostArg); - for (String host : hosts) { - builder.addCluster(new Cluster.Builder().addEndpoint(Endpoint.create(host, portArg, useTls)) - .build()); - } - } - return builder.build(); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/FormatInputStream.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/FormatInputStream.java deleted file mode 100644 index dd6fbc29e5f..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/FormatInputStream.java +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.runner; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.format.DataFormatDetector; -import com.fasterxml.jackson.core.format.DataFormatMatcher; -import com.fasterxml.jackson.core.format.MatchStrength; -import com.fasterxml.jackson.dataformat.xml.XmlFactory; - -import java.io.BufferedInputStream; -import java.io.ByteArrayInputStream; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.SequenceInputStream; -import java.util.Arrays; -import java.util.Collections; -import java.util.Optional; - -/** - * @author valerijf - */ -public class FormatInputStream { - - private InputStream inputStream; - private final Format format; - - /** - * Creates a single data input stream from either file or InputStream depending on which one is present. Preference - * for file if both present. Additionally also detects input data format of the result stream, throws - * IllegalArgumentException if unable to determine data format. - * - * @param stream InputStream of the data if present - * @param inputFile path to file to use as input - * @param addRootElementToXml to add vespafeed root element around the input data stream - * @throws IOException on errors - */ - public FormatInputStream(InputStream stream, Optional<String> inputFile, boolean addRootElementToXml) - throws IOException { - DataFormatDetector dataFormatDetector = new DataFormatDetector(new JsonFactory(), new XmlFactory()); - DataFormatMatcher formatMatcher; - - if (inputFile.isPresent()) { - try (FileInputStream fileInputStream = new FileInputStream(inputFile.get())) { - formatMatcher = dataFormatDetector.findFormat(fileInputStream); - } - inputStream = new FileInputStream(inputFile.get()); - - } else { - if (stream.available() == 0) - System.out.println("No data in stream yet and no file specified, waiting for data."); - - inputStream = stream.markSupported() ? stream : new BufferedInputStream(stream); - inputStream.mark(DataFormatDetector.DEFAULT_MAX_INPUT_LOOKAHEAD); - formatMatcher = dataFormatDetector.findFormat(inputStream); - inputStream.reset(); - } - - if (addRootElementToXml) { - inputStream = addVespafeedTag(inputStream); - format = Format.XML; - return; - } - - if (formatMatcher.getMatchStrength() == MatchStrength.INCONCLUSIVE - || formatMatcher.getMatchStrength() == MatchStrength.NO_MATCH) { - throw new IllegalArgumentException("Could not detect input format"); - } - - switch (formatMatcher.getMatchedFormatName().toLowerCase()) { - case "json": - format = Format.JSON; - break; - case "xml": - format = Format.XML; - break; - default: - throw new IllegalArgumentException("Unknown data format"); - } - } - - private static InputStream addVespafeedTag(InputStream inputStream) { - return new SequenceInputStream(Collections.enumeration(Arrays.asList( - new ByteArrayInputStream("<vespafeed>".getBytes()), inputStream, - new ByteArrayInputStream("</vespafeed>".getBytes()))) - ); - } - - public InputStream getInputStream() { - return inputStream; - } - - public Format getFormat() { - return format; - } - - public enum Format { - JSON, XML - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java deleted file mode 100644 index e3e90c8bbfc..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.runner; - -import com.yahoo.vespa.http.client.FeedClient; -import com.yahoo.vespa.http.client.FeedClientFactory; -import com.yahoo.vespa.http.client.SimpleLoggerResultCallback; -import com.yahoo.vespa.http.client.core.JsonReader; -import com.yahoo.vespa.http.client.core.XmlFeedReader; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.time.Clock; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * @author Einar M R Rosenvinge - * @author dybis - */ -public class Runner { - - /** - * Feed data from inputFile to session. - * - * @param feedClient where to send data to - * @param inputStream source of data - * @param isJson if input stream is of json formatted data - * @param numSent is updated while sending by this method - * @param verbose if true will print some information to stderr - * @return send time in ms, not including validating - */ - public static long send(FeedClient feedClient, - InputStream inputStream, - boolean isJson, - AtomicInteger numSent, - boolean verbose) { - Clock clock = Clock.systemUTC(); - if (verbose) - System.err.println("Now sending data."); - - long sendStartTime = clock.millis(); - if (isJson) { - JsonReader.read(inputStream, feedClient, numSent); - } else { - try { - XmlFeedReader.read(inputStream, feedClient, numSent); - } catch (Exception e) { - System.err.println("Stopped reading feed, got problems with XML: " + e.getMessage()); - } - } - - long sendTotalTime = clock.millis() - sendStartTime; - - if (verbose) - System.err.println("Waiting for all results, sent " + numSent.get() + " docs."); - - feedClient.close(); - if (verbose) - System.err.println("Session closed."); - return sendTotalTime; - } - - - public static void main(String[] args) throws IOException { - CommandLineArguments commandLineArgs = CommandLineArguments.build(args); - if (commandLineArgs == null) - System.exit(1); - - FormatInputStream formatInputStream = new FormatInputStream(System.in, - Optional.ofNullable(commandLineArgs.getFile()), - commandLineArgs.getAddRootElementToXml()); - - int intervalOfLogging = - commandLineArgs.getVerbose() - ? commandLineArgs.getWhenVerboseEnabledPrintMessageForEveryXDocuments() - : Integer.MAX_VALUE; - AtomicInteger numSent = new AtomicInteger(0); - SimpleLoggerResultCallback callback = new SimpleLoggerResultCallback(numSent, intervalOfLogging, commandLineArgs.getIgnoreConditionNotMet()); - - FeedClient feedClient = FeedClientFactory.create(commandLineArgs.createSessionParams(formatInputStream.getFormat()== FormatInputStream.Format.JSON), - callback); - - long sendTotalTimeMs = send(feedClient, - formatInputStream.getInputStream(), - formatInputStream.getFormat() == FormatInputStream.Format.JSON, - numSent, - commandLineArgs.getVerbose()); - - if (commandLineArgs.getVerbose()) { - System.err.println(feedClient.getStatsAsJson()); - double transferTimeSec = ((double) sendTotalTimeMs) / 1000.0; - if (transferTimeSec > 0) - System.err.printf("Docs/sec %.3f%n", numSent.get() / transferTimeSec); - - if (commandLineArgs.getFile() != null) { - double fileSizeMb = ((double) new File(commandLineArgs.getFile()).length()) / 1024.0 / 1024.0; - System.err.println("Sent " + fileSizeMb + " MB in " + transferTimeSec + " seconds."); - System.err.println("Speed: " + ((fileSizeMb / transferTimeSec) * 8.0) + " Mbits/sec, + HTTP overhead " + - "(not taking compression into account)"); - } - } - callback.printProgress(); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/package-info.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/package-info.java deleted file mode 100644 index 8be767d67cf..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/package-info.java +++ /dev/null @@ -1,2 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.runner; diff --git a/vespa-http-client/src/test/java/ExampleUsageFeedClientTest.java b/vespa-http-client/src/test/java/ExampleUsageFeedClientTest.java deleted file mode 100644 index 2ca2ea4d14a..00000000000 --- a/vespa-http-client/src/test/java/ExampleUsageFeedClientTest.java +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -import com.yahoo.vespa.http.client.FeedClient; -import com.yahoo.vespa.http.client.FeedClientFactory; -import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.Server; -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.handlers.V3MockParsingRequestHandler; -import org.junit.Test; - -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Unit test that test documentation code. - * - * @author dybis - */ -public class ExampleUsageFeedClientTest { - - @Test - public void testExampleCode() { - Server serverA = - new Server(new V3MockParsingRequestHandler(200, V3MockParsingRequestHandler.Scenario.ALL_OK), 0); - Server serverB = - new Server(new V3MockParsingRequestHandler(200, V3MockParsingRequestHandler.Scenario.ALL_OK), 0); - - exampleCode("localhost", serverA.getPort(), "localhost", serverB.getPort()); - serverA.close(); - serverB.close(); - } - - private static CharSequence generateDocument(String docId) { - // Just a dummy example of an update document operation. - return "{\"update\": \""+ docId + "\"," - + " \"fields\": { \"actualMapStringToArrayOfInt\": {" - + " \"assign\": [" - + "{ \"key\": \"fooKey\", \"value\": [ 2,1, 3] }" - + "]}}}"; - } - - // Example usage of FeedClient - public static void exampleCode(String hostNameA, int portServerA, String hostNameB, int portServerB) { - boolean useSsl = false; - final SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create(hostNameA, portServerA, useSsl)).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create(hostNameB, portServerB, useSsl)).build()) - .setFeedParams(new FeedParams.Builder() - .setDataFormat(FeedParams.DataFormat.JSON_UTF8) - .build()) - .build(); - - AtomicInteger resultsReceived = new AtomicInteger(0); - AtomicInteger errorsReceived = new AtomicInteger(0); - - FeedClient feedClient = FeedClientFactory.create(sessionParams, new FeedClient.ResultCallback() { - @Override - public void onCompletion(String docId, Result documentResult) { - resultsReceived.incrementAndGet(); - if (! documentResult.getContext().equals(docId)) { - System.err.println("Context does not work as expected."); - errorsReceived.incrementAndGet(); - } - if (!documentResult.isSuccess()) { - System.err.println("Problems with docID " + docId + ":" + documentResult.toString()); - errorsReceived.incrementAndGet(); - } - } - }); - int sentCounter = 0; - List<String> docIds = Arrays.asList("1", "2", "3", "4"); - for (final String docId : docIds) { - CharSequence docData = generateDocument(docId); - feedClient.stream(docId, docData, docId); - sentCounter++; - } - feedClient.close(); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/FeedClientTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/FeedClientTest.java deleted file mode 100644 index ca956110a34..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/FeedClientTest.java +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.core.api.FeedClientImpl; -import org.junit.Test; - -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.time.Clock; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Tests for the API, using dryrun option to mock gateway. - * - * @author dybis - */ -public class FeedClientTest { - - private final static String DOCID = "doc_id"; - - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder() - .addEndpoint(Endpoint.create("localhost")) - .build()) - .setConnectionParams(new ConnectionParams.Builder() - .setDryRun(true) - .build()) - .build(); - final AtomicInteger resultsReceived = new AtomicInteger(0); - - FeedClient.ResultCallback resultCallback = (docId, documentResult) -> { - assertTrue(documentResult.isSuccess()); - assertEquals(DOCID, docId); - resultsReceived.incrementAndGet(); - }; - - FeedClient feedClient = new FeedClientImpl(sessionParams, resultCallback, FeedClientFactory.createTimeoutExecutor(), Clock.systemUTC()); - - @Test - public void testStreamAndClose() { - feedClient.stream(DOCID, "blob"); - feedClient.close(); - assertEquals(1, resultsReceived.get()); - } - - @Test - public void testGetStatsAsJson() throws Exception { - feedClient.stream(DOCID, "blob"); - while (resultsReceived.get() == 0) {Thread.sleep(3); } - String stats = feedClient.getStatsAsJson(); - assertTrue(stats.contains("\"dryRun\":true")); - feedClient.close(); - } - - @Test - public void testFeedJson() { - InputStream stream = new ByteArrayInputStream((String.format("[{\"remove\": \"%s\"}]", DOCID) - .getBytes(StandardCharsets.UTF_8))); - AtomicInteger docCounter = new AtomicInteger(0); - FeedClient.feedJson(stream, feedClient, docCounter); - assertEquals(1, docCounter.get()); - feedClient.close(); - assertEquals(1, resultsReceived.get()); - } - - @Test - public void testFeedXml() { - InputStream stream = new ByteArrayInputStream((String.format( - "<document documenttype=\"music\" documentid=\"%s\">\n</document>\n", DOCID) - .getBytes(StandardCharsets.UTF_8))); - AtomicInteger docCounter = new AtomicInteger(0); - FeedClient.feedXml(stream, feedClient, docCounter); - assertEquals(1, docCounter.get()); - feedClient.close(); - assertEquals(1, resultsReceived.get()); - } - -} - diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/ManualClock.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/ManualClock.java deleted file mode 100644 index 72edba7adb3..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/ManualClock.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import java.time.Clock; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.time.temporal.TemporalAmount; - -/** - * A clock which initially has the time of its creation but can only be advanced by calling advance - * - * @author bratseth - */ -public class ManualClock extends Clock { - - private Instant currentTime = Instant.now(); - - public ManualClock() {} - - public ManualClock(String utcIsoTime) { - this(at(utcIsoTime)); - } - - public ManualClock(Instant currentTime) { - this.currentTime = currentTime; - } - - public void advance(TemporalAmount temporal) { - currentTime = currentTime.plus(temporal); - } - - public void setInstant(Instant time) { - currentTime = time; - } - - @Override - public Instant instant() { return currentTime; } - - @Override - public ZoneId getZone() { return null; } - - @Override - public Clock withZone(ZoneId zone) { return null; } - - @Override - public long millis() { return currentTime.toEpochMilli(); } - - public static Instant at(String utcIsoTime) { - return LocalDateTime.parse(utcIsoTime, DateTimeFormatter.ISO_DATE_TIME).atZone(ZoneOffset.UTC).toInstant(); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/Server.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/Server.java deleted file mode 100644 index 078606274d6..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/Server.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import org.eclipse.jetty.server.ServerConnector; -import org.eclipse.jetty.server.handler.AbstractHandler; - -/** - * @author Einar M R Rosenvinge - */ -public final class Server implements AutoCloseable { - - private final org.eclipse.jetty.server.Server server; - - public Server(AbstractHandler handler, int port) { - this.server = new org.eclipse.jetty.server.Server(port); - server.setHandler(handler); - try { - server.start(); - assert(server.isStarted()); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Override - public void close() throws RuntimeException { - try { - server.stop(); - } catch (RuntimeException e) { - throw e; - } catch (Exception e) { - throw new RuntimeException("jetty server.stop() failed", e); - } - } - - public int getPort() { - return ((ServerConnector)server.getConnectors()[0]).getLocalPort(); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallbackTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallbackTest.java deleted file mode 100644 index 5f4a0fdec9a..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallbackTest.java +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import org.junit.Test; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.hamcrest.CoreMatchers.containsString; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class SimpleLoggerResultCallbackTest { - @Test - public void testAverageCalculation() { - SimpleLoggerResultCallback logger = new SimpleLoggerResultCallback(new AtomicInteger(3), 0, false); - Instant now = Instant.now(); - logger.newSamplingPeriod(now); - Result result = mock(Result.class); - when(result.isSuccess()).thenReturn(true); - // 3 documents in 0.2 secs --> 15 docs/sec - logger.onCompletion("1", result); - logger.onCompletion("1", result); - logger.onCompletion("1", result); - double rate = logger.newSamplingPeriod(now.plusMillis(200)).rate; - assertEquals(rate, 15., 0.1 /* delta */); - } - - @Test - public void testAverageCalculationExteremeValues() { - SimpleLoggerResultCallback logger = new SimpleLoggerResultCallback(new AtomicInteger(3), 0, false); - Instant now = Instant.now(); - logger.newSamplingPeriod(now); - // 0 duration, 0 documents - double rate = logger.newSamplingPeriod(now).rate; - assertEquals(rate, 0, 0.1 /* delta */); - } - - @Test - public void testOutput() { - SimpleLoggerResultCallback logger = new SimpleLoggerResultCallback(new AtomicInteger(3), 0, false); - Instant now = Instant.now(); - logger.newSamplingPeriod(now); - Result result = mock(Result.class); - when(result.isSuccess()).thenReturn(true); - // 3 documents in 0.2 secs --> 15 docs/sec - logger.onCompletion("1", result); - logger.onCompletion("1", result); - logger.onCompletion("1", result); - double rate = logger.newSamplingPeriod(now.plusMillis(200)).rate; - assertEquals(rate, 15., 0.1 /* delta */); - } - - private void verifyPrintout(boolean ignoreConditionNotMet) { - ArrayList<String> outputList = new ArrayList<>(); - - SimpleLoggerResultCallback logger = new SimpleLoggerResultCallback(new AtomicInteger(30), 0, ignoreConditionNotMet) { - @Override - protected void println(String output) { - outputList.add(output); - } - @Override - protected DocumentRate newSamplingPeriod(Instant now) { - return new DocumentRate(19999999.2342342366664); - } - }; - // 2 success, 1 failure - Result result = mock(Result.class); - when(result.isSuccess()).thenReturn(true); - when(result.isSuccessOrConditionNotMet()).thenReturn(true); - logger.onCompletion("1", result); - logger.onCompletion("1", result); - when(result.isSuccess()).thenReturn(false); - when(result.isSuccessOrConditionNotMet()).thenReturn(false); - when(result.toString()).thenReturn("fooError"); - logger.onCompletion("1", result); - logger.printProgress(); - assertThat(outputList.toString(), - containsString("Result received: 3 (1 failed so far, 30 sent, success rate 19999999.23 docs/sec).")); - assertThat(outputList.toString(), containsString("Failure: fooError")); - } - - @Test - public void testPrintout() { - verifyPrintout(false); - verifyPrintout(true); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SyncFeedClientTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SyncFeedClientTest.java deleted file mode 100644 index 53b2b236d50..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SyncFeedClientTest.java +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.config.SessionParams; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -import com.yahoo.vespa.http.client.SyncFeedClient.SyncOperation; -import com.yahoo.vespa.http.client.SyncFeedClient.SyncResult; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertNull; - -/** - * Tests the sync wrapper to the feed client - * - * @author bratseth - */ -public class SyncFeedClientTest { - - @Test - public void testFeedJson() { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder() - .addEndpoint(Endpoint.create("localhost")) - .build()) - .setConnectionParams(new ConnectionParams.Builder() - .setDryRun(true) - .build()) - .build(); - SyncFeedClient feedClient = new SyncFeedClient(sessionParams); - - assertFeedSuccessful(feedClient); - assertFeedSuccessful(feedClient); // ensure the client can be reused - feedClient.close(); - } - - private void assertFeedSuccessful(SyncFeedClient feedClient) { - List<SyncOperation> operations = new ArrayList<>(); - - operations.add(new SyncOperation("id::test::1", - "{" + - " \"put\": \"id::test::1\"," + - " \"fields\": {" + - " \"title\": \"Title 1\"" + - " }" + - "}")); - operations.add(new SyncOperation("id::test::2", - "{" + - " \"put\": \"id::test::2\"," + - " \"fields\": {" + - " \"title\": \"Title 2\"" + - " }" + - "}")); - operations.add(new SyncOperation("id::test::3", - "{" + - " \"put\": \"id::test::3\"," + - " \"fields\": {" + - " \"title\": \"Title 3\"" + - " }" + - "}")); - operations.add(new SyncOperation("id::test::3", // Another operation for the same document - "{" + - " \"put\": \"id::test::3\"," + - " \"fields\": {" + - " \"title\": \"Title 4\"" + - " }" + - "}")); - operations.add(new SyncOperation("id::test::4", - "{" + - " \"put\": \"id::test::4\"," + - " \"fields\": {" + - " \"title\": \"Title 4\"" + - " }" + - "}", "opId_4", null)); - operations.add(new SyncOperation("id::test::4", // Another operation for the same document - "{" + - " \"put\": \"id::test::4\"," + - " \"fields\": {" + - " \"title\": \"Title 44\"" + - " }" + - "}", "opId_44", null)); - - SyncResult result = feedClient.stream(operations); - - assertTrue(result.isSuccess()); - assertEquals(6, result.results().size()); - assertNull(result.exception()); - assertEquals("id::test::1", result.results().get(0).getDocumentId()); - assertEquals("id::test::2", result.results().get(1).getDocumentId()); - assertEquals("id::test::3", result.results().get(2).getDocumentId()); - assertEquals("id::test::3", result.results().get(3).getDocumentId()); - assertEquals("id::test::4", result.results().get(4).getDocumentId()); - assertEquals("id::test::4", result.results().get(5).getDocumentId()); - assertEquals("opId_4", result.results().get(4).getOperationId()); - assertEquals("opId_44", result.results().get(5).getOperationId()); - assertTrue(result.results().get(4).getDocumentDataAsCharSequence().toString().contains("\"Title 4\"")); - assertTrue(result.results().get(5).getDocumentDataAsCharSequence().toString().contains("\"Title 44\"")); - - result.results().forEach(r -> assertNotNull(r.getOperationId())); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestDocument.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestDocument.java deleted file mode 100644 index 67b3fc7653d..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestDocument.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -/** -* @author Einar M R Rosenvinge -*/ -public class TestDocument { - private final String documentId; - private final byte[] contents; - - TestDocument(String documentId, byte[] contents) { - this.documentId = documentId; - this.contents = contents; - } - - public String getDocumentId() { - return documentId; - } - - public byte[] getContents() { - return contents; - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestUtils.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestUtils.java deleted file mode 100644 index b27fbba3e96..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestUtils.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.zip.GZIPInputStream; - -import static org.junit.Assert.assertNull; - -/** - * @author Einar M R Rosenvinge - */ -@SuppressWarnings("deprecation") -public class TestUtils { - - public static void writeDocuments(Session session, List<TestDocument> documents) throws IOException { - for (TestDocument document : documents) { - writeDocument(session, document); - } - } - - public static void writeDocument(Session session, TestDocument document) throws IOException { - OutputStream operation = session.stream(document.getDocumentId()); - operation.write(document.getContents()); - operation.close(); - } - - public static Map<String, Result> getResults(Session session, int num) throws InterruptedException { - Map<String, Result> results = new HashMap<>(); - for (int i = 0; i < num; i++) { - Result r = session.results().poll(120, TimeUnit.SECONDS); - if (r == null) { - String extraInfo = ""; - extraInfo = "stats=" + session.getStatsAsJson(); - throw new IllegalStateException("Did not receive result within timeout. (" + extraInfo + ") " + - "Results received: " + results.values()); - } - results.put(r.getDocumentId(), r); - } - assertNull(session.results().poll(100, TimeUnit.MILLISECONDS)); - return results; - } - - public static String zipStreamToString(InputStream inputStream) throws IOException { - GZIPInputStream gzipInputStream = new GZIPInputStream(inputStream); - final StringBuilder rawContent = new StringBuilder(); - while (true) { - int x = gzipInputStream.read(); - if (x < 0) { - break; - } - rawContent.append((char) x); - } - return rawContent.toString(); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ClusterTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ClusterTest.java deleted file mode 100644 index f4c12974f6d..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ClusterTest.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.config; - -import org.junit.Test; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.22 - */ -public class ClusterTest { - - @Test - public void testSimple() { - Cluster cluster = new Cluster.Builder().build(); - - assertThat(cluster.getEndpoints().size(), is(0)); - assertThat(cluster.getRoute(), nullValue()); - } - - @Test - public void testConfig() { - Cluster cluster = new Cluster.Builder() - .addEndpoint(Endpoint.create("a")) - .addEndpoint(Endpoint.create("b")) - .setRoute("blah") - .build(); - - assertThat(cluster.getEndpoints().size(), is(2)); - assertThat(cluster.getEndpoints().get(0).getHostname(), equalTo("a")); - assertThat(cluster.getEndpoints().get(1).getHostname(), equalTo("b")); - assertThat(cluster.getRoute(), equalTo("blah")); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ConnectionParamsTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ConnectionParamsTest.java deleted file mode 100644 index 3fcc73e3cdc..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ConnectionParamsTest.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.config; - -import org.junit.Test; - -import javax.net.ssl.SSLContext; -import java.security.NoSuchAlgorithmException; -import java.util.Iterator; -import java.util.Map; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.core.Is.is; -import static org.hamcrest.core.IsNull.nullValue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.22 - */ -public class ConnectionParamsTest { - - @Test - public void testDefaults() { - ConnectionParams params = new ConnectionParams.Builder().build(); - - assertThat(params.getHeaders().isEmpty(), is(true)); - assertThat(params.getNumPersistentConnectionsPerEndpoint(), is(1)); - assertThat(params.getSslContext(), nullValue()); - } - - @Test - public void testSetters() throws NoSuchAlgorithmException { - ConnectionParams params = new ConnectionParams.Builder() - .addHeader("Foo", "Bar") - .addHeader("Foo", "Baz") - .addHeader("Banana", "Apple") - .setNumPersistentConnectionsPerEndpoint(2) - .setSslContext(SSLContext.getDefault()) - .build(); - - assertThat(params.getNumPersistentConnectionsPerEndpoint(), is(2)); - - assertThat(params.getHeaders().isEmpty(), is(false)); - assertThat(params.getHeaders().size(), is(3)); - //Iteration order seems stable, let's keep it like this for now - Iterator<Map.Entry<String, String>> headers = params.getHeaders().iterator(); - Map.Entry<String, String> header1 = headers.next(); - assertThat(header1.getKey(), equalTo("Foo")); - assertThat(header1.getValue(), equalTo("Bar")); - Map.Entry<String, String> header2 = headers.next(); - assertThat(header2.getKey(), equalTo("Foo")); - assertThat(header2.getValue(), equalTo("Baz")); - Map.Entry<String, String> header3 = headers.next(); - assertThat(header3.getKey(), equalTo("Banana")); - assertThat(header3.getValue(), equalTo("Apple")); - } - - @Test - public void header_providers_are_registered() { - ConnectionParams.HeaderProvider dummyProvider1 = () -> "fooValue"; - ConnectionParams.HeaderProvider dummyProvider2 = () -> "barValue"; - ConnectionParams params = new ConnectionParams.Builder() - .addDynamicHeader("foo", dummyProvider1) - .addDynamicHeader("bar", dummyProvider2) - .build(); - Map<String, ConnectionParams.HeaderProvider> headerProviders = params.getDynamicHeaders(); - assertEquals(2, headerProviders.size()); - assertEquals(dummyProvider1, headerProviders.get("foo")); - assertEquals(dummyProvider2, headerProviders.get("bar")); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/EndpointTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/EndpointTest.java deleted file mode 100644 index 415a7746b3a..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/EndpointTest.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.config; - -import org.junit.Test; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.22 - */ -public class EndpointTest { - - @Test - public void testBasic() { - Endpoint endpoint = Endpoint.create("foo"); - - assertThat(endpoint.getHostname(), equalTo("foo")); - assertThat(endpoint.getPort(), equalTo(4080)); - assertThat(endpoint.isUseSsl(), is(false)); - } - - @Test - public void testBasicWithHttpProtocolPrefix() { - Endpoint endpoint = Endpoint.create("http://foo"); - assertThat(endpoint.getHostname(), equalTo("foo")); - } - - @Test(expected = RuntimeException.class) - public void testBasicWithHttpsProtocolPrefix() { - Endpoint.create("https://foo"); - } - - @Test - public void testAdvanced() { - Endpoint endpoint = Endpoint.create("bar", 1234, true); - - assertThat(endpoint.getHostname(), equalTo("bar")); - assertThat(endpoint.getPort(), equalTo(1234)); - assertThat(endpoint.isUseSsl(), is(true)); - } - - @Test - public void testMethods() { - Endpoint a = Endpoint.create("a"); - Endpoint b = Endpoint.create("b"); - - assertThat(a, not(equalTo(b))); - assertThat(a.hashCode(), not(equalTo(b.hashCode()))); - - Endpoint a2 = Endpoint.create("a"); - - assertThat(a, equalTo(a2)); - assertThat(a.hashCode(), equalTo(a2.hashCode())); - - a.toString(); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/FeedParamsTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/FeedParamsTest.java deleted file mode 100644 index 82e0b4deac0..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/FeedParamsTest.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.config; - -import org.junit.Test; - -import java.util.concurrent.TimeUnit; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.22 - */ -public class FeedParamsTest { - - @Test - public void testDefaults() { - FeedParams params = new FeedParams.Builder().build(); - - assertThat(params.getDataFormat(), equalTo(FeedParams.DataFormat.JSON_UTF8)); - assertThat(params.getMaxChunkSizeBytes(), is(50 * 1024)); - assertThat(params.getRoute(), nullValue()); - assertThat(params.getServerTimeout(TimeUnit.SECONDS), is(180L)); - assertThat(params.getClientTimeout(TimeUnit.SECONDS), is(20L)); - } - - @Test - public void testConfig() { - FeedParams params = new FeedParams.Builder() - .setDataFormat(FeedParams.DataFormat.XML_UTF8) - .setMaxChunkSizeBytes(123) - .setRoute("abc") - .setClientTimeout(321, TimeUnit.SECONDS) - .build(); - - assertThat(params.getDataFormat(), equalTo(FeedParams.DataFormat.XML_UTF8)); - assertThat(params.getMaxChunkSizeBytes(), is(123)); - assertThat(params.getRoute(), equalTo("abc")); - assertThat(params.getServerTimeout(TimeUnit.SECONDS), is(180L)); - assertThat(params.getClientTimeout(TimeUnit.SECONDS), is(321L)); - - params = new FeedParams.Builder() - .setServerTimeout(333L, TimeUnit.SECONDS) - .setClientTimeout(222L, TimeUnit.SECONDS) - .build(); - - assertThat(params.getServerTimeout(TimeUnit.SECONDS), is(333L)); - assertThat(params.getClientTimeout(TimeUnit.SECONDS), is(222L)); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/DocumentTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/DocumentTest.java deleted file mode 100644 index 8f707f2426e..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/DocumentTest.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import org.junit.Test; - -import java.nio.ByteBuffer; -import java.nio.ReadOnlyBufferException; -import java.time.Clock; - -import static org.junit.Assert.assertEquals; - -public class DocumentTest { - - @Test - public void simpleCaseOk() { - String docId = "doc id"; - String docContent = "foo"; - Document document = new Document(docId, docContent.getBytes(), null, Clock.systemUTC().instant()); - assertEquals(docId, document.getDocumentId()); - assertEquals(ByteBuffer.wrap(docContent.getBytes()), document.getData()); - assertEquals(docContent, document.getDataAsString().toString()); - // Make sure that data is not modified on retrieval. - assertEquals(docContent, document.getDataAsString().toString()); - assertEquals(ByteBuffer.wrap(docContent.getBytes()), document.getData()); - assertEquals(docId, document.getDocumentId()); - } - - @Test(expected = ReadOnlyBufferException.class) - public void notMutablePutTest() { - Document document = new Document("id", null, "data", null, Clock.systemUTC().instant()); - document.getData().put("a".getBytes()); - } - - @Test(expected = ReadOnlyBufferException.class) - public void notMutableCompactTest() { - Document document = new Document("id", null, "data", null, Clock.systemUTC().instant()); - document.getData().compact(); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/EncoderTestCase.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/EncoderTestCase.java deleted file mode 100644 index aa926578a94..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/EncoderTestCase.java +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import static org.junit.Assert.*; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * Functional tests for encoding Encoder, i.e. encoding scheme only producing - * ASCII and never containing white space or control characters. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class EncoderTestCase { - private final String basic = "abc123"; - private final String basic2 = "abc{20}123"; - private final String quotedIsLast = "abc{20}"; - private final String quotedIsLastDecoded = "abc "; - private final String basic2Decoded = "abc 123"; - private final String unterminated = "abc{33"; - private final String unterminated2 = "abc{"; - private final String emptyQuoted = "abc{}123"; - private final String outsideUnicode = "abc{7fffffff}"; - private final String noise = "abc{7fff{||\\ffff}"; - private final String fullAsciiEncoded = "{0}{1}{2}{3}{4}{5}{6}{7}{8}{9}" - + "{a}{b}{c}{d}{e}{f}{10}{11}{12}{13}{14}{15}{16}{17}" - + "{18}{19}{1a}{1b}{1c}{1d}{1e}{1f}{20}" - + "!\"#$%&'()*+,-./0123456789:;<=>?@" - + "ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`" - + "abcdefghijklmnopqrstuvwxyz{7b}|{7d}~{7f}"; - private final int[] testCodepoints = { 0x0, '\n', ' ', 'a', '{', '}', 0x7f, 0x80, - 0x7ff, 0x800, 0xd7ff, 0xe000, 0xffff, 0x10000, 0x10ffff, 0x34, - 0x355, 0x2567, 0xfff, 0xe987, 0x100abc }; - private final String semiNastyEncoded = "{0}{a}{20}a{7b}{7d}{7f}{80}" - + "{7ff}{800}{d7ff}{e000}{ffff}{10000}{10ffff}4" - + "{355}{2567}{fff}{e987}{100abc}"; - private final String invalidUnicode = "abc\ud812"; - private final String invalidUnicodeEncoded = "abc{d812}"; - - StringBuilder s; - - @Before - public void setUp() { - s = new StringBuilder(); - } - - @After - public void tearDown() { - s = null; - } - - @Test - public final void testBasic() { - Encoder.encode(basic, s); - assertEquals(basic, s.toString()); - } - - @Test - public final void testBasic2() { - Encoder.encode(basic2Decoded, s); - assertEquals(basic2, s.toString()); - } - - @Test - public final void testEncodeAscii() { - Encoder.encode(fullAscii(), s); - assertEquals(fullAsciiEncoded, s.toString()); - } - - @Test - public final void testEncodeMixed() { - Encoder.encode(semiNasty(), s); - assertEquals(semiNastyEncoded, s.toString()); - } - - @Test - public final void testEncodeQuotedIsLast() { - Encoder.encode(quotedIsLastDecoded, s); - assertEquals(quotedIsLast, s.toString()); - } - - @Test - public final void testInvalidUnicode() { - Encoder.encode(invalidUnicode, s); - assertEquals(invalidUnicodeEncoded, s.toString()); - } - - - @Test - public final void testDecodeBasic() { - Encoder.decode(basic, s); - assertEquals(basic, s.toString()); - } - - @Test - public final void testDecodeBasic2() { - Encoder.decode(basic2, s); - assertEquals(basic2Decoded, s.toString()); - } - - @Test - public final void testDecodeAscii() { - Encoder.decode(fullAsciiEncoded, s); - assertEquals(fullAscii(), s.toString()); - } - - @Test - public final void testDecodeMixed() { - Encoder.decode(semiNastyEncoded, s); - assertEquals(semiNasty(), s.toString()); - } - - - - @Test - public final void testDecodeQuotedIsLast() { - Encoder.decode(quotedIsLast, s); - assertEquals(quotedIsLastDecoded, s.toString()); - } - - - @Test - public final void testDecodeUnterminated() { - try { - Encoder.decode(unterminated, s); - } catch (IllegalArgumentException e) { - return; - } - fail("Expected IllegalArgumentException"); - } - - @Test - public final void testDecodeUnterminated2() { - try { - Encoder.decode(unterminated2, s); - } catch (IllegalArgumentException e) { - return; - } - fail("Expected IllegalArgumentException"); - - } - - @Test - public final void testEmptyQuoted() { - try { - Encoder.decode(emptyQuoted, s); - } catch (IllegalArgumentException e) { - return; - } - fail("Expected IllegalArgumentException"); - } - - @Test - public final void testOutsideUnicode() { - try { - Encoder.decode(outsideUnicode, s); - } catch (IllegalArgumentException e) { - return; - } - fail("Expected IllegalArgumentException"); - } - - - @Test - public final void testNoise() { - try { - Encoder.decode(noise, s); - } catch (IllegalArgumentException e) { - return; - } - fail("Expected IllegalArgumentException"); - } - - @Test - public final void testIllegalInputCharacter() { - try { - Encoder.decode("abc\u00e5", s); - } catch (IllegalArgumentException e) { - return; - } - fail("Expected IllegalArgumentException"); - } - - - @Test - public final void testNoIllegalCharactersInOutputForAscii() { - Encoder.encode(fullAscii(), s); - checkNoNonAscii(s.toString()); - } - - @Test - public final void testNoIllegalCharactersInOutputForMixedInput() { - Encoder.encode(semiNasty(), s); - checkNoNonAscii(s.toString()); - } - - @Test - public final void testSymmetryAscii() { - StringBuilder forDecoding = new StringBuilder(); - Encoder.encode(fullAscii(), s); - Encoder.decode(s.toString(), forDecoding); - assertEquals(fullAscii(), forDecoding.toString()); - } - - @Test - public final void testSymmetryMixed() { - StringBuilder forDecoding = new StringBuilder(); - Encoder.encode(semiNasty(), s); - Encoder.decode(s.toString(), forDecoding); - assertEquals(semiNasty(), forDecoding.toString()); - } - - - private void checkNoNonAscii(String input) { - for (int i = 0; i < input.length(); ++i) { - char c = input.charAt(i); - if (c > '~' || c <= ' ') { - fail("Encoded data contained character ordinal " + Integer.toHexString(c)); - } - } - } - - private String fullAscii() { - StringBuilder s = new StringBuilder(); - for (int i = 0; i <= 0x7f; ++i) { - s.append((char) i); - } - return s.toString(); - } - - private String semiNasty() { - StringBuilder s = new StringBuilder(); - for (int i = 0; i < testCodepoints.length; ++i) { - s.append(Character.toChars(testCodepoints[i])); - } - return s.toString(); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/OperationProcessorTester.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/OperationProcessorTester.java deleted file mode 100644 index 2e6efffdd2c..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/OperationProcessorTester.java +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import com.yahoo.vespa.http.client.FeedClient; -import com.yahoo.vespa.http.client.FeedEndpointException; -import com.yahoo.vespa.http.client.ManualClock; -import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.core.communication.ClusterConnection; -import com.yahoo.vespa.http.client.core.communication.IOThread; -import com.yahoo.vespa.http.client.core.communication.IOThreadTest; -import com.yahoo.vespa.http.client.core.operationProcessor.IncompleteResultsThrottler; -import com.yahoo.vespa.http.client.core.operationProcessor.OperationProcessor; - -import java.time.Instant; -import java.util.List; -import java.util.concurrent.ScheduledThreadPoolExecutor; - -import static org.junit.Assert.assertEquals; - -/** - * Helper for testing with an operation processor - * - * @author bratseth - */ -public class OperationProcessorTester { - - private final Endpoint endpoint; - private final int clusterId = 0; - private final ManualClock clock; - private final TestResultCallback resultCallback; - private final OperationProcessor operationProcessor; - - public OperationProcessorTester() { - endpoint = Endpoint.create("test-endpoint"); - SessionParams.Builder params = new SessionParams.Builder(); - Cluster.Builder clusterParams = new Cluster.Builder(); - clusterParams.addEndpoint(endpoint); - params.addCluster(clusterParams.build()); - ConnectionParams.Builder connectionParams = new ConnectionParams.Builder(); - connectionParams.setDryRun(true); - connectionParams.setRunThreads(false); - params.setConnectionParams(connectionParams.build()); - - clock = new ManualClock(Instant.ofEpochMilli(0)); - resultCallback = new TestResultCallback(); - operationProcessor = new OperationProcessor(new IncompleteResultsThrottler(1, 100, clock, new ThrottlePolicy()), - resultCallback, - params.build(), - new ScheduledThreadPoolExecutor(1), - clock); - } - - public ManualClock clock() { return clock; } - - /** Asserts that this has but a single IOThread and returns it */ - public IOThread getSingleIOThread() { - assertEquals(1, clusterConnections().size()); - assertEquals(1, clusterConnections().get(0).ioThreads().size()); - return clusterConnections().get(0).ioThreads().get(0); - } - - /** Do n iteration of work in all io threads of this */ - public void tick(int n) { - for (int i = 0; i < n; i++) - for (ClusterConnection cluster : operationProcessor.clusters()) - for (IOThread thread : cluster.ioThreads()) - thread.tick(); - } - - public void send(String documentId) { - operationProcessor.sendDocument(new Document(documentId, documentId, "data of " + documentId, null, clock.instant())); - } - - public int incomplete() { - return operationProcessor.getIncompleteResultQueueSize(); - } - - public int success() { - return resultCallback.successes; - } - - public List<ClusterConnection> clusterConnections() { - return operationProcessor.clusters(); - } - - public int failures() { - return resultCallback.failures; - } - - public int endpointExceptions() { - return resultCallback.endpointExceptions; - } - - public Result lastResult() { - return resultCallback.lastResult; - } - - private static class TestResultCallback implements FeedClient.ResultCallback { - - private int successes = 0; - private int failures = 0; - private int endpointExceptions = 0; - private Result lastResult; - - @Override - public void onCompletion(String docId, Result documentResult) { - this.lastResult = documentResult; - if (documentResult.isSuccess()) - successes++; - else - failures++; - } - - @Override - public void onEndpointException(FeedEndpointException exception) { - endpointExceptions++; - } - - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/ThrottlePolicyTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/ThrottlePolicyTest.java deleted file mode 100644 index d5a4d09fb8f..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/ThrottlePolicyTest.java +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import org.junit.Test; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; - -public class ThrottlePolicyTest { - - private final ThrottlePolicy throttlePolicy = new ThrottlePolicy(); - // Default values for tests. - private int numOk = 1000; - private int prevOk = 1000; - private int prevMax = 100; - private int max = 100; - private boolean queued = true; - private double dynamicFactor = 0.1; - - @Test - public void samePerformanceShouldTuneDown() { - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(95)); - } - - @Test - public void improvedPerformanceSameSizeShouldTuneDown() { - numOk += 200; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(89)); - } - - @Test - public void improvedPerformanceSmallerSizeTuneDownFurther() { - numOk += 200; - max = 70; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(63)); - } - - @Test - public void improvedPerformanceLargerSizeIncrease() { - numOk += 200; - max = 130; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(143)); - dynamicFactor = 100; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(156)); - } - - @Test - public void improvedPerformanceLargerSizeButQueuedFalse() { - numOk += 200; - max = 130; - queued = false; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(128)); - } - - @Test - public void lowerPerformanceSameSizeShouldIncrease() { - numOk -= 200; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(110)); - } - - @Test - public void lowerPerformanceSmallerSizeShouldIncreaseSize() { - numOk -= 200; - max = 30; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(33)); - dynamicFactor = 100; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(36)); - } - - @Test - public void lowerPerformanceLargerSizeTuneDownFurther() { - numOk -= 200; - max = 130; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(116)); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/XmlFeedReaderTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/XmlFeedReaderTest.java deleted file mode 100644 index e71cee0f4c2..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/XmlFeedReaderTest.java +++ /dev/null @@ -1,254 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import com.yahoo.vespa.http.client.FeedClient; -import org.junit.Test; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.xml.sax.SAXParseException; - -import java.io.BufferedInputStream; -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.Is.is; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; - -public class XmlFeedReaderTest { - private final static String feedResource = "/vespacorpfeed-prod-sample.xml"; - - private final static String feedResource2 = "/xml-challenge.xml"; - private final static String feedResource3 = "/xml-challenge2.xml"; - private final static String feedResource4 = "/xml-challenge3.xml"; - - private static final XmlMapper xmlMapper = new XmlMapper(); - - private final String updateDocUpdate = - "<?xml version=\"1.0\"?>\n" + - "<vespafeed>\n" + - "<update documentid=\"id:banana:banana::complex\" documenttype=\"banana\">\n" + - " <add fieldpath=\"structarr\">\n" + - " <item>\n" + - " <bytearr>\n" + - " <item>30</item>\n" + - " <item>55</item>\n" + - " </bytearr>\n" + - " </item>\n" + - " </add>\n" + - "</update>\n" + - "</vespafeed>\n"; - - @Test - public void testReadUpdate() throws Exception { - InputStream stream = new ByteArrayInputStream(updateDocUpdate.getBytes(StandardCharsets.UTF_8)); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - XmlFeedReader.read(stream, feedClient, numSent); - assertThat(numSent.get(), is(1)); - } - - private final String updateDocRemove = - "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "\n" + - "<vespafeed>\n" + - " <remove documentid=\"id:music:music::http://music.yahoo.com/Bob0/BestOf\" />\n" + - " <remove documentid=\"id:music:music::http://music.yahoo.com/Bob9/BestOf\" />\n" + - "</vespafeed>"; - - @Test - public void testReadRemove() throws Exception { - InputStream stream = new ByteArrayInputStream(updateDocRemove.getBytes(StandardCharsets.UTF_8)); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - XmlFeedReader.read(stream, feedClient, numSent); - assertThat(numSent.get(), is(2)); - } - - private final String insertDocOperation = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"+ - "<vespafeed>\n"+ - "\n"+ - " <document type=\"music\" documentid=\"id:music:music::http://music.yahoo.com/bobdylan/BestOf\">\n"+ - " <title>Best of Bob Dylan</title>\n"+ - " </document>\n"+ - "\n"+ - " <document type=\"music\" documentid=\"id:music:music::http://music.yahoo.com/metallica/BestOf\">\n"+ - " <title>Best of Metallica</title>\n"+ - " </document>\n"+ - "</vespafeed>"; - - @Test - public void testInsert() throws Exception { - InputStream stream = new ByteArrayInputStream(insertDocOperation.getBytes(StandardCharsets.UTF_8)); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - XmlFeedReader.read(stream, feedClient, numSent); - assertThat(numSent.get(), is(2)); - } - - private final String badperation = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"+ - "<vespafeed>\n"+ - " <badtag type=\"music\" documentid=\"id:music:music::http://music.yahoo.com/bobdylan/BestOf\">\n"+ - " <title>Best of Bob Dylan</title>\n"+ - " </badtag>\n"+ - "</vespafeed>"; - - @Test - public void testNonDocument() throws Exception { - InputStream stream = new ByteArrayInputStream(badperation.getBytes(StandardCharsets.UTF_8)); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - XmlFeedReader.read(stream, feedClient, numSent); - assertThat(numSent.get(), is(0)); - } - - @Test(expected=SAXParseException.class) - public void testGarbage() throws Exception { - InputStream stream = new ByteArrayInputStream("eehh".getBytes(StandardCharsets.UTF_8)); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - XmlFeedReader.read(stream, feedClient, numSent); - } - - @Test - public void testEncoding() throws Exception { - InputStream stream = new ByteArrayInputStream("<?xml version=\"1.0\" encoding=\"utf8\"?><vespafeed><remove documentid=\"id:&\"/></vespafeed>" - .getBytes(StandardCharsets.UTF_8)); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - - doAnswer(new Answer<Object>() { - public Object answer(InvocationOnMock invocation) { - Object[] args = invocation.getArguments(); - String docId = (String) args[0]; - CharSequence value = (CharSequence)args[1]; - assertThat(value.toString(), is("<remove documentid=\"id:&\"></remove>")); - assertThat(docId, is("id:&")); - return null; - } - }).when(feedClient).stream(anyString(), any()); - XmlFeedReader.read(stream, feedClient, numSent); - assertThat(numSent.get(), is(1)); - } - - private final String characterDocs = "" + - "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" + - "<!-- GENERATED VESPA-XML BY YSSTOXML -->\n" + - "<!-- USE ONLY FOR BATCH INDEXING -->\n" + - "<vespafeed>\n" + - " <document documenttype=\"simple\" documentid=\"id:test::&http://www.e.no/matprat\">\n" + - " <language><![CDATA[ja]]></language>\n" + - " <title><![CDATA[test document1]]></title>\n" + - " <description><![CDATA[Bjørnen' blåbær på øy nærheten.]]></description>\n" + - " <date>1091356845</date>\n" + - " <surl><![CDATA[http://www.eventyr.no/matprat]]></surl>\n" + - " </document>\n" + - "\n" + - "</vespafeed>\n"; - - @Test - public void testCharacterEndcoding() throws Exception { - InputStream stream = new ByteArrayInputStream(characterDocs.getBytes(StandardCharsets.UTF_8)); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - final AtomicBoolean success = new AtomicBoolean(false); - doAnswer(new Answer<Object>() { - public Object answer(InvocationOnMock invocation) { - Object[] args = invocation.getArguments(); - String docId = (String) args[0]; - CharSequence value = (CharSequence)args[1]; - assertThat(value.toString(), is( - "<document documenttype=\"simple\" documentid=\"id:test::&http://www.e.no/matprat\">\n" + - " <language><![CDATA[ja]]></language>\n" + - " <title><![CDATA[test document1]]></title>\n" + - " <description><![CDATA[Bjørnen' blåbær på øy nærheten.]]></description>\n" + - " <date>1091356845</date>\n" + - " <surl><![CDATA[http://www.eventyr.no/matprat]]></surl>\n" + - " </document>")); - success.set(true); - return null; - } - }).when(feedClient).stream(anyString(), any()); - XmlFeedReader.read(stream, feedClient, numSent); - assertThat(numSent.get(), is(1)); - assert(success.get()); - } - - @Test - public void testRealData() throws Exception { - InputStream inputStream = XmlFeedReaderTest.class.getResourceAsStream(feedResource); - BufferedInputStream bis = new BufferedInputStream(inputStream); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - - XmlFeedReader.read(bis, feedClient, numSent); - assertThat(numSent.get(), is(6)); - } - - private static class XmlTestFeedClient implements FeedClient { - - public List<String> documentIds = new ArrayList<>(); - public List<CharSequence> datas = new ArrayList<>(); - public List<Object> contexts = new ArrayList<>(); - - @Override - public void stream(String documentId, CharSequence documentData) { - stream(documentId, documentData, null); - } - - @Override - public void stream(String documentId, String operationId, CharSequence documentData, Object context) { - documentIds.add(documentId); - datas.add(documentData); - contexts.add(context); - } - - - @Override - public void close() { } - - @Override - public String getStatsAsJson() { return null; } - } - - // Only for xml with single doc. - private void verifyNoTransformationOfXml(String filename) throws Exception { - InputStream inputStream = XmlFeedReaderTest.class.getResourceAsStream(filename); - BufferedInputStream bis = new BufferedInputStream(inputStream); - AtomicInteger numSent = new AtomicInteger(0); - XmlTestFeedClient feedClient = new XmlTestFeedClient(); - XmlFeedReader.read(bis, feedClient, numSent); - assertThat(numSent.get(), is(1)); - String document = feedClient.datas.get(0).toString(); - - InputStream inputStream2 = XmlFeedReaderTest.class.getResourceAsStream(filename); - String rawXML = new java.util.Scanner(inputStream2, "UTF-8").useDelimiter("\\A").next(); - - JsonNode decodedDocument = xmlMapper.readTree(document); - JsonNode rawDocuments = xmlMapper.readTree(rawXML); - assertThat(decodedDocument, is(rawDocuments.get("document"))); - } - - @Test public void testCData() throws Exception { - verifyNoTransformationOfXml(feedResource2); - } - - @Test public void testPCData() throws Exception { - verifyNoTransformationOfXml(feedResource3); - } - - @Test public void testAposData() throws Exception { - verifyNoTransformationOfXml(feedResource4); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/api/FeedClientImplTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/api/FeedClientImplTest.java deleted file mode 100644 index 283c1169440..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/api/FeedClientImplTest.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.api; - -import org.junit.Test; - -import java.time.Instant; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; - -/** - * @author dybis - */ -public class FeedClientImplTest { - - int sleepValueMillis = 1; - - @Test - public void testCloseWaitTimeOldTimestamp() { - assertThat(FeedClientImpl.waitForOperations(Instant.now().minusSeconds(1000), sleepValueMillis, 10), is(false)); - } - - @Test - public void testCloseWaitTimeOutInFutureStillOperations() { - assertThat(FeedClientImpl.waitForOperations(Instant.now(), sleepValueMillis, 2000), is(true)); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionTest.java deleted file mode 100644 index 2597ddddd88..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionTest.java +++ /dev/null @@ -1,355 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.TestUtils; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.ServerResponseException; -import org.apache.http.Header; -import org.apache.http.HeaderElement; -import org.apache.http.HttpEntity; -import org.apache.http.HttpResponse; -import org.apache.http.ParseException; -import org.apache.http.StatusLine; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.InputStreamEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.message.BasicHeader; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.mockito.stubbing.Answer; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.time.Clock; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class ApacheGatewayConnectionTest { - - @Rule - public ExpectedException expectedException = ExpectedException.none(); - - @Test - public void testProtocolV3() throws Exception { - Endpoint endpoint = Endpoint.create("localhost", 666, false); - FeedParams feedParams = new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.JSON_UTF8).build(); - String clusterSpecificRoute = ""; - ConnectionParams connectionParams = new ConnectionParams.Builder().build(); - List<Document> documents = new ArrayList<>(); - - String vespaDocContent = "Hello, I a JSON doc."; - String docId = "42"; - - // This is the fake server, takes header client ID and uses this as session Id. - ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> { - Header clientIdHeader = post.getFirstHeader(Headers.CLIENT_ID); - return httpResponse(clientIdHeader.getValue(), "3"); - }); - - ApacheGatewayConnection apacheGatewayConnection = - new ApacheGatewayConnection( - endpoint, - feedParams, - clusterSpecificRoute, - connectionParams, - mockFactory, - "clientId", - Clock.systemUTC()); - apacheGatewayConnection.connect(); - apacheGatewayConnection.handshake(); - documents.add(createDoc(docId, vespaDocContent, true)); - - apacheGatewayConnection.write(documents); - } - - @Test(expected=IllegalArgumentException.class) - public void testServerReturnsBadSessionInV3() throws Exception { - Endpoint endpoint = Endpoint.create("localhost", 666, false); - FeedParams feedParams = new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.JSON_UTF8).build(); - String clusterSpecificRoute = ""; - ConnectionParams connectionParams = new ConnectionParams.Builder().build(); - - // This is the fake server, returns wrong session Id. - ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> httpResponse("Wrong Id from server", "3")); - - ApacheGatewayConnection apacheGatewayConnection = - new ApacheGatewayConnection( - endpoint, - feedParams, - clusterSpecificRoute, - connectionParams, - mockFactory, - "clientId", - Clock.systemUTC()); - apacheGatewayConnection.connect(); - List<Document> documents = new ArrayList<>(); - apacheGatewayConnection.write(documents); - } - - @Test - public void testJsonDocumentHeader() throws Exception { - Endpoint endpoint = Endpoint.create("localhost", 666, false); - FeedParams feedParams = new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.JSON_UTF8).build(); - String clusterSpecificRoute = ""; - ConnectionParams connectionParams = new ConnectionParams.Builder().setUseCompression(true).build(); - List<Document> documents = new ArrayList<>(); - - String vespaDocContent ="Hello, I a JSON doc."; - String docId = "42"; - - AtomicInteger requestsReceived = new AtomicInteger(0); - - // This is the fake server, checks that DATA_FORMAT header is set properly. - ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> { - Header header = post.getFirstHeader(Headers.DATA_FORMAT); - if (requestsReceived.incrementAndGet() == 1) { - // This is handshake, it is not json. - assert (header == null); - return httpResponse("clientId", "3"); - } - assertNotNull(header); - assertEquals(FeedParams.DataFormat.JSON_UTF8.name(), header.getValue()); - // Test is done. - return httpResponse("clientId", "3"); - }); - - ApacheGatewayConnection apacheGatewayConnection = - new ApacheGatewayConnection( - endpoint, - feedParams, - clusterSpecificRoute, - connectionParams, - mockFactory, - "clientId", - Clock.systemUTC()); - apacheGatewayConnection.connect(); - apacheGatewayConnection.handshake(); - - documents.add(createDoc(docId, vespaDocContent, true)); - - apacheGatewayConnection.write(documents); - } - - @Test - public void testZipAndCreateEntity() throws IOException { - String testString = "Hello world"; - InputStream stream = new ByteArrayInputStream(testString.getBytes(StandardCharsets.UTF_8)); - // Send in test data to method. - InputStreamEntity inputStreamEntity = ApacheGatewayConnection.zipAndCreateEntity(stream); - // Verify zipped data by comparing unzipped data with test data. - String rawContent = TestUtils.zipStreamToString(inputStreamEntity.getContent()); - assertEquals(testString, rawContent); - } - - /** - * Mocks the HttpClient, and verifies that the compressed data is sent. - */ - @Test - public void testCompressedWriteOperations() throws Exception { - Endpoint endpoint = Endpoint.create("localhost", 666, false); - FeedParams feedParams = new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.XML_UTF8).build(); - String clusterSpecificRoute = ""; - ConnectionParams connectionParams = new ConnectionParams.Builder().setUseCompression(true).build(); - List<Document> documents = new ArrayList<>(); - - String vespaDocContent ="Hello, I am the document data."; - String docId = "42"; - - Document doc = createDoc(docId, vespaDocContent, false); - - // When sending data on http client, check if it is compressed. If compressed, unzip, check result, - // and count down latch. - ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> { - Header header = post.getFirstHeader("Content-Encoding"); - if (header != null && header.getValue().equals("gzip")) { - final String rawContent = TestUtils.zipStreamToString(post.getEntity().getContent()); - final String vespaHeaderText = "<vespafeed>\n"; - final String vespaFooterText = "</vespafeed>\n"; - - assertEquals(doc.getOperationId() + " 38\n" + vespaHeaderText + vespaDocContent + "\n" + vespaFooterText, - rawContent); - } - return httpResponse("clientId", "3"); - }); - - StatusLine statusLineMock = mock(StatusLine.class); - when(statusLineMock.getStatusCode()).thenReturn(200); - - ApacheGatewayConnection apacheGatewayConnection = - new ApacheGatewayConnection( - endpoint, - feedParams, - clusterSpecificRoute, - connectionParams, - mockFactory, - "clientId", - Clock.systemUTC()); - apacheGatewayConnection.connect(); - apacheGatewayConnection.handshake(); - - documents.add(doc); - - apacheGatewayConnection.write(documents); - } - - @Test - public void dynamic_headers_are_added_to_the_response() throws IOException, ServerResponseException { - ConnectionParams.HeaderProvider headerProvider = mock(ConnectionParams.HeaderProvider.class); - when(headerProvider.getHeaderValue()) - .thenReturn("v1") - .thenReturn("v2") - .thenReturn("v3"); - - ConnectionParams connectionParams = new ConnectionParams.Builder() - .addDynamicHeader("foo", headerProvider) - .build(); - - AtomicInteger counter = new AtomicInteger(1); - ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> { - Header[] fooHeader = post.getHeaders("foo"); - assertEquals(1, fooHeader.length); - assertEquals("foo", fooHeader[0].getName()); - assertEquals("v" + counter.getAndIncrement(), fooHeader[0].getValue()); - return httpResponse("clientId", "3"); - - }); - - ApacheGatewayConnection apacheGatewayConnection = - new ApacheGatewayConnection( - Endpoint.create("localhost", 666, false), - new FeedParams.Builder().build(), - "", - connectionParams, - mockFactory, - "clientId", - Clock.systemUTC()); - apacheGatewayConnection.connect(); - apacheGatewayConnection.handshake(); - - List<Document> documents = new ArrayList<>(); - documents.add(createDoc("42", "content", true)); - apacheGatewayConnection.write(documents); - apacheGatewayConnection.write(documents); - - verify(headerProvider, times(3)).getHeaderValue(); // 1x connect(), 2x writeOperations() - } - - @Test - public void detailed_error_message_is_extracted_from_error_responses_with_json() throws IOException, ServerResponseException { - String reasonPhrase = "Unauthorized"; - String errorMessage = "Invalid credentials"; - expectedException.expect(ServerResponseException.class); - expectedException.expectMessage(reasonPhrase + " - " + errorMessage); - - ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> createErrorHttpResponse(401, reasonPhrase, errorMessage)); - - ApacheGatewayConnection apacheGatewayConnection = - new ApacheGatewayConnection( - Endpoint.create("localhost", 666, false), - new FeedParams.Builder().build(), - "", - new ConnectionParams.Builder().build(), - mockFactory, - "clientId", - Clock.systemUTC()); - apacheGatewayConnection.connect(); - apacheGatewayConnection.handshake(); - - apacheGatewayConnection.write(Collections.singletonList(createDoc("42", "content", true))); - } - - private static ApacheGatewayConnection.HttpClientFactory mockHttpClientFactory(HttpExecuteMock httpExecuteMock) throws IOException { - ApacheGatewayConnection.HttpClientFactory mockFactory = - mock(ApacheGatewayConnection.HttpClientFactory.class); - CloseableHttpClient httpClientMock = mock(CloseableHttpClient.class); - when(mockFactory.createClient()).thenReturn(httpClientMock); - when(httpClientMock.execute(any())).thenAnswer((Answer) invocation -> { - Object[] args = invocation.getArguments(); - HttpPost post = (HttpPost) args[0]; - return httpExecuteMock.execute(post); - }); - return mockFactory; - } - - @FunctionalInterface private interface HttpExecuteMock { - HttpResponse execute(HttpPost httpPost) throws IOException; - } - - private Document createDoc(String docId, String content, boolean useJson) { - return new Document(docId, content.getBytes(), null, Clock.systemUTC().instant()); - } - - private void addMockedHeader(HttpResponse httpResponseMock, String name, String value, HeaderElement[] elements) { - Header header = new Header() { - @Override - public String getName() { - return name; - } - @Override - public String getValue() { - return value; - } - @Override - public HeaderElement[] getElements() throws ParseException { - return elements; - } - }; - when(httpResponseMock.getFirstHeader(name)).thenReturn(header); - } - - private HttpResponse httpResponse(String sessionIdInResult, String version) throws IOException { - CloseableHttpResponse httpResponseMock = mock(CloseableHttpResponse.class); - - StatusLine statusLineMock = mock(StatusLine.class); - when(httpResponseMock.getStatusLine()).thenReturn(statusLineMock); - when(statusLineMock.getStatusCode()).thenReturn(200); - - addMockedHeader(httpResponseMock, Headers.SESSION_ID, sessionIdInResult, null); - addMockedHeader(httpResponseMock, Headers.VERSION, version, null); - HeaderElement[] headerElements = new HeaderElement[1]; - headerElements[0] = mock(HeaderElement.class); - - final HttpEntity httpEntityMock = mock(HttpEntity.class); - when(httpResponseMock.getEntity()).thenReturn(httpEntityMock); - - final InputStream inputs = new ByteArrayInputStream("fake response data".getBytes()); - - when(httpEntityMock.getContent()).thenReturn(inputs); - return httpResponseMock; - } - - private static HttpResponse createErrorHttpResponse(int statusCode, String reasonPhrase, String message) throws IOException { - CloseableHttpResponse response = mock(CloseableHttpResponse.class); - - StatusLine statusLine = mock(StatusLine.class); - when(statusLine.getStatusCode()).thenReturn(statusCode); - when(statusLine.getReasonPhrase()).thenReturn(reasonPhrase); - when(response.getStatusLine()).thenReturn(statusLine); - - HttpEntity httpEntity = mock(HttpEntity.class); - when(httpEntity.getContentType()).thenReturn(new BasicHeader("Content-Type", "application/json")); - String json = String.format("{\"message\": \"%s\"}", message); - when(httpEntity.getContent()).thenReturn(new ByteArrayInputStream(json.getBytes())); - when(response.getEntity()).thenReturn(httpEntity); - return response; - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStreamTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStreamTest.java deleted file mode 100644 index 9845768280e..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStreamTest.java +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.core.communication.ByteBufferInputStream; -import org.junit.Test; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.20 - */ -public class ByteBufferInputStreamTest { - - private static ByteBuffer[] getAbcde() { - ByteBuffer[] buffers = new ByteBuffer[5]; - buffers[0] = ByteBuffer.wrap("a".getBytes(StandardCharsets.UTF_8)); - buffers[1] = ByteBuffer.wrap("b".getBytes(StandardCharsets.UTF_8)); - buffers[2] = ByteBuffer.wrap("c".getBytes(StandardCharsets.UTF_8)); - buffers[3] = ByteBuffer.wrap("d".getBytes(StandardCharsets.UTF_8)); - buffers[4] = ByteBuffer.wrap("e".getBytes(StandardCharsets.UTF_8)); - return buffers; - } - - @Test - public void requireThatExhaustedBufferWorks() throws IOException { - ByteBuffer[] buffers = getAbcde(); - buffers[2].get(); - ByteBufferInputStream in = new ByteBufferInputStream(buffers); - - byte[] out = new byte[100]; - int pos = 0; - - final int GUARD = 1000; - int i; - for (i = 0; i < GUARD; i++) { - int r = in.read(); - if (r == -1) { - break; - } - out[pos] = (byte) (0xFF & r); - ++pos; - } - assertTrue(i < GUARD); - assertThat(pos, is(4)); - - String outString = new String(out, 0, pos, StandardCharsets.UTF_8); - assertThat(outString, equalTo("abde")); - - } - - @Test - public void requireThatBulkReadWorks() throws IOException { - ByteBuffer[] buffers = getAbcde(); - ByteBufferInputStream in = new ByteBufferInputStream(buffers); - - byte[] out = new byte[100]; - int pos = 0; - - final int GUARD = 1000; - int i; - for (i = 0; i < GUARD; i++) { - int numReadNow; - if (i == 0) { - numReadNow = in.read(out); - } else { - numReadNow = in.read(out, pos, (out.length - pos)); - } - if (numReadNow == -1) { - break; - } - pos += numReadNow; - } - assertTrue(i < GUARD); - assertThat(pos, is(5)); - - String outString = new String(out, 0, pos, StandardCharsets.UTF_8); - assertThat(outString, equalTo("abcde")); - } - - @Test - public void requireThatSingleByteReadWorks() throws IOException { - ByteBuffer[] buffers = getAbcde(); - ByteBufferInputStream in = new ByteBufferInputStream(buffers); - - byte[] out = new byte[100]; - int pos = 0; - - final int GUARD = 1000; - int i; - for (i = 0; i < GUARD; i++) { - int r = in.read(); - if (r == -1) { - break; - } - out[pos] = (byte) (0xFF & r); - ++pos; - } - assertTrue(i < GUARD); - assertThat(pos, is(5)); - - String outString = new String(out, 0, pos, StandardCharsets.UTF_8); - assertThat(outString, equalTo("abcde")); - } - - @Test - public void requireThatMarkIsNotSupported() throws IOException { - ByteBuffer[] buffers = getAbcde(); - ByteBufferInputStream in = new ByteBufferInputStream(buffers); - assertThat(in.markSupported(), is(false)); - in.mark(0); //a no-op - } - - @Test(expected = IOException.class) - public void requireThatResetFails() throws IOException { - ByteBuffer[] buffers = getAbcde(); - ByteBufferInputStream in = new ByteBufferInputStream(buffers); - in.reset(); - } - - @Test(expected = IOException.class) - public void requireThatSkipFails() throws IOException { - ByteBuffer[] buffers = getAbcde(); - ByteBufferInputStream in = new ByteBufferInputStream(buffers); - in.skip(1L); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/CloseableQTestCase.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/CloseableQTestCase.java deleted file mode 100644 index 227f4a5239c..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/CloseableQTestCase.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.core.Document; -import org.junit.Test; - -import java.time.Clock; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -public class CloseableQTestCase { - - @Test - public void requestThatPutIsInterruptedOnClose() throws InterruptedException { - Clock clock = Clock.systemUTC(); - DocumentQueue q = new DocumentQueue(1, clock); - q.put(new Document("id", null, "data", null, clock.instant()), false); - Thread t = new Thread(new Runnable() { - @Override - public void run() { - try { - Thread.sleep(3000); - } catch (InterruptedException e) { - } - q.close(); - q.clear(); - } - }); - t.start(); - try { - q.put(new Document("id2", null, "data2", null, Clock.systemUTC().instant()), false); - fail("This shouldn't have worked."); - } catch (IllegalStateException ise) { - // ok! - } - try { - t.join(); - } catch (InterruptedException e) { - } - } - - @Test - public void requireThatSelfIsUnbounded() throws InterruptedException { - DocumentQueue q = new DocumentQueue(1, Clock.systemUTC()); - q.put(new Document("1", null, "data", null, Clock.systemUTC().instant()), true); - q.put(new Document("2", null, "data", null, Clock.systemUTC().instant()), true); - q.put(new Document("3", null, "data", null, Clock.systemUTC().instant()), true); - assertEquals(3, q.size()); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueueTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueueTest.java deleted file mode 100644 index 78ccfed3dbc..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueueTest.java +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.core.EndpointResult; -import com.yahoo.vespa.http.client.core.operationProcessor.OperationProcessor; -import org.junit.Test; - -import java.time.Clock; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; - -/** - * @author Einar M R Rosenvinge - */ -public class EndpointResultQueueTest { - - @Test - public void testBasics() { - Endpoint endpoint = Endpoint.create("a"); - - GatewayConnection connection = new DryRunGatewayConnection(endpoint, Clock.systemUTC()); - OperationProcessor mockAggregator = mock(OperationProcessor.class); - final AtomicInteger resultCount = new AtomicInteger(0); - - doAnswer(invocationOnMock -> { - resultCount.getAndIncrement(); - return null; - }).when(mockAggregator).resultReceived(any(), eq(0)); - - EndpointResultQueue q = new EndpointResultQueue( - mockAggregator, endpoint, 0, new ScheduledThreadPoolExecutor(1), 100L * 1000L); - - q.operationSent("op1", connection); - assertThat(q.getPendingSize(), is(1)); - q.operationSent("op2", connection); - assertThat(q.getPendingSize(), is(2)); - q.operationSent("op3", connection); - assertThat(q.getPendingSize(), is(3)); - q.resultReceived(new EndpointResult("op1", new Result.Detail(endpoint)), 0); - assertThat(q.getPendingSize(), is(2)); - q.resultReceived(new EndpointResult("op2", new Result.Detail(endpoint)), 0); - assertThat(q.getPendingSize(), is(1)); - q.resultReceived(new EndpointResult("op3", new Result.Detail(endpoint)), 0); - assertThat(q.getPendingSize(), is(0)); - q.resultReceived(new EndpointResult("op1", new Result.Detail(endpoint)), 0); - assertThat(q.getPendingSize(), is(0)); - q.resultReceived(new EndpointResult("abc", new Result.Detail(endpoint)), 0); - assertThat(q.getPendingSize(), is(0)); - - assertThat(resultCount.get(), is(5)); - - q.operationSent("op4", connection); - assertThat(q.getPendingSize(), is(1)); - q.operationSent("op5", connection); - assertThat(q.getPendingSize(), is(2)); - - q.failPending(new RuntimeException()); - - assertThat(resultCount.get(), is(7)); - } - - - @Test - public void testTimeout() throws InterruptedException { - Endpoint endpoint = Endpoint.create("a"); - OperationProcessor mockAggregator = mock(OperationProcessor.class); - CountDownLatch latch = new CountDownLatch(1); - doAnswer(invocationOnMock -> { - latch.countDown(); - return null; - }).when(mockAggregator).resultReceived(any(), eq(0)); - EndpointResultQueue q = new EndpointResultQueue( - mockAggregator, endpoint, 0, new ScheduledThreadPoolExecutor(1), 100L); - q.operationSent("1234", new DryRunGatewayConnection(endpoint, Clock.systemUTC())); - assert(latch.await(120, TimeUnit.SECONDS)); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottlerTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottlerTest.java deleted file mode 100644 index 780947b525a..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottlerTest.java +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import org.junit.Before; -import org.junit.Test; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; - - -public class GatewayThrottlerTest { - - GatewayThrottler gatewayThrottler; - long lastSleepValue = 0; - - @Before - public void before() { - gatewayThrottler = new GatewayThrottler(900) { - @Override - protected void sleepMs(long sleepTime) { - lastSleepValue = sleepTime; - } - }; - } - - @Test - public void noSleepOnNormalCase() { - gatewayThrottler.handleCall(0); - gatewayThrottler.handleCall(0); - assertThat(lastSleepValue, is(0L)); - } - - @Test - public void increastingSleepTime() { - gatewayThrottler.handleCall(1); - long sleepTime1 = lastSleepValue; - gatewayThrottler.handleCall(1); - long sleepTime2 = lastSleepValue; - assertTrue(sleepTime1 > 0); - assertTrue(sleepTime2 > sleepTime1); - int x; - // Check for max value of sleep time. - for (x = 0 ; x < 10000; x++) { - long prevSleepTime = lastSleepValue; - gatewayThrottler.handleCall(1); - if (prevSleepTime == lastSleepValue) break; - } - assertTrue(x < 5000); - // Check that it goes down back to zero when no errors. - for (x = 0 ; x < 10000; x++) { - gatewayThrottler.handleCall(0); - if (lastSleepValue == 0) break; - } - assertTrue(x < 5000); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java deleted file mode 100644 index 9984e43374a..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.core.OperationProcessorTester; -import com.yahoo.vespa.http.client.core.ServerResponseException; -import org.junit.Test; - -import java.io.IOException; -import java.time.Duration; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotSame; - -/** - * TODO: Migrate IOThreadTests here. - * - * @author bratseth - */ -public class IOThreadTest { - - @Test - public void testSuccessfulWriting() { - OperationProcessorTester tester = new OperationProcessorTester(); - assertEquals(0, tester.incomplete()); - assertEquals(0, tester.success()); - assertEquals(0, tester.failures()); - tester.send("doc1"); - tester.send("doc2"); - tester.send("doc3"); - assertEquals(3, tester.incomplete()); - assertEquals(0, tester.success()); - assertEquals(0, tester.failures()); - tester.tick(1); // connect - assertEquals(3, tester.incomplete()); - tester.tick(1); // sync - assertEquals(3, tester.incomplete()); - tester.tick(1); // process queue - assertEquals(0, tester.incomplete()); - assertEquals(3, tester.success()); - assertEquals(0, tester.failures()); - } - - @Test - public void testFatalExceptionOnHandshake() { - OperationProcessorTester tester = new OperationProcessorTester(); - IOThread ioThread = tester.getSingleIOThread(); - DryRunGatewayConnection firstConnection = (DryRunGatewayConnection)ioThread.currentConnection(); - firstConnection.throwOnHandshake(new ServerResponseException(403, "Not authorized")); - - tester.send("doc1"); - tester.send("doc2"); - tester.send("doc3"); - tester.tick(3); - assertEquals(0, tester.incomplete()); - assertEquals(0, ioThread.resultQueue().getPendingSize()); - assertEquals(0, tester.success()); - assertEquals(3, tester.failures()); - } - - @Test - public void testExceptionOnHandshake() { - OperationProcessorTester tester = new OperationProcessorTester(); - IOThread ioThread = tester.getSingleIOThread(); - DryRunGatewayConnection firstConnection = (DryRunGatewayConnection)ioThread.currentConnection(); - firstConnection.throwOnHandshake(new ServerResponseException(418, "I'm a teapot")); - - tester.send("doc1"); - tester.tick(3); - assertEquals(1, tester.incomplete()); - assertEquals(0, ioThread.resultQueue().getPendingSize()); - assertEquals(0, tester.success()); - assertEquals("Awaiting retry", 0, tester.failures()); - } - - @Test - public void testExceptionOnWrite() { - OperationProcessorTester tester = new OperationProcessorTester(); - IOThread ioThread = tester.getSingleIOThread(); - DryRunGatewayConnection firstConnection = (DryRunGatewayConnection)ioThread.currentConnection(); - firstConnection.throwOnWrite(new IOException("Test failure")); - - tester.send("doc1"); - tester.tick(3); - assertEquals(1, tester.incomplete()); - assertEquals(0, ioThread.resultQueue().getPendingSize()); - assertEquals(0, tester.success()); - assertEquals("Awaiting retry since write exceptions is a transient failure", - 0, tester.failures()); - } - - @Test - public void testPollingOldConnections() { - OperationProcessorTester tester = new OperationProcessorTester(); - tester.tick(3); - - IOThread ioThread = tester.getSingleIOThread(); - DryRunGatewayConnection firstConnection = (DryRunGatewayConnection)ioThread.currentConnection(); - assertEquals(0, ioThread.oldConnections().size()); - - firstConnection.hold(true); - tester.send("doc1"); - tester.tick(1); - - tester.clock().advance(Duration.ofSeconds(31)); // Default connection ttl is 30 - tester.tick(3); - - assertEquals(1, ioThread.oldConnections().size()); - assertEquals(firstConnection, ioThread.oldConnections().get(0)); - assertNotSame(firstConnection, ioThread.currentConnection()); - assertEquals(31, firstConnection.lastPollTime().toEpochMilli() / 1000); - - // Check old connection poll pattern (exponential backoff) - assertLastPollTimeWhenAdvancing(31, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - - tester.clock().advance(Duration.ofSeconds(200)); - tester.tick(1); - assertEquals("Old connection is eventually removed", 0, ioThread.oldConnections().size()); - } - - @Test - public void old_connections_are_closed_early_if_no_inflight_operations() { - OperationProcessorTester tester = new OperationProcessorTester(); - tester.tick(3); - - IOThread ioThread = tester.getSingleIOThread(); - DryRunGatewayConnection firstConnection = (DryRunGatewayConnection)ioThread.currentConnection(); - assertEquals(0, ioThread.oldConnections().size()); - - firstConnection.hold(true); // do not send result for doc1 in next http response - tester.send("doc1"); - tester.tick(1); - tester.clock().advance(Duration.ofSeconds(31)); // Default connection TTL + 1 - tester.tick(1); - assertEquals(1, ioThread.oldConnections().size()); - - firstConnection.hold(false); // send result for both doc1 and doc2 in next http response - tester.send("doc2"); - tester.tick(1); - assertEquals(1, ioThread.oldConnections().size()); - tester.clock().advance(Duration.ofSeconds(2)); - tester.tick(3); - assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester); - assertEquals(0, ioThread.oldConnections().size()); - } - - private void assertLastPollTimeWhenAdvancing(int lastPollTimeSeconds, - int advanceSeconds, - DryRunGatewayConnection connection, - OperationProcessorTester tester) { - tester.clock().advance(Duration.ofSeconds(advanceSeconds)); - tester.tick(1); - assertEquals(lastPollTimeSeconds, connection.lastPollTime().toEpochMilli() / 1000); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlockerTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlockerTest.java deleted file mode 100644 index cd5da6b8589..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlockerTest.java +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.operationProcessor; - -import com.yahoo.vespa.http.client.core.operationProcessor.ConcurrentDocumentOperationBlocker; -import org.junit.Before; -import org.junit.Test; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; - -public class ConcurrentDocumentOperationBlockerTest { - - final ConcurrentDocumentOperationBlocker blocker = new ConcurrentDocumentOperationBlocker(); - final CountDownLatch latch = new CountDownLatch(1); - - @Before - public void setup() throws InterruptedException { - blocker.setMaxConcurrency(2); - blocker.startOperation(); - assertThat(blocker.availablePermits(), is(1)); - blocker.startOperation(); - } - - private void spawnThreadPushOperationThenCountDown() { - new Thread(() -> { - try { - blocker.startOperation(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - latch.countDown(); - }).start(); - } - - @Test - public void testBasics() throws InterruptedException { - spawnThreadPushOperationThenCountDown(); - assertFalse(latch.await(10, TimeUnit.MILLISECONDS)); - blocker.operationDone(); - assertTrue(latch.await(120, TimeUnit.SECONDS)); - } - - @Test - public void testResizeLarger() throws InterruptedException { - spawnThreadPushOperationThenCountDown(); - assertFalse(latch.await(10, TimeUnit.MILLISECONDS)); - blocker.setMaxConcurrency(3); - assertTrue(latch.await(120, TimeUnit.SECONDS)); - } - - @Test - public void testResizeSmaller() throws InterruptedException { - spawnThreadPushOperationThenCountDown(); - blocker.setMaxConcurrency(1); - blocker.operationDone(); - assertFalse(latch.await(10, TimeUnit.MILLISECONDS)); - blocker.operationDone(); - assertTrue(latch.await(120, TimeUnit.SECONDS)); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottlerTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottlerTest.java deleted file mode 100644 index 82337cd4dcf..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottlerTest.java +++ /dev/null @@ -1,294 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.operationProcessor; - -import com.yahoo.vespa.http.client.ManualClock; -import com.yahoo.vespa.http.client.core.ThrottlePolicy; -import org.junit.Test; - -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; -import java.util.Random; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.anyDouble; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class IncompleteResultsThrottlerTest { - - @Test - public void simpleStaticQueueSizeTest() { - IncompleteResultsThrottler incompleteResultsThrottler = new IncompleteResultsThrottler(2, 2, null, null); - assertEquals(0, incompleteResultsThrottler.waitingThreads()); - incompleteResultsThrottler.operationStart(); - incompleteResultsThrottler.operationStart(); - assertEquals(2, incompleteResultsThrottler.waitingThreads()); - incompleteResultsThrottler.resultReady(true); - assertEquals(1, incompleteResultsThrottler.waitingThreads()); - incompleteResultsThrottler.resultReady(true); - assertEquals(0, incompleteResultsThrottler.waitingThreads()); - } - - /** - * Simulate running requests. - * @param clientCount number of parallel clients. - * @param breakPoint how many requests the server should handle in parallel before it gets slower. - * @param simulationTimeMs how many ms to simulate. - * @return median queue length. - */ - int getAverageQueue(int clientCount, int breakPoint, int simulationTimeMs) { - ManualClock clock = new ManualClock(Instant.ofEpochMilli(0)); - - ArrayList<IncompleteResultsThrottler> incompleteResultsThrottlers = new ArrayList<>(); - - MockServer mockServer = new MockServer(breakPoint); - for (int x = 0; x < clientCount; x++) { - IncompleteResultsThrottler incompleteResultsThrottler = - new IncompleteResultsThrottler(10, 50000, clock, new ThrottlePolicy()); - incompleteResultsThrottlers.add(incompleteResultsThrottler); - } - long sum = 0; - long samples = 0; - - for (long time = 0; time < simulationTimeMs; time++) { - // Fast forward, if we can. If all clients are blocked, we can move to the time when the server has a - // request that is finished. - boolean fastForward = true; - for (int x = 0; x < clientCount; x++) { - if (incompleteResultsThrottlers.get(x).availableCapacity() > 0 ) { - fastForward = false; - break; - } - } - if (fastForward) { - time = mockServer.nextRequestFinished(); - } - clock.setInstant(Instant.ofEpochMilli(time)); - mockServer.moveTime(clock.instant().toEpochMilli()); - for (int y = 0; y < clientCount; y++) { - // Fill up, but don't block as that would stop the simulation. - while (incompleteResultsThrottlers.get(y).availableCapacity() > 0) { - incompleteResultsThrottlers.get(y).operationStart(); - mockServer.newRequest(incompleteResultsThrottlers.get(y)); - } - } - // Don't take the first iterations into account as the system is eagerly learning. - if (time > 60*1000) { - sum += mockServer.messageDoneByTime.size(); - samples ++; - } - } - return (int)(sum/samples); - } - - private void testAndPrintVariousClientSizes(int breakPoint) { - final int sampleRuns = 6; - final int maxParallelClients = 4; - final int minParallelClients = 1; - final int simulationTimeMs = 400000; - System.out.print("\nBreakpoint is " + breakPoint + ", average queue on server:"); - int[][] resultQueuesAverage = new int[maxParallelClients][sampleRuns]; - for (int clientNo = minParallelClients; clientNo <= maxParallelClients; clientNo++) { - System.out.print("\nNow with " + clientNo + " parallel clients:"); - long sum = 0; - for (int x = 0; x < sampleRuns; x++) { - resultQueuesAverage[clientNo-minParallelClients][x] = getAverageQueue(1 + x, breakPoint, simulationTimeMs); - System.out.print(" " + resultQueuesAverage[clientNo-minParallelClients][x]); - sum += resultQueuesAverage[clientNo-minParallelClients][x]; - } - System.out.print(" average is " + sum/sampleRuns); - Arrays.sort(resultQueuesAverage[clientNo - minParallelClients]); - int median = resultQueuesAverage[clientNo - minParallelClients][sampleRuns/2]; - System.out.print(" median is " + median); - System.out.print(" min " + resultQueuesAverage[clientNo - minParallelClients][0]); - System.out.print(" max " + resultQueuesAverage[clientNo - minParallelClients][sampleRuns - 1]); - assertTrue(median < 2 * breakPoint + 200); - assertTrue(median > breakPoint / 10); - } - } - - @Test - public void testVariousBreakpoints() { - testAndPrintVariousClientSizes(200); - testAndPrintVariousClientSizes(1000); - } - - List<Thread> threads = new ArrayList<>(); - - private void postOperations(int count, final IncompleteResultsThrottler throttler) { - for (int i = 0; i < count; i++) { - Thread thread = new Thread(()->throttler.operationStart()); - thread.start(); - threads.add(thread); - } - } - - private void waitForThreads() throws InterruptedException { - while(!threads.isEmpty()) { - threads.remove(0).join(); - } - } - - private void postSuccesses(int count, final IncompleteResultsThrottler throttler) { - for (int i = 0; i < count; i++) { - throttler.resultReady(true); - } - } - - private void moveToNextCycle(final IncompleteResultsThrottler throttler, ManualClock clock) - throws InterruptedException { - waitForThreads(); - // Enter an adaption phase, we don't care about this phase. - clock.advance(Duration.ofMillis(throttler.phaseSizeMs)); - throttler.operationStart(); - throttler.resultReady(false); - // Now enter the real next phase. - clock.advance(Duration.ofMillis(throttler.phaseSizeMs)); - throttler.operationStart(); - throttler.resultReady(false); - } - - @Test - public void testInteractionWithPolicyByMockingPolicy() throws InterruptedException { - ManualClock clock = new ManualClock(Instant.ofEpochMilli(0)); - final int MAX_SIZE = 1000; - final int MORE_THAN_MAX_SIZE = MAX_SIZE + 20; - final int SIZE_AFTER_CYCLE_FIRST = 30; - final int SIZE_AFTER_CYCLE_SECOND = 5000; - ThrottlePolicy policy = mock(ThrottlePolicy.class); - IncompleteResultsThrottler incompleteResultsThrottler = - new IncompleteResultsThrottler(2, MAX_SIZE, clock, policy); - long bucketSizeMs = incompleteResultsThrottler.phaseSizeMs; - - // Cycle 1 - Algorithm has fixed value for max-in-flight: INITIAL_MAX_IN_FLIGHT_VALUE. - // We post a few operations, not all finishing in this cycle. We explicitly do not fill the window - // size to test the argument about any requests blocked. - assertEquals(IncompleteResultsThrottler.INITIAL_MAX_IN_FLIGHT_VALUE, - incompleteResultsThrottler.availableCapacity()); - postOperations(20, incompleteResultsThrottler); - postSuccesses(15, incompleteResultsThrottler); - moveToNextCycle(incompleteResultsThrottler, clock); - - - // Cycle 2 - Algorithm has fixed value also for second iteration: SECOND_MAX_IN_FLIGHT_VALUE. - // Test verifies that this value is used, and insert a value to be used for next phase SIZE_AFTER_CYCLE_FIRST. - assertEquals("5 slots already taken earlier", - IncompleteResultsThrottler.SECOND_MAX_IN_FLIGHT_VALUE - 5, - incompleteResultsThrottler.availableCapacity()); - postSuccesses(5, incompleteResultsThrottler); - when(policy.calcNewMaxInFlight( - anyDouble(), // Max performance change - eq(5), //numOk - eq(15), // previousNumOk - eq(IncompleteResultsThrottler.INITIAL_MAX_IN_FLIGHT_VALUE), // previous size - eq(IncompleteResultsThrottler.SECOND_MAX_IN_FLIGHT_VALUE), // current size - eq(false))) // is any request blocked, should be false since we only posted 20 docs. - .thenReturn(SIZE_AFTER_CYCLE_FIRST); - moveToNextCycle(incompleteResultsThrottler, clock); - - // Cycle 3 - Test that value set in previous phase is used. Now return a very large number. - // However, this number should be cropped by the system (tested in next cycle). - assertEquals(SIZE_AFTER_CYCLE_FIRST, incompleteResultsThrottler.availableCapacity()); - postOperations(MORE_THAN_MAX_SIZE, incompleteResultsThrottler); - postSuccesses(MORE_THAN_MAX_SIZE, incompleteResultsThrottler); - when(policy.calcNewMaxInFlight( - anyDouble(), // Max performance change - eq(MORE_THAN_MAX_SIZE), //numOk - eq(5), // previousNumOk - eq(IncompleteResultsThrottler.SECOND_MAX_IN_FLIGHT_VALUE), // previous size - eq(SIZE_AFTER_CYCLE_FIRST),// current size - eq(true))) // is any request blocked, should be true since we posted MORE_THAN_MAX_SIZE docs. - .thenReturn(SIZE_AFTER_CYCLE_SECOND); - moveToNextCycle(incompleteResultsThrottler, clock); - - // Cycle 4 - Test that the large number from previous cycle is cropped and that max value is used instead. - assertEquals(MAX_SIZE, incompleteResultsThrottler.availableCapacity()); - } - - private long inversesU(int size, int sweetSpot) { - // Peak performance at sweetSPot. - int distance = Math.abs(sweetSpot - size); - return 1 + 20 * distance; - } - - /** - * A mock 'gateway' this is slower with more requests in-flight. It starts to become really much slower at - * 'breakPoint' number of parallel requests. - */ - class MockServer { - final LinkedList<Tuple2<Long, IncompleteResultsThrottler> > messageDoneByTime = new LinkedList<>(); - final int breakPoint; - final Random random = new Random(); - long time = 0; - - MockServer(int breakPoint) { - this.breakPoint = breakPoint; - } - - /** - * Figures out when next processed data will be ready. - * @return time in ms for next request to be finished. - */ - long nextRequestFinished() { - if (messageDoneByTime.isEmpty()) { - return Integer.MAX_VALUE; - } - return messageDoneByTime.peek().first; - } - - /** - * Advance simulation time and call finished on any requests. - * @param time to move to - */ - void moveTime(long time) { - this.time = time; - while (!messageDoneByTime.isEmpty() && messageDoneByTime.peek().first <= time) { - messageDoneByTime.pop().second.resultReady(true); - } - } - - /** - * New request. - * @param blocker do callback on blocker when request is done. - */ - void newRequest(IncompleteResultsThrottler blocker) { - long nextTime = (long)(20 + 0.1 * messageDoneByTime.size()); - - if (messageDoneByTime.size() > breakPoint) { - nextTime += (long) (40 + (random.nextDouble()) * 0.01 * messageDoneByTime.size()* messageDoneByTime.size()); - } - nextTime += time + random.nextInt()%4; - messageDoneByTime.push(new Tuple2<>(nextTime, blocker)); - } - } - - private static class Tuple2<T1, T2> { - - public final T1 first; - public final T2 second; - - public Tuple2(final T1 first, final T2 second) { - this.first = first; - this.second = second; - } - - @Override - public int hashCode() { throw new UnsupportedOperationException(); } - - @Override - public boolean equals(final Object obj) { throw new UnsupportedOperationException(); } - - @Override - public String toString() { - return "Tuple2(" + first + ", " + second + ")"; - } - - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessorTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessorTest.java deleted file mode 100644 index 7799f6089db..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessorTest.java +++ /dev/null @@ -1,438 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.operationProcessor; - -import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.EndpointResult; -import org.junit.Test; - -import java.time.Clock; -import java.util.ArrayDeque; -import java.util.Queue; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author Einar M R Rosenvinge - */ -public class OperationProcessorTest { - - final Queue<Result> queue = new ArrayDeque<>(); - final Document doc1 = new Document("id:a:type::b", null, "data doc 1", null, Clock.systemUTC().instant()); - final Document doc1b = new Document("id:a:type::b", null, "data doc 1b", null, Clock.systemUTC().instant()); - final Document doc2 = new Document("id:a:type::b2", null, "data doc 2", null, Clock.systemUTC().instant()); - final Document doc3 = new Document("id:a:type::b3", null, "data doc 3", null, Clock.systemUTC().instant()); - - @Test - public void testBasic() { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .build(); - - OperationProcessor q = new OperationProcessor( - new IncompleteResultsThrottler(1000, 1000, null, null), - (docId, documentResult) -> queue.add(documentResult), - sessionParams, null, Clock.systemUTC()); - - - q.resultReceived(new EndpointResult("foo", new Result.Detail(null)), 0); - assertEquals(0, queue.size()); - - - q.sendDocument(doc1); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("d"))), 3); - assertEquals(1, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("e"))), 0); - assertEquals(1, queue.size()); - - //check a, b, c, d - Result aggregated = queue.poll(); - assertEquals("id:a:type::b", aggregated.getDocumentId()); - assertEquals(4, aggregated.getDetails().size()); - assertEquals("a", aggregated.getDetails().get(0).getEndpoint().getHostname()); - assertEquals("b", aggregated.getDetails().get(1).getEndpoint().getHostname()); - assertEquals("c", aggregated.getDetails().get(2).getEndpoint().getHostname()); - assertEquals("d", aggregated.getDetails().get(3).getEndpoint().getHostname()); - assertEquals("data doc 1", aggregated.getDocumentDataAsCharSequence().toString()); - - assertEquals(0, queue.size()); - - q.sendDocument(doc2); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("d"))), 3); - assertEquals(1, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("e"))), 0); - assertEquals(1, queue.size()); - - // check a, b, c, d - aggregated = queue.poll(); - assertEquals("id:a:type::b2", aggregated.getDocumentId()); - assertEquals(4, aggregated.getDetails().size()); - assertEquals("a", aggregated.getDetails().get(0).getEndpoint().getHostname()); - assertEquals("b", aggregated.getDetails().get(1).getEndpoint().getHostname()); - assertEquals("c", aggregated.getDetails().get(2).getEndpoint().getHostname()); - assertEquals("d", aggregated.getDetails().get(3).getEndpoint().getHostname()); - assertEquals("data doc 2", aggregated.getDocumentDataAsCharSequence().toString()); - - assertEquals(0, queue.size()); - } - - @Test - public void testBlockingOfOperationsTwoEndpoints() { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .setConnectionParams(new ConnectionParams.Builder().build()) - .build(); - OperationProcessor operationProcessor = new OperationProcessor( - new IncompleteResultsThrottler(1000, 1000, null, null), - (docId, documentResult) -> queue.add(documentResult), - sessionParams, null, Clock.systemUTC()); - - operationProcessor.sendDocument(doc1); - operationProcessor.sendDocument(doc1b); - - assertEquals(0, queue.size()); - // Only one operations should be in flight. - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertEquals(0, queue.size()); - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 1); - assertEquals(1, queue.size()); - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertEquals(1, queue.size()); - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 1); - assertEquals(2, queue.size()); - assertEquals(0, operationProcessor.getIncompleteResultQueueSize()); - // This should have no effect. - operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 1); - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 1); - assertEquals(2, queue.size()); - } - - @Test - public void testBlockingOfOperationsToSameDocIdWithTwoOperations() { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .setConnectionParams(new ConnectionParams.Builder().build()) - .build(); - - OperationProcessor operationProcessor = new OperationProcessor( - new IncompleteResultsThrottler(1000, 1000, null, null), - (docId, documentResult) -> queue.add(documentResult), - sessionParams, null, Clock.systemUTC()); - - operationProcessor.sendDocument(doc1); - operationProcessor.sendDocument(doc1b); - - assertEquals(0, queue.size()); - // Only one operations should be in flight. - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); - operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertEquals(1, queue.size()); - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - assertEquals(doc1b.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertEquals(2, queue.size()); - assertEquals(0, operationProcessor.getIncompleteResultQueueSize()); - assertFalse(operationProcessor.oldestIncompleteResultId().isPresent()); - // This should have no effect. - operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertEquals(2, queue.size()); - } - - @Test - public void testBlockingOfOperationsToSameDocIdMany() { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .setConnectionParams(new ConnectionParams.Builder().build()) - .build(); - - OperationProcessor operationProcessor = new OperationProcessor( - new IncompleteResultsThrottler(1000, 1000, null, null), - (docId, documentResult) -> queue.add(documentResult), - sessionParams, null, Clock.systemUTC()); - - Queue<Document> documentQueue = new ArrayDeque<>(); - for (int x = 0; x < 100; x++) { - Document document = new Document("id:a:type::b", null, String.valueOf(x), null, Clock.systemUTC().instant()); - operationProcessor.sendDocument(document); - documentQueue.add(document); - } - - for (int x = 0; x < 100; x++) { - assertEquals(x, queue.size()); - // Only one operations should be in flight. - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - Document document = documentQueue.poll(); - operationProcessor.resultReceived(new EndpointResult(document.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertEquals(x+1, queue.size()); - if (x < 99) { - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - } else { - assertEquals(0, operationProcessor.getIncompleteResultQueueSize()); - } - } - } - - @Test - public void testMixOfBlockingAndNonBlocking() { - Endpoint endpoint = Endpoint.create("localhost"); - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(endpoint).build()) - .setConnectionParams(new ConnectionParams.Builder().build()) - .build(); - - OperationProcessor operationProcessor = new OperationProcessor( - new IncompleteResultsThrottler(1000, 1000, null, null), - (docId, documentResult) -> queue.add(documentResult), - sessionParams, null, Clock.systemUTC()); - - operationProcessor.sendDocument(doc1); - operationProcessor.sendDocument(doc1b); // Blocked - operationProcessor.sendDocument(doc2); - operationProcessor.sendDocument(doc3); - - assertEquals(0, queue.size()); - assertEquals(3, operationProcessor.getIncompleteResultQueueSize()); - assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); - // This should have no effect since it should not be sent. - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(endpoint)), 0); - assertEquals(3, operationProcessor.getIncompleteResultQueueSize()); - assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); - - operationProcessor.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(endpoint)), 0); - assertEquals(2, operationProcessor.getIncompleteResultQueueSize()); - assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); - operationProcessor.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(endpoint)), 0); - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); - operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(endpoint)), 0); - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - assertEquals(doc1b.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(endpoint)), 0); - assertEquals(0, operationProcessor.getIncompleteResultQueueSize()); - assertFalse(operationProcessor.oldestIncompleteResultId().isPresent()); - } - - @Test - public void assertThatDuplicateResultsFromOneClusterWorks() { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .build(); - - OperationProcessor q = new OperationProcessor( - new IncompleteResultsThrottler(1000, 1000, null, null), - (docId, documentResult) -> queue.add(documentResult), - sessionParams, null, Clock.systemUTC()); - - q.sendDocument(doc1); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("b"))), 0); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("c"))), 0); - assertEquals(0, queue.size()); - } - - @Test - public void testMultipleDuplicateDocIds() { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .build(); - - OperationProcessor q = new OperationProcessor( - new IncompleteResultsThrottler(1000, 1000, null, null), - (docId, documentResult) -> queue.add(documentResult), - sessionParams, null, Clock.systemUTC()); - - q.sendDocument(doc1); - assertEquals(0, queue.size()); - q.sendDocument(doc2); - assertEquals(0, queue.size()); - q.sendDocument(doc3); - - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertEquals(1, queue.size()); - - q.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertEquals(1, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertEquals(1, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertEquals(1, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertEquals(2, queue.size()); - - q.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertEquals(2, queue.size()); - - q.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertEquals(2, queue.size()); - - q.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertEquals(3, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertEquals(3, queue.size()); - assertEquals("data doc 1", queue.remove().getDocumentDataAsCharSequence().toString()); - assertEquals("data doc 2", queue.remove().getDocumentDataAsCharSequence().toString()); - assertEquals("data doc 3", queue.remove().getDocumentDataAsCharSequence().toString()); - } - - @Test - public void testWaitBlocks() throws InterruptedException { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .build(); - - OperationProcessor operationProcessor = new OperationProcessor( - new IncompleteResultsThrottler(1, 1, null, null), - (docId, documentResult) -> {}, - sessionParams, null, Clock.systemUTC()); - - operationProcessor.sendDocument(doc1); - - final CountDownLatch started = new CountDownLatch(1); - final CountDownLatch done = new CountDownLatch(1); - - Thread shouldWait = new Thread(()-> { - started.countDown(); - operationProcessor.sendDocument(doc2); - done.countDown(); - }); - shouldWait.start(); - started.await(); - // We want the test to pass fast so we only wait 40mS to see that it is blocking. This might lead to - // some false positives, but that is ok. - assertFalse(done.await(40, TimeUnit.MILLISECONDS)); - operationProcessor.resultReceived( - new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("d"))), 0); - assertTrue(done.await(120, TimeUnit.SECONDS)); - - } - - @Test - public void testSendsResponseToQueuedDocumentOnClose() throws InterruptedException { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .build(); - - ScheduledThreadPoolExecutor executor = mock(ScheduledThreadPoolExecutor.class); - when(executor.awaitTermination(anyLong(), any())).thenReturn(true); - - CountDownLatch countDownLatch = new CountDownLatch(3); - - OperationProcessor operationProcessor = new OperationProcessor( - new IncompleteResultsThrottler(19, 19, null, null), - (docId, documentResult) -> { - countDownLatch.countDown(); - }, - sessionParams, executor, Clock.systemUTC()); - - // Will fail due to bogus host name, but will be retried. - operationProcessor.sendDocument(doc1); - operationProcessor.sendDocument(doc2); - operationProcessor.sendDocument(doc3); - - // Will create fail results. - operationProcessor.close(); - countDownLatch.await(); - } - - @Test - public void unknownHostThrowsExceptionAtConstructionTime() { - try { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("unknown.invalid")).build()) - .build(); - ScheduledThreadPoolExecutor executor = mock(ScheduledThreadPoolExecutor.class); - - CountDownLatch countDownLatch = new CountDownLatch(3); - - new OperationProcessor( - new IncompleteResultsThrottler(19, 19, null, null), - (docId, documentResult) -> { - countDownLatch.countDown(); - }, - sessionParams, executor, Clock.systemUTC()); - - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - assertEquals("Unknown host: unknown.invalid:4080 ssl=false", e.getMessage()); - } - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/handlers/V3MockParsingRequestHandler.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/handlers/V3MockParsingRequestHandler.java deleted file mode 100644 index bf4d4d0800b..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/handlers/V3MockParsingRequestHandler.java +++ /dev/null @@ -1,417 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.handlers; - -import com.yahoo.vespa.http.client.core.Encoder; -import com.yahoo.vespa.http.client.core.ErrorCode; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.OperationStatus; -import org.eclipse.jetty.server.Request; -import org.eclipse.jetty.server.handler.AbstractHandler; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.PrintWriter; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.27 - */ -public class V3MockParsingRequestHandler extends AbstractHandler { - private final int responseCode; - private volatile Scenario scenario; - private final BlockingQueue<CountDownLatch> delayedRequests = new LinkedBlockingQueue<>(); - private final AtomicBoolean delayedResponseShouldBlock = new AtomicBoolean(true); - public final AtomicBoolean badRequestScenarioShouldReturnBadRequest = new AtomicBoolean(false); - private final String name; - private static final AtomicInteger sessionIdGenerator = new AtomicInteger(0); - private AtomicInteger internalCounter = new AtomicInteger(0); - - public enum Scenario { - ALL_OK, RETURN_WRONG_SESSION_ID, - DISCONNECT_IMMEDIATELY, DONT_ACCEPT_VERSION, RETURN_UNEXPECTED_VERSION, - INTERNAL_SERVER_ERROR, COULD_NOT_FEED, MBUS_RETURNED_ERROR, - NEVER_RETURN_ANY_RESULTS, DELAYED_RESPONSE, BAD_REQUEST, SERVER_ERROR_TWICE_THEN_OK, - EXPECT_HIGHEST_PRIORITY_AND_TRACELEVEL_123, CONDITON_NOT_MET - } - - public V3MockParsingRequestHandler() { - this("", HttpServletResponse.SC_OK, Scenario.ALL_OK); - } - - public V3MockParsingRequestHandler(String name) { - this(name, HttpServletResponse.SC_OK, Scenario.ALL_OK); - } - - public V3MockParsingRequestHandler(int responseCode) { - this("", responseCode, Scenario.ALL_OK); - } - - public V3MockParsingRequestHandler(int responseCode, Scenario scenario) { - this("", responseCode, scenario); - } - - public V3MockParsingRequestHandler(String name, int responseCode, Scenario scenario) { - this.name = name; - this.responseCode = responseCode; - this.scenario = scenario; - } - - public void setScenario(Scenario scenario) { - this.scenario = scenario; - } - - @Override - public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { - System.err.println("Server " + name + " got request from: " + request.getHeader(Headers.SESSION_ID)); - switch (scenario) { - case ALL_OK: - allOk(baseRequest, request, response); - break; - case RETURN_WRONG_SESSION_ID: - wrongSessionId(baseRequest, request, response); - break; - case DISCONNECT_IMMEDIATELY: - disconnect(baseRequest, response); - break; - case DONT_ACCEPT_VERSION: - dontAcceptVersion(baseRequest, request, response); - break; - case RETURN_UNEXPECTED_VERSION: - unexpectedVersion(baseRequest, request, response); - break; - case INTERNAL_SERVER_ERROR: - internalServerError(baseRequest, request, response); - break; - case COULD_NOT_FEED: - couldNotFeed(baseRequest, request, response); - break; - case MBUS_RETURNED_ERROR: - mbusReturnedError(baseRequest, request, response); - break; - case NEVER_RETURN_ANY_RESULTS: - neverReturnAnyResults(baseRequest, request, response); - break; - case DELAYED_RESPONSE: - delayedResponse(baseRequest, request, response); - break; - case BAD_REQUEST: - badRequest(baseRequest, request, response); - break; - case SERVER_ERROR_TWICE_THEN_OK: - int state = internalCounter.getAndIncrement(); - if (state >= 2) { - allOk(baseRequest, request, response); - } else { - couldNotFeed(baseRequest, request, response); - } - break; - case EXPECT_HIGHEST_PRIORITY_AND_TRACELEVEL_123: - checkIfSessionThenHighPriorityAndTraceLevel123(request); - allOk(baseRequest, request, response); - break; - case CONDITON_NOT_MET: - conditionNotMetRequest(baseRequest, request, response); - break; - default: - throw new IllegalArgumentException("Test scenario " + scenario + " not supported."); - } - } - - private void checkIfSessionThenHighPriorityAndTraceLevel123(HttpServletRequest request) { - if (request.getHeader(Headers.SESSION_ID) != null) { - assert (request.getHeader(Headers.PRIORITY).equals("HIGHEST")); - assert (request.getHeader(Headers.TRACE_LEVEL).equals("123")); - } - } - - private void conditionNotMetRequest(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - setHeaders(response, sessionId); - response.setStatus(responseCode); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - String operationId; - while ((operationId = readOperationId(request.getInputStream())) != null) { - long lengthToSkip = readByteLength(request.getInputStream()); - while (lengthToSkip > 0) { - long skipped = request.getInputStream().skip(lengthToSkip); - lengthToSkip -= skipped; - } - respondConditionNotMet(responseWriter, operationId); - } - closeChannel(responseWriter); - - } - private void badRequest(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - if (badRequestScenarioShouldReturnBadRequest.get()) { - response.setStatus(HttpServletResponse.SC_BAD_REQUEST); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - BufferedReader reader = new BufferedReader(new InputStreamReader(request.getInputStream())); - while (reader.readLine() != null) { - //consume input, not really needed? - } - reader.close(); - closeChannel(responseWriter); - } else { - allOk(baseRequest, request, response); - } - } - - private void delayedResponse(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - if (delayedResponseShouldBlock.get()) { - CountDownLatch latch = new CountDownLatch(1); - delayedRequests.add(latch); - try { - latch.await(120, TimeUnit.SECONDS); //wait "forever" - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - if (latch.getCount() != 0L) { - throw new RuntimeException("Delayed request handler did not get poke()d."); - } - } else { - } - allOk(baseRequest, request, response); - } - - public void poke() throws InterruptedException { - CountDownLatch latch = delayedRequests.poll(10, TimeUnit.SECONDS); - latch.countDown(); - } - - public void pokeAllAndUnblockFromNowOn() { - delayedResponseShouldBlock.set(false); - while (!delayedRequests.isEmpty()) { - CountDownLatch latch = delayedRequests.remove(); - latch.countDown(); - } - } - - private void allOk(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - setHeaders(response, sessionId); - response.setStatus(responseCode); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - String operationId; - while ((operationId = readOperationId(request.getInputStream())) != null) { - long lengthToSkip = readByteLength(request.getInputStream()); - while (lengthToSkip > 0) { - long skipped = request.getInputStream().skip(lengthToSkip); - lengthToSkip -= skipped; - } - respondOK(responseWriter, operationId); - } - closeChannel(responseWriter); - } - - private void wrongSessionId(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = generateMockSessionId(); - setHeaders(response, sessionId); - response.setStatus(responseCode); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - String operationId; - while ((operationId = readOperationId(request.getInputStream())) != null) { - long lengthToSkip = readByteLength(request.getInputStream()); - while (lengthToSkip > 0) { - long skipped = request.getInputStream().skip(lengthToSkip); - lengthToSkip -= skipped; - } - respondOK(responseWriter, operationId); - } - closeChannel(responseWriter); - } - - private void disconnect(Request baseRequest, HttpServletResponse response) throws IOException { - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - closeChannel(responseWriter); - } - - private void dontAcceptVersion(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - setHeaders(response, sessionId); - response.setStatus(Headers.HTTP_NOT_ACCEPTABLE); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - responseWriter.write("Go away, no such version."); - responseWriter.flush(); - closeChannel(responseWriter); - } - - private void unexpectedVersion(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - response.setHeader(Headers.SESSION_ID, sessionId); - response.setHeader(Headers.VERSION, "12345678"); - response.setStatus(responseCode); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - String operationId; - while ((operationId = readOperationId(request.getInputStream())) != null) { - long lengthToSkip = readByteLength(request.getInputStream()); - while (lengthToSkip > 0) { - long skipped = request.getInputStream().skip(lengthToSkip); - lengthToSkip -= skipped; - } - respondOK(responseWriter, operationId); - } - closeChannel(responseWriter); - } - - private void internalServerError(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - setHeaders(response, sessionId); - response.setStatus(500); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - responseWriter.write("boom"); - responseWriter.flush(); - closeChannel(responseWriter); - } - - private void couldNotFeed(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - setHeaders(response, sessionId); - response.setStatus(responseCode); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - String operationId; - while ((operationId = readOperationId(request.getInputStream())) != null) { - long lengthToSkip = readByteLength(request.getInputStream()); - while (lengthToSkip > 0) { - long skipped = request.getInputStream().skip(lengthToSkip); - lengthToSkip -= skipped; - } - respondTransientFailed(responseWriter, operationId); - } - closeChannel(responseWriter); - } - - private void mbusReturnedError(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - setHeaders(response, sessionId); - response.setStatus(responseCode); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - String operationId; - while ((operationId = readOperationId(request.getInputStream())) != null) { - long lengthToSkip = readByteLength(request.getInputStream()); - while (lengthToSkip > 0) { - long skipped = request.getInputStream().skip(lengthToSkip); - lengthToSkip -= skipped; - } - respondFailedWithTransitiveErrorSeenFromClient(responseWriter, operationId); - } - closeChannel(responseWriter); - } - - private void neverReturnAnyResults(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - setHeaders(response, sessionId); - response.setStatus(responseCode); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - BufferedReader reader = new BufferedReader(new InputStreamReader(request.getInputStream())); - while (reader.readLine() != null) { - //consume input, not really needed? - } - reader.close(); - closeChannel(responseWriter); - } - - void closeChannel(PrintWriter responseWriter) { - System.err.println("Mock server " + name + " closing channel."); - responseWriter.close(); - } - - private String readOperationId(InputStream requestInputStream) throws IOException { - StringBuilder idBuf = new StringBuilder(100); - int c; - while ((c = requestInputStream.read()) != -1) { - if (c == 32) { - break; - } - idBuf.append((char) c); //it's ASCII - } - if (c == -1) { - return null; - } - return Encoder.decode(idBuf.toString(), new StringBuilder(idBuf.length())).toString(); - } - - private int readByteLength(InputStream requestInputStream) throws IOException { - StringBuilder lenBuf = new StringBuilder(8); - int c; - while ((c = requestInputStream.read()) != -1) { - if (c == 10) { - break; - } - lenBuf.append((char) c); //it's ASCII - } - if (lenBuf.length() == 0) { - throw new IllegalStateException("Operation length missing."); - } - return Integer.valueOf(lenBuf.toString(), 16); - } - - private static void setHeaders(HttpServletResponse response, String sessionId) { - response.setHeader(Headers.SESSION_ID, sessionId); - response.setHeader(Headers.VERSION, "3"); - } - - private void respondFailed(PrintWriter responseWriter, String docId) { - final OperationStatus operationStatus = - new OperationStatus("mbus returned boom", docId, ErrorCode.ERROR, false, "trace"); - writeResponse(responseWriter, operationStatus); - } - - private void respondTransientFailed(PrintWriter responseWriter, String docId) { - final OperationStatus operationStatus = new OperationStatus( - "Could not put", docId, ErrorCode.TRANSIENT_ERROR, false, ""); - writeResponse(responseWriter, operationStatus); - } - - private void respondFailedWithTransitiveErrorSeenFromClient(PrintWriter responseWriter, String docId) { - final OperationStatus operationStatus = - new OperationStatus("NETWORK_ERROR", docId, ErrorCode.ERROR, false, "trace"); - writeResponse(responseWriter, operationStatus); - } - - private void respondConditionNotMet(PrintWriter responseWriter, String docId) { - final OperationStatus operationStatus = - new OperationStatus("this is a test", docId, ErrorCode.ERROR, true, "trace"); - writeResponse(responseWriter, operationStatus); - } - private void respondOK(PrintWriter responseWriter, String docId) { - final OperationStatus operationStatus = new OperationStatus("Doc fed", docId, ErrorCode.OK, false, "Trace message"); - writeResponse(responseWriter, operationStatus); - } - - private void writeResponse(PrintWriter responseWriter, - final OperationStatus operationStatus) { - responseWriter.print(operationStatus.render()); - responseWriter.flush(); - System.err.println("Mock " + name + " server wrote: " + operationStatus.render()); - } - - private String getSessionId(HttpServletRequest request) { - return request.getHeader(Headers.CLIENT_ID); - } - - private String generateMockSessionId() { - return String.valueOf(sessionIdGenerator.getAndIncrement()); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java deleted file mode 100644 index 55f6af9ed32..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.runner; - -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.config.SessionParams; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import static org.hamcrest.CoreMatchers.hasItem; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.core.Is.is; -import static org.hamcrest.core.IsNot.not; -import static org.junit.Assert.assertThat; - -public class CommandLineArgumentsTest { - - private String[] asArray() { - String[] array = new String[args.size()]; - args.toArray(array); - return array; - } - - private void add(String key, String value) { - args.add("--" + key); - args.add(value); - } - - private void addMinimum() { - add("host", "hostValue"); - } - - private ArrayList<String> args = new ArrayList<>(); - - @Test - public void testRequiredFlags() { - assertThat(CommandLineArguments.build(asArray()), is(nullValue())); - add("file", "fileValue"); - assertThat(CommandLineArguments.build(asArray()), is(nullValue())); - args.clear(); - addMinimum(); - assertThat(CommandLineArguments.build(asArray()), is(not(nullValue()))); - } - - @Test - public void testStreaming() { - add("host", "hostValue"); - add("file", null); // Not yet implemented support for streaming - assertThat(CommandLineArguments.build(asArray()), is(nullValue())); - } - - @Test - public void testBrokenFlags() { - addMinimum(); - args.add("FOO"); - assertThat(CommandLineArguments.build(asArray()), is(nullValue())); - } - - @Test - public void testBadPriority() { - addMinimum(); - add("priority", "non existing"); - assertThat(CommandLineArguments.build(asArray()), is(nullValue())); - } - - @Test - public void testOkPriority() { - addMinimum(); - add("priority", "HIGHEST"); - assertThat(CommandLineArguments.build(asArray()).createSessionParams(false).getFeedParams().getPriority(), - is("HIGHEST")); - } - - @Test - public void testDefaults() { - addMinimum(); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(false /* use json */); - assertThat(params.getClientQueueSize(), is(10000)); - assertThat(params.getThrottlerMinSize(), is(0)); - assertThat(params.getClusters().size(), is(1)); - assertThat(params.getClusters().get(0).getEndpoints().size(), is(1)); - assertThat(params.getClusters().get(0).getEndpoints().get(0).getHostname(), is("hostValue")); - assertThat(params.getClusters().get(0).getEndpoints().get(0).getPort(), is(4080)); - assertThat(params.getClusters().get(0).getEndpoints().get(0).isUseSsl(), is(false)); - assertThat(params.getConnectionParams().getUseCompression(), is(false)); - assertThat(params.getConnectionParams().getNumPersistentConnectionsPerEndpoint(), is(4)); - assertThat(params.getFeedParams().getRoute(), is("default")); - assertThat(params.getFeedParams().getDataFormat(), is(FeedParams.DataFormat.XML_UTF8)); - assertThat(params.getFeedParams().getLocalQueueTimeOut(), is(180000L)); - assertThat(params.getFeedParams().getMaxInFlightRequests(), is(10000)); - assertThat(params.getFeedParams().getClientTimeout(TimeUnit.MILLISECONDS), is(180000L)); - } - - @Test - public void testAllImplementedFlags() { - add("file", "fileValue.json"); - add("route", "routeValue"); - add("host", "hostValue"); - add("port", "1234"); - add("timeout", "2345"); - add("numPersistentConnectionsPerEndpoint", "7"); - args.add("--useCompression"); - args.add("--useDynamicThrottling"); - add("maxpending", "3456"); - args.add("--verbose"); - args.add("--useTls"); - add("header", "Header-Name: Header-Value"); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(true /* use json */); - assertThat(params.getClientQueueSize(), is(3456)); - assertThat(params.getThrottlerMinSize(), is(10)); - assertThat(params.getClusters().get(0).getEndpoints().get(0).getPort(), is(1234)); - assertThat(params.getClusters().get(0).getEndpoints().get(0).isUseSsl(), is(true)); - assertThat(params.getConnectionParams().getUseCompression(), is(true)); - assertThat(params.getConnectionParams().getHeaders().size(), is(1)); - assertThat(params.getFeedParams().getRoute(), is("routeValue")); - assertThat(params.getFeedParams().getDataFormat(), is(FeedParams.DataFormat.JSON_UTF8)); - assertThat(params.getFeedParams().getLocalQueueTimeOut(), is(2345000L)); - assertThat(params.getFeedParams().getMaxInFlightRequests(), is(3456)); - assertThat(params.getFeedParams().getClientTimeout(TimeUnit.MILLISECONDS), is(2345000L)); - assertThat(params.getConnectionParams().getNumPersistentConnectionsPerEndpoint(), is(7)); - } - - @Test - public void testAddingMultipleHttpHeaders() { - add("host", "hostValue"); - String header1Name = "Header-Name-1"; - String header1Value = "Header-Value"; - add("header", header1Name + ": " + header1Value); - String header2Name = "Header-Name-2"; - String header2Value = "Another-Header-Value"; - add("header", header2Name + ": " + header2Value); - - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(true /* use json */); - - List<Map.Entry<String, String>> headers = new ArrayList<>(params.getConnectionParams().getHeaders()); - headers.sort(Comparator.comparing(Map.Entry::getKey)); - - assertThat(headers.size(), is(2)); - Map.Entry<String, String> actualHeader1 = headers.get(0); - assertThat(actualHeader1.getKey(), is(header1Name)); - assertThat(actualHeader1.getValue(), is(header1Value)); - Map.Entry<String, String> actualHeader2 = headers.get(1); - assertThat(actualHeader2.getKey(), is(header2Name)); - assertThat(actualHeader2.getValue(), is(header2Value)); - } - - @Test - public void testMultiHost() { - add("file", "fileValue.json"); - add("port", "1234"); - add("host", "hostValue1,hostValue2, hostValue3"); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(true /* use json */); - assertThat(params.getClusters().size(), is(3)); - final Set<String> hosts = new HashSet<>(); - for (Cluster cluster : params.getClusters()) { - assertThat(cluster.getEndpoints().size(), is(1)); - hosts.add(cluster.getEndpoints().get(0).getHostname()); - assertThat(cluster.getEndpoints().get(0).getPort(), is(1234)); - } - assertThat(hosts, hasItem("hostValue1")); - assertThat(hosts, hasItem("hostValue2")); - assertThat(hosts, hasItem("hostValue3")); - } - - @Test - public void testUseV3Protocol() { - addMinimum(); - args.add("--useV3Protocol"); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(true /* use json */); - } - - @Test - public void testEndpoint() { - add("endpoint", "http://myendpoint:1234"); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(true); - assertThat(params.getClusters().get(0).getEndpoints().get(0).getHostname(), is("myendpoint")); - assertThat(params.getClusters().get(0).getEndpoints().get(0).getPort(), is(1234)); - assertThat(params.getClusters().get(0).getEndpoints().get(0).isUseSsl(), is(false)); - } - - @Test - public void testEndpointHttps() { - add("endpoint", "https://myendpoint:1234"); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(true); - assertThat(params.getClusters().get(0).getEndpoints().get(0).isUseSsl(), is(true)); - } - - @Test - public void testEndpointAndHost() { - add("host", "myhost"); - add("port", "2345"); - add("endpoint", "http://myendpoint:1234"); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - assertThat(arguments, is(nullValue())); // cannot have both endpoint and host - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/FormatInputStreamTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/FormatInputStreamTest.java deleted file mode 100644 index 9c77ed9a6d9..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/FormatInputStreamTest.java +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.runner; - -import org.junit.Test; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStream; -import java.util.Optional; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; - -/** - * @author valerijf - */ -public class FormatInputStreamTest { - @Test(expected=IllegalArgumentException.class) - public void testWithGarbageText() throws IOException { - String streamString = "This is neither XML nor JSON!"; - InputStream jsonStream = getInputStreamOf(streamString); - FormatInputStream formatInputStream = new FormatInputStream(jsonStream, Optional.empty(), false); - } - - @Test - public void testWithFileInput() throws IOException { - String fileString = "{\"format\": \"json\"}"; - File file = File.createTempFile("feeddata", "json"); - file.deleteOnExit(); - try (FileWriter writer = new FileWriter(file)) { - writer.write(fileString); - } - - FormatInputStream formatInputStream = new FormatInputStream(null, Optional.of(file.getAbsolutePath()), false); - assertThat(fileString, is(convertStreamToString(formatInputStream.getInputStream()))); - assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.JSON)); - } - - @Test - public void testPreferenceFileOverStream() throws IOException { - String streamString = "something entirely different"; - String fileString = "{\"format\": \"json\"}"; - - InputStream jsonStream = getInputStreamOf(streamString); - File file = File.createTempFile("feeddata", "json"); - file.deleteOnExit(); - try (FileWriter writer = new FileWriter(file)) { - writer.write(fileString); - } - - FormatInputStream formatInputStream = new FormatInputStream(jsonStream, Optional.of(file.getAbsolutePath()), false); - assertThat(fileString, is(convertStreamToString(formatInputStream.getInputStream()))); - assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.JSON)); - } - - @Test - public void testSimpleJsonInputStream() throws IOException { - String streamString = "{\"format\": \"json\"}"; - InputStream jsonStream = getInputStreamOf(streamString); - FormatInputStream formatInputStream = new FormatInputStream(jsonStream, Optional.empty(), false); - - assertThat(streamString, is(convertStreamToString(formatInputStream.getInputStream()))); - assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.JSON)); - } - - @Test - public void testSimpleXmlInputStream() throws IOException { - String streamString = "<scope><tag>format</tag><value>xml</value></scope>"; - InputStream jsonStream = getInputStreamOf(streamString); - FormatInputStream formatInputStream = new FormatInputStream(jsonStream, Optional.empty(), false); - - assertThat(streamString, is(convertStreamToString(formatInputStream.getInputStream()))); - assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.XML)); - } - - @Test - public void testSparselyFormattedXml() throws IOException { - String streamString = " \t\t\n<scope>\n\n\n<tag>format</tag><value>xml</value></scope>"; - InputStream jsonStream = getInputStreamOf(streamString); - FormatInputStream formatInputStream = new FormatInputStream(jsonStream, Optional.empty(), false); - - assertThat(streamString, is(convertStreamToString(formatInputStream.getInputStream()))); - assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.XML)); - } - - @Test - public void testAddRootToXml() throws IOException { - String streamString = "some random text"; - InputStream textStream = getInputStreamOf(streamString); - FormatInputStream formatInputStream = new FormatInputStream(textStream, Optional.empty(), true); - - assertThat("<vespafeed>" + streamString + "</vespafeed>", - is(convertStreamToString(formatInputStream.getInputStream()))); - assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.XML)); - } - - private static String convertStreamToString(InputStream inputStream) throws IOException { - StringBuilder builder = new StringBuilder(); - while (true) { - int character = inputStream.read(); - if (character == -1) { - inputStream.close(); - return builder.toString(); - } - builder.append((char)character); - } - } - - private static InputStream getInputStreamOf(String text) { - return new ByteArrayInputStream(text.getBytes()); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/JsonReaderTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/JsonReaderTest.java deleted file mode 100644 index 0a5b3771958..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/JsonReaderTest.java +++ /dev/null @@ -1,270 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.runner; - -import com.yahoo.vespa.http.client.FeedClient; -import com.yahoo.vespa.http.client.core.JsonReader; -import org.junit.Test; - -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -import static com.yahoo.test.json.JsonTestHelper.inputJson; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; - -public class JsonReaderTest { - - private static String doc1_id = "id:unittest:testMapStringToArrayOfInt::whee"; - - private static String doc1 = inputJson( - "{", - " 'update': '"+ doc1_id + "',", - " 'fields': {", - " 'actualMapStringToArrayOfInt': {", - " 'assign': [", - " { 'key': 'bamse', 'value': [ 2, 1, 3] }", - " ]", - " }", - " }", - "}"); - - private static String doc2_id = "id:unittest:smoke::whee"; - - private static String doc2 = inputJson( - "{", - " 'put': '" + doc2_id + "',", - " 'fields': {", - " 'something': 'smoketest',", - " 'nalle': 'bamse'", - " }", - "}"); - - private static String doc3 = inputJson( - "{", - " 'update': 'id:unittest:testarray::whee',", - " 'fields': {", - " 'actualarray': {", - " 'add': [", - " 'person naïve',", - " 'another person'", - " ]", - " }", - " }", - "}"); - - private static String doc4 = inputJson( - "{", - " 'remove': '" + doc2_id + "'", - "}"); - - private static String doc5_id = "id:unittest:smoking::wheels"; - - private static String doc5 = inputJson( - "{", - " 'id': '" + doc5_id + "',", - " 'fields': {", - " 'something': 'smoketest',", - " 'nalle': 'bamse'", - " }", - "}"); - - private static class TestFeedClient implements FeedClient { - - public List<String> documentIds = new ArrayList<>(); - public List<CharSequence> datas = new ArrayList<>(); - public List<Object> contexts = new ArrayList<>(); - - @Override - public void stream(String documentId, CharSequence documentData) { - stream(documentId, documentData, null); - } - - @Override - public void stream(String documentId, String operationId, CharSequence documentData, Object context) { - documentIds.add(documentId); - datas.add(documentData); - contexts.add(context); - } - - @Override - public void close() { } - - @Override - public String getStatsAsJson() { return null; } - } - - final TestFeedClient session = new TestFeedClient(); - final AtomicInteger numSent = new AtomicInteger(0); - - @Test - public void testReadNoDocument() throws Exception { - InputStream inputStream = new ByteArrayInputStream( - " ".getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - inputStream.close(); - assertThat(session.documentIds.size(), is(0)); - } - - @Test - public void testReadOneDocument() throws Exception { - InputStream inputStream = new ByteArrayInputStream( - ("["+ doc1 + "]" ).getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - inputStream.close(); - assertThat(session.documentIds.size(), is(1)); - assertThat(session.documentIds.get(0), is(doc1_id)); - assertThat(session.datas.size(), is(1)); - assertThat(session.datas.get(0), is(doc1)); - } - - @Test - public void testReadFourDocuments() throws Exception { - InputStream inputStream = new ByteArrayInputStream( - (" [ "+ doc1 + " , " + doc2 + ", " + doc3 + "," + doc4 + " ] ").getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - inputStream.close(); - assertThat(session.documentIds.size(), is(4)); - assertThat(session.documentIds.get(0), is(doc1_id)); - assertThat(session.documentIds.get(1), is(doc2_id)); - assertThat(session.datas.size(), is(4)); - assertThat(session.datas.get(0), is(doc1)); - assertThat(session.datas.get(1).toString(), is(doc2)); - assertThat(session.datas.get(2).toString(), is(doc3)); - assertThat(session.datas.get(3).toString(), is(doc4)); - } - - @Test - public void testDocWithIdAndNotPut() throws Exception { - InputStream inputStream = new ByteArrayInputStream( - (" [ "+ doc5 + " ] ").getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - inputStream.close(); - assertThat(session.documentIds.size(), is(1)); - assertThat(session.documentIds.get(0), is(doc5_id)); - } - - @Test - public void simpleMicroBenchmarkTest() throws Exception { - StringBuilder stream = new StringBuilder(); - stream.append("["); - int docsInStream = 15000; - for (int x = 0; x < docsInStream -1; x++) { - if (x % 10 == 0) { - stream.append(doc1 + ", "); - } else { - // Add some randomness to the layout to trigger potential bugs in parsing. - stream.append("{\"remove\": \"id:unittest:smoke::whee"); - for (int y = 0 ; y < x % 277 ; y++) { - stream.append("X"); - } - stream.append("\"}, "); - } - } - stream.append(doc3); - stream.append("]"); - - InputStream inputStream = new ByteArrayInputStream(stream.toString().getBytes(StandardCharsets.UTF_8)); - long startTime = System.currentTimeMillis(); - JsonReader.read(inputStream, session, numSent); - // At time of writing, it took about 200 ms on my mac. - System.err.println("Run time is " + (System.currentTimeMillis() - startTime) + " ms"); - inputStream.close(); - - // Verify that content is not rubbish. - for (int x = 0; x < docsInStream - 1; x++) { - if (x % 10 == 0) { - assertThat(session.datas.get(x).toString(), is(doc1)); - assertThat(session.documentIds.get(x), is(doc1_id)); - } - } - assertThat(session.datas.get(docsInStream-1).toString(), is(doc3)); - assertThat(numSent.get(), is(docsInStream)); - } - - @Test(expected=RuntimeException.class) - public void testBadJsonCommaAfterLastElement() { - InputStream inputStream = new ByteArrayInputStream( - ("["+ doc1 + ",]" ).getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - } - - @Test(expected=RuntimeException.class) - public void testTotalGarbage() { - InputStream inputStream = new ByteArrayInputStream(("garbage" ).getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - } - - @Test(expected=RuntimeException.class) - public void testTwoDocIds() { - InputStream inputStream = new ByteArrayInputStream(("[{\"remove\": \"id\", \"update\": \"id:\"}]" - .getBytes(StandardCharsets.UTF_8))); - JsonReader.read(inputStream, session, numSent); - } - - @Test(expected = IllegalArgumentException.class) - public void throwsOnMissingId() { - InputStream inputStream = new ByteArrayInputStream( - inputJson("[{'fields':{ 'something': 'smoketest', 'nalle': 'bamse' }}]").getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - } - - @Test - public void testFullDocument() throws Exception { - InputStream inputStream = new ByteArrayInputStream(( - "[{\n" + - " \"update\": \"id:foo:music:doc:foo:bar\",\n" + - " \n" + - " \"fields\": {\n" + - " \"artist\": {\n" + - " \"assign\": null" + - " },\n" + - " \n" + - " \"albums\": {\n" + - " \"assign\": [\n" + - " \"Kramgoda laatar 4\",\n" + - " \"Kramgoda laatar 5\",\n" + - " \"Kramgoda laatar 6\"\n" + - " ],\n" + - " \"add\": [\n" + - " \"Kramgoda laatar 7\",\n" + - " \"Kramgoda laatar 8\"\n" + - " ]\n" + - " },\n" + - " \"inceptionYear\": {\n" + - " \"increment\": 4\n" + - " },\n" + - " \"concerts\": {\n" + - " \"assign\": {\n" + - " \"Torsby 1993\": 1000,\n" + - " \"Uddevalla 2000\": 34\n" + - " },\n" + - " \"match\": {\n" + - " \"element\": \"Sundsvall 1980\",\n" + - " \"increment\": 5392\n" + - " },\n" + - " \"add\": {\n" + - " \"Kiruna 1999\": 200,\n" + - " \"Oslo 1998\": 2000\n" + - " }\n" + - " },\n" + - " \"scores\": {\n" + - " \"match\": {\n" + - " \"element\": \"Sven Ingvars\",\n" + - " \"match\": {\n" + - " \"element\": 0,\n" + - " \"increment\": 78\n" + - " }\n" + - " }\n" + - " }\n" + - " }\n" + - "}]\n").getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - inputStream.close(); - assertThat(session.documentIds.size(), is(1)); - assertThat(session.documentIds.get(0), is("id:foo:music:doc:foo:bar")); - } -} diff --git a/vespa-http-client/src/test/resources/vespacorpfeed-prod-sample.xml b/vespa-http-client/src/test/resources/vespacorpfeed-prod-sample.xml deleted file mode 100644 index 95f0e4b1961..00000000000 --- a/vespa-http-client/src/test/resources/vespacorpfeed-prod-sample.xml +++ /dev/null @@ -1,187 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<vespafeed> -<document documenttype="vespacorp" documentid="id:vespacorp:vespacorp::7138bc793a096a78b86a6501ae0c6e7b"> - <threadId>f5078d76c7541ab15387ab62fef22a01</threadId> - <contentsHash>186e631cb3b09ac33e5c124a79e20915</contentsHash> - <authors> - <item>bratseth</item> - </authors> - <urlPath>/vespa-users/msg/4498622c3f131e1067c4e7bc18ac96db</urlPath> - <contents>This was done to make it simpler for people to get started with jars loaded through OSGi on the assumption that most wouldn't need to import anything I'm not sure it actually gives a net saving though for the reason you point out We'll probably make the existence of global packages optional somehow later Jon Den 21 mai 2011 kl 03.21 skrev LG Just out of curiosity why motivated the choice of exposing them as global packages instead of exported packages I had personally a hard time figuring out which packages I had to import and to not import javax com.yahoo.vespa etc and I'm wondering if it means I need to repackage some 3rdparty component if they decide to import some of these global packages like javax.foo lg geoinformatics software engineer mail@host.com direct y!im 701 first avenue sunnyvale ca 94089-0703 us phone fax Le 5/10/11 12:10 AM Jon S Bratseth a écrit The packages in the Vespa public api is always available global That's the ones included here http://vespa/javadoc/5.0/all In addition the packages belonging to the Java SE API javax etc are global The exported list is not quite as well defined though You can always just try to import when some package is not in the global category above if it cannot be resolved then we do not export it Or check the source as Francois mentions </contents> - <parentId>8bce4afebf9dca1aab84651db62d4269</parentId> - <title>How to load JSON library in vespa 5.0.8</title> - <my_contents>This was done to make it simpler for people to get started with jars loaded through OSGi on the assumption that most wouldn't need to import anything I'm not sure it actually gives a net saving though for the reason you point out We'll probably make the existence of global packages optional somehow later Jon Den 21 mai 2011 kl 03.21 skrev LG Just out of curiosity why motivated the choice of exposing them as global packages instead of exported packages I had personally a hard time figuring out which packages I had to import and to not import javax com.yahoo.vespa etc and I'm wondering if it means I need to repackage some 3rdparty component if they decide to import some of these global packages like javax.foo laurent goujon geoinformatics software engineer mail@host.com direct y!im 701 first avenue sunnyvale ca 94089-0703 us phone fax Le 5/10/11 12:10 AM Jon S Bratseth a écrit The packages in the Vespa public api is always available global That's the ones included here http://vespa/javadoc/5.0/all In addition the packages belonging to the Java SE API javax etc are global The exported list is not quite as well defined though You can always just try to import when some package is not in the global category above if it cannot be resolved then we do not export it Or check the source as Francois mentions </my_contents> - <lastUpdate>1306231886</lastUpdate> - <pagerank>0</pagerank> - <date>1306228138</date> - <headers></headers> - <articleType>email</articleType> - <emailProcessedContents binaryencoding="base64"></emailProcessedContents> - <attachments></attachments> - <documentAbstract></documentAbstract> - <audience>(NULL)</audience> - <docDirId>(NULL)</docDirId> - <visibility>1</visibility> - <headings> - <item></item> - </headings> - <my_title>How to load JSON library in vespa 5.0.8</my_title> - <messageId>4498622c3f131e1067c4e7bc18ac96db</messageId> - <origContents binaryencoding="base64">VGhpcyB3YXMgZG9uZSB0byBtYWtlIGl0IHNpbXBsZXIgZm9yIHBlb3BsZSB0byBnZXQgc3RhcnRlZCB3aXRoIGphcnMgbG9hZGVkIHRocm91Z2ggT1NHaSwgb24gdGhlIGFzc3VtcHRpb24gdGhhdCBtb3N0IHdvdWxkbid0IG5lZWQgdG8gaW1wb3J0IGFueXRoaW5nLgpJJ20gbm90IHN1cmUgaXQgYWN0dWFsbHkgZ2l2ZXMgYSBuZXQgc2F2aW5nIHRob3VnaCwgZm9yIHRoZSByZWFzb24geW91IHBvaW50IG91dC4gV2UnbGwgcHJvYmFibHkgbWFrZSB0aGUgZXhpc3RlbmNlIG9mIGdsb2JhbCBwYWNrYWdlcyBvcHRpb25hbCBzb21laG93IGxhdGVyLgoKLS0KSm9uCgpEZW4gMjEuIG1haSAyMDExIGtsLiAwMy4yMSBza3JldiBMYXVyZW50IEdvdWpvbjoKCkp1c3Qgb3V0IG9mIGN1cmlvc2l0eSwgd2h5IG1vdGl2YXRlZCB0aGUgY2hvaWNlIG9mIGV4cG9zaW5nIHRoZW0gYXMgZ2xvYmFsIHBhY2thZ2VzIGluc3RlYWQgb2YgZXhwb3J0ZWQgcGFja2FnZXM/CgpJIGhhZCBwZXJzb25hbGx5IGEgaGFyZCB0aW1lIGZpZ3VyaW5nIG91dCB3aGljaCBwYWNrYWdlcyBJIGhhZCB0byBpbXBvcnQgYW5kIHRvIG5vdCBpbXBvcnQgKGphdmF4LiosIGNvbS55YWhvby52ZXNwYSwgZXRjLi4uKSBhbmQgSSdtIHdvbmRlcmluZyBpZiBpdCBtZWFucyBJIG5lZWQgdG8gcmVwYWNrYWdlIHNvbWUgM3JkcGFydHkgY29tcG9uZW50IGlmIHRoZXkgZGVjaWRlIHRvIGltcG9ydCBzb21lIG9mIHRoZXNlIGdsb2JhbCBwYWNrYWdlcyBsaWtlIGphdmF4LmZvby4KCmxhdXJlbnQKZ291am9uCgpnZW9pbmZvcm1hdGljcyBzb2Z0d2FyZSBlbmdpbmVlcgoKZ291am9ubEB5YWhvby1pbmMuY29tPG1haWx0bzpnb3Vqb25sQHlhaG9vLWluYy5jb20+CmRpcmVjdCAoNDA4KSAzNDkgOTMwMgp5IWltIHlsYXVyZW50Z28KCjcwMSBmaXJzdCBhdmVudWUsIHN1bm55dmFsZSwgY2EsIDk0MDg5LTA3MDMsIHVzCnBob25lICg0MDgpIDM0OSAzMzAwICAgIGZheCAoNDA4KSAzNDkgMzMwMQoKPHlhaG9vLmdpZj4KCgpMZSA1LzEwLzExIDEyOjEwIEFNLCBKb24gUyBCcmF0c2V0aCBhIMOpY3JpdCA6CgpUaGUgcGFja2FnZXMgaW4gdGhlIFZlc3BhIHB1YmxpYyBhcGkgaXMgYWx3YXlzIGF2YWlsYWJsZSAoZ2xvYmFsKS4KVGhhdCdzIHRoZSBvbmVzIGluY2x1ZGVkIGhlcmU6IGh0dHA6Ly92ZXNwYS5jb3JwLnlhaG9vLmNvbS9qYXZhZG9jLzUuMC9hbGwvCkluIGFkZGl0aW9uLCB0aGUgcGFja2FnZXMgYmVsb25naW5nIHRvIHRoZSBKYXZhIFNFIEFQSSAoamF2YXggZXRjLikgYXJlIGdsb2JhbC4KClRoZSAiZXhwb3J0ZWQiIGxpc3QgaXMgbm90IHF1aXRlIGFzIHdlbGwgZGVmaW5lZCB0aG91Z2guIFlvdSBjYW4gYWx3YXlzIGp1c3QgdHJ5IHRvIGltcG9ydCB3aGVuIHNvbWUgcGFja2FnZSBpcyBub3QgaW4gdGhlICJnbG9iYWwiIGNhdGVnb3J5IGFib3ZlLCBpZiBpdCBjYW5ub3QgYmUgcmVzb2x2ZWQgdGhlbiB3ZSBkbyBub3QgZXhwb3J0IGl0LiBPciBjaGVjayB0aGUgc291cmNlIGFzIEZyYW5jb2lzIG1lbnRpb25zLg==</origContents> - <threadUrl>/2011/05/09/how_to_load_json_library_in_vespa_5_0_8</threadUrl> - <level>(NULL)</level> -</document> -<document documenttype="vespacorp" documentid="id:vespacorp:vespacorp::11b6e7f86d650b995289dcadaaf902fc"> - <threadId>f4467ca341409984b9c3ab4c431f0a7c</threadId> - <contentsHash>c0d00a07cc1486404f5e23a2cc3cc5ed</contentsHash> - <authors> - <item>bratseth</item> - </authors> - <urlPath>/vespa-users/msg/2538973d3c06d9d5163bf9bd069d0bd6</urlPath> - <contents>The unit test and doc I pointed to had some examples were you looking for something else The unit tests are using processors from this set http://vespa/view/vespa/trunk/container/processing/src/main/java/com/yahoo/processing/test/ProcessorLibrary.java?view=markup From this you might be interested in Federator forks the execution to multiple chains executed in parallel FutureDataSource creates and returns response containing future data BlockingSplitter An example of waiting for some future data to complete before using them for some processing of course don t do this if you want full async for some reason AsyncDataProcessingInitiator An example of registering future processing when a list of data is completed i.e process the data without blocking StreamProcessingInitiator An example of registering future processing on every additional piece of data entering a list None of this shows how to create a real async data source which gets its data from the network the FutureDataSource above is of course just a mock I have an example of that somewhere which I can dig up if you need it Jon On 6 feb 2014 at 02:40 GOs wrote Is there an example which uses the futures We have async working but it's not sending all of the data until the request is marked as completed Best GGO Tech Yahoo Software Systems Development Engineer M 701 First Avenue Sunnyvale CA 94089 http://forgood.zenfs.com/logos/yahoo.png On Wednesday February 5 2014 2:02 AM Jon Bratseth wrote See the processing framework in particular http://vespa/5/documentation/jdisc/processing.html#asynchronous-processing and the example AsyncDataProducer towards the end of the page In short the Processing framework supports this use case out of the box just return a Response with data futures and the renderer will render as much as possible at once and then the future data whenever it becomes available See http://vespa/view/vespa/trunk/container/core/src/test/java/com/yahoo/processing/handler/ProcessingHandlerTestCase.java?view=markup for some complete examples the tests named something with async Also note that by default the renderer will preserve the order of the future placeholders in the response such that if a Response containing future data list A and B it will wait for A before rendering further even if B is available If you want to return data as soon as possible in any order have your DataList instances implement com.yahoo.processing.response.Ordered and return false from isOrdered There s an example of that as well in the test above Jon On 5 feb 2014 at 10:43 Kristian Aune wrote vespa-users is the best list K On 5 feb 2014 at 01:24 GO wrote Hey Kristian Not sure what the ilist is for JDisc user support so if you know just reply and I'll forward this mail that way Basically here's the scenario I'm looking at We get a request from client and receive it in JDisc Then we do a bunch of processing and serve responses as they come this way the client can immediately render finished responses while still waiting for pending responses What is the easiest way to do this in a JDisc container I was looking at websockets but not sure how much they are supported Best GE desde mi iPhone Vespa Information http://vespa </contents> - <parentId>1c809fe04d3807dc72be71cf1aa559d6</parentId> - <title>JDisc streaming / websockets</title> - <my_contents>The unit test and doc I pointed to had some examples were you looking for something else The unit tests are using processors from this set http://vespa/view/vespa/trunk/container/processing/src/main/java/com/yahoo/processing/test/ProcessorLibrary.java?view=markup From this you might be interested in Federator forks the execution to multiple chains executed in parallel FutureDataSource creates and returns response containing future data BlockingSplitter An example of waiting for some future data to complete before using them for some processing of course don t do this if you want full async for some reason AsyncDataProcessingInitiator An example of registering future processing when a list of data is completed i.e process the data without blocking StreamProcessingInitiator An example of registering future processing on every additional piece of data entering a list None of this shows how to create a real async data source which gets its data from the network the FutureDataSource above is of course just a mock I have an example of that somewhere which I can dig up if you need it Jon On 6 feb 2014 at 02:40 GO wrote Is there an example which uses the futures We have async working but it's not sending all of the data until the request is marked as completed Best GGO Tech Yahoo Software Systems Development Engineer MFirst Avenue Sunnyvale CA 94089 http://forgood.zenfs.com/logos/yahoo.png On Wednesday February 5 2014 2:02 AM Jon Bratseth wrote See the processing framework in particular http://vespa/5/documentation/jdisc/processing.html#asynchronous-processing and the example AsyncDataProducer towards the end of the page In short the Processing framework supports this use case out of the box just return a Response with data futures and the renderer will render as much as possible at once and then the future data whenever it becomes available See http://vespa/view/vespa/trunk/container/core/src/test/java/com/yahoo/processing/handler/ProcessingHandlerTestCase.java?view=markup for some complete examples the tests named something with async Also note that by default the renderer will preserve the order of the future placeholders in the response such that if a Response containing future data list A and B it will wait for A before rendering further even if B is available If you want to return data as soon as possible in any order have your DataList instances implement com.yahoo.processing.response.Ordered and return false from isOrdered There s an example of that as well in the test above Jon On 5 feb 2014 at 10:43 Kristian Aune wrote vespa-users is the best list K On 5 feb 2014 at 01:24 GO wrote Hey Kristian Not sure what the ilist is for JDisc user support so if you know just reply and I'll forward this mail that way Basically here's the scenario I'm looking at We get a request from client and receive it in JDisc Then we do a bunch of processing and serve responses as they come this way the client can immediately render finished responses while still waiting for pending responses What is the easiest way to do this in a JDisc container I was looking at websockets but not sure how much they are supported Best GE desde mi iPhone Vespa Information http://vespa</my_contents> - <lastUpdate>1391683805</lastUpdate> - <pagerank>0</pagerank> - <date>1391680103</date> - <headers></headers> - <articleType>email</articleType> - <emailProcessedContents binaryencoding="base64"></emailProcessedContents> - <attachments></attachments> - <documentAbstract></documentAbstract> - <audience>(NULL)</audience> - <docDirId>(NULL)</docDirId> - <visibility>1</visibility> - <headings> - <item></item> - </headings> - <my_title>JDisc streaming / websockets</my_title> - <messageId>2538973d3c06d9d5163bf9bd069d0bd6</messageId> - <origContents binaryencoding="base64">VGhlIHVuaXQgdGVzdCBhbmQgZG9jIEkgcG9pbnRlZCB0byBoYWQgc29tZSBleGFtcGxlcywgd2VyZSB5b3UgbG9va2luZyBmb3Igc29tZXRoaW5nIGVsc2U/CgpUaGUgdW5pdCB0ZXN0cyBhcmUgdXNpbmcgcHJvY2Vzc29ycyBmcm9tIHRoaXMgc2V0OgpodHRwOi8vc3ZuLnRyb25kaGVpbS5jb3JwLnlhaG9vLmNvbS92aWV3L3Zlc3BhL3RydW5rL2NvbnRhaW5lci9wcm9jZXNzaW5nL3NyYy9tYWluL2phdmEvY29tL3lhaG9vL3Byb2Nlc3NpbmcvdGVzdC9Qcm9jZXNzb3JMaWJyYXJ5LmphdmE/dmlldz1tYXJrdXAKRnJvbSB0aGlzLiB5b3UgbWlnaHQgYmUgaW50ZXJlc3RlZCBpbgpGZWRlcmF0b3IgLSBmb3JrcyB0aGUgZXhlY3V0aW9uIHRvIG11bHRpcGxlIGNoYWlucyBleGVjdXRlZCBpbiBwYXJhbGxlbC4KRnV0dXJlRGF0YVNvdXJjZSAtIGNyZWF0ZXMgYW5kIHJldHVybnMgcmVzcG9uc2UgY29udGFpbmluZyBmdXR1cmUgZGF0YS4KQmxvY2tpbmdTcGxpdHRlciAtIEFuIGV4YW1wbGUgb2Ygd2FpdGluZyBmb3Igc29tZSBmdXR1cmUgZGF0YSB0byBjb21wbGV0ZSBiZWZvcmUgdXNpbmcgdGhlbSBmb3Igc29tZSBwcm9jZXNzaW5nIChvZiwgY291cnNlLCBkb27CknQgZG8gdGhpcyBpZiB5b3Ugd2FudCBmdWxsIGFzeW5jIGZvciBzb21lIHJlYXNvbikKQXN5bmNEYXRhUHJvY2Vzc2luZ0luaXRpYXRvciAtIEFuIGV4YW1wbGUgb2YgcmVnaXN0ZXJpbmcgZnV0dXJlIHByb2Nlc3Npbmcgd2hlbiBhIGxpc3Qgb2YgZGF0YSBpcyBjb21wbGV0ZWQgKGkuZSBwcm9jZXNzIHRoZSBkYXRhIHdpdGhvdXQgYmxvY2tpbmcpClN0cmVhbVByb2Nlc3NpbmdJbml0aWF0b3IgLSBBbiBleGFtcGxlIG9mIHJlZ2lzdGVyaW5nIGZ1dHVyZSBwcm9jZXNzaW5nIG9uIGV2ZXJ5IGFkZGl0aW9uYWwgcGllY2Ugb2YgZGF0YSBlbnRlcmluZyBhIGxpc3QKCk5vbmUgb2YgdGhpcyBzaG93cyBob3cgdG8gY3JlYXRlIGEgKnJlYWwqIGFzeW5jIGRhdGEgc291cmNlIHdoaWNoIGdldHMgaXRzIGRhdGEgZnJvbSB0aGUgbmV0d29yayAodGhlIEZ1dHVyZURhdGFTb3VyY2UgYWJvdmUgaXMgb2YgY291cnNlIGp1c3QgYSBtb2NrKS4KSSBoYXZlIGFuIGV4YW1wbGUgb2YgdGhhdCBzb21ld2hlcmUgd2hpY2ggSSBjYW4gZGlnIHVwIGlmIHlvdSBuZWVkIGl0LgoKwpcKSm9uCgpPbiA2LiBmZWIuIDIwMTQsIGF0IDAyOjQwLCBHYXZpbiBPd2VucyA8Z293ZW5zQHlhaG9vLWluYy5jb208bWFpbHRvOmdvd2Vuc0B5YWhvby1pbmMuY29tPj4gd3JvdGU6CgpJcyB0aGVyZSBhbiBleGFtcGxlIHdoaWNoIHVzZXMgdGhlIGZ1dHVyZXM/CgpXZSBoYXZlIGFzeW5jIHdvcmtpbmcsIGJ1dCBpdCdzIG5vdCBzZW5kaW5nIGFsbCBvZiB0aGUgZGF0YSB1bnRpbCB0aGUgcmVxdWVzdCBpcyBtYXJrZWQgYXMgY29tcGxldGVkLgoKQmVzdCwKR2F2aW4KCgpHYXZpbiBPd2VucwpUZWNoIFlhaG9vLCBTb2Z0d2FyZSBTeXN0ZW1zIERldmVsb3BtZW50IEVuZ2luZWVyCk06ICg0MDgpIDMwNi03NTM2CjcwMSBGaXJzdCBBdmVudWUKU3Vubnl2YWxlIENBIDk0MDg5CgpbaHR0cDovL2Zvcmdvb2QuemVuZnMuY29tL2xvZ29zL3lhaG9vLnBuZ10KCgpPbiBXZWRuZXNkYXksIEZlYnJ1YXJ5IDUsIDIwMTQgMjowMiBBTSwgSm9uIEJyYXRzZXRoIDxicmF0c2V0aEB5YWhvby1pbmMuY29tPG1haWx0bzpicmF0c2V0aEB5YWhvby1pbmMuY29tPj4gd3JvdGU6ClNlZSB0aGUgcHJvY2Vzc2luZyBmcmFtZXdvcmssIGluIHBhcnRpY3VsYXIgaHR0cDovL3Zlc3BhLmNvcnAueWFob28uY29tLzUvZG9jdW1lbnRhdGlvbi9qZGlzYy9wcm9jZXNzaW5nLmh0bWwjYXN5bmNocm9ub3VzLXByb2Nlc3NpbmcKYW5kIHRoZSBleGFtcGxlIEFzeW5jRGF0YVByb2R1Y2VyIHRvd2FyZHMgdGhlIGVuZCBvZiB0aGUgcGFnZS4KCkluIHNob3J0LCB0aGUgUHJvY2Vzc2luZyBmcmFtZXdvcmsgc3VwcG9ydHMgdGhpcyB1c2UgY2FzZSBvdXQgb2YgdGhlIGJveCAtIGp1c3QgcmV0dXJuIGEgUmVzcG9uc2Ugd2l0aCBkYXRhIGZ1dHVyZXMgYW5kIHRoZSByZW5kZXJlciB3aWxsIHJlbmRlciBhcyBtdWNoIGFzIHBvc3NpYmxlIGF0IG9uY2UgYW5kIHRoZW4gdGhlIGZ1dHVyZSBkYXRhIHdoZW5ldmVyIGl0IGJlY29tZXMgYXZhaWxhYmxlLgoKClNlZSBodHRwOi8vc3ZuLnRyb25kaGVpbS5jb3JwLnlhaG9vLmNvbS92aWV3L3Zlc3BhL3RydW5rL2NvbnRhaW5lci9jb3JlL3NyYy90ZXN0L2phdmEvY29tL3lhaG9vL3Byb2Nlc3NpbmcvaGFuZGxlci9Qcm9jZXNzaW5nSGFuZGxlclRlc3RDYXNlLmphdmE/dmlldz1tYXJrdXAKZm9yIHNvbWUgY29tcGxldGUgZXhhbXBsZXMgKHRoZSB0ZXN0cyBuYW1lZCBzb21ldGhpbmcgd2l0aCBhc3luYykuCgoKQWxzbywgbm90ZSB0aGF0IGJ5IGRlZmF1bHQgdGhlIHJlbmRlcmVyIHdpbGwgcHJlc2VydmUgdGhlIG9yZGVyIG9mIHRoZSBmdXR1cmUgcGxhY2Vob2xkZXJzIGluIHRoZSByZXNwb25zZSwgc3VjaCB0aGF0IGlmIGEgUmVzcG9uc2UgY29udGFpbmluZyBmdXR1cmUgZGF0YSBsaXN0IEEgYW5kIEIsIGl0IHdpbGwgd2FpdCBmb3IgQSBiZWZvcmUgcmVuZGVyaW5nIGZ1cnRoZXIgZXZlbiBpZiBCIGlzIGF2YWlsYWJsZS4KSWYgeW91IHdhbnQgdG8gcmV0dXJuIGRhdGEgYXMgc29vbiBhcyBwb3NzaWJsZSBpbiBhbnkgb3JkZXIsIGhhdmUgeW91ciBEYXRhTGlzdCBpbnN0YW5jZXMgaW1wbGVtZW50IGNvbS55YWhvby5wcm9jZXNzaW5nLnJlc3BvbnNlLk9yZGVyZWQgYW5kIHJldHVybiBmYWxzZSBmcm9tIGlzT3JkZXJlZC4gVGhlcmXCknMgYW4gZXhhbXBsZSBvZiB0aGF0IGFzIHdlbGwgaW4gdGhlIHRlc3QgYWJvdmUuCgrClwpKb24KCk9uIDUuIGZlYi4gMjAxNCwgYXQgMTA6NDMsIEtyaXN0aWFuIEF1bmUgPEtyaXN0aWFuLkF1bmVAeWFob28taW5jLmNvbTxtYWlsdG86S3Jpc3RpYW4uQXVuZUB5YWhvby1pbmMuY29tPj4gd3JvdGU6Cgo+IHZlc3BhLXVzZXJzQCBpcyB0aGUgYmVzdCBsaXN0IDstKQo+Cj4gLUsKPgo+IE9uIDUuIGZlYi4gMjAxNCwgYXQgMDE6MjQsIEdhdmluIE93ZW5zIDxnb3dlbnNAeWFob28taW5jLmNvbTxtYWlsdG86Z293ZW5zQHlhaG9vLWluYy5jb20+PiB3cm90ZToKPgo+PiBIZXkgS3Jpc3RpYW4sCj4+Cj4+IE5vdCBzdXJlIHdoYXQgdGhlIGlsaXN0IGlzIGZvciBKRGlzYyB1c2VyIHN1cHBvcnQsIHNvIGlmIHlvdSBrbm93IGp1c3QgcmVwbHkgYW5kIEknbGwgZm9yd2FyZCB0aGlzIG1haWwgdGhhdCB3YXkuCj4+Cj4+IEJhc2ljYWxseSwgaGVyZSdzIHRoZSBzY2VuYXJpbyBJJ20gbG9va2luZyBhdC4uLgo+Pgo+PiBXZSBnZXQgYSByZXF1ZXN0IGZyb20gY2xpZW50IGFuZCByZWNlaXZlIGl0IGluIEpEaXNjLiBUaGVuIHdlIGRvIGEgYnVuY2ggb2YgcHJvY2Vzc2luZyBhbmQgc2VydmUgcmVzcG9uc2VzIGFzIHRoZXkgY29tZSAodGhpcyB3YXkgdGhlIGNsaWVudCBjYW4gaW1tZWRpYXRlbHkgcmVuZGVyIGZpbmlzaGVkIHJlc3BvbnNlcyB3aGlsZSBzdGlsbCB3YWl0aW5nIGZvciBwZW5kaW5nIHJlc3BvbnNlcykuCj4+Cj4+IFdoYXQgaXMgdGhlIGVhc2llc3Qgd2F5IHRvIGRvIHRoaXMgaW4gYSBKRGlzYyBjb250YWluZXI/IEkgd2FzIGxvb2tpbmcgYXQgd2Vic29ja2V0cyBidXQgbm90IHN1cmUgaG93IG11Y2ggdGhleSBhcmUgc3VwcG9ydGVkLgo+Pgo+PiBCZXN0LAo+PiBHYXZpbgo+Pgo+PiBFbnZpYWRvIGRlc2RlIG1pIGlQaG9uZQoKPgo+Cj4KPiBWZXNwYSBJbmZvcm1hdGlvbjoKPiAgICBodHRwOi8vdmVzcGEuY29ycC55YWhvby5jb20vCj4gICAgaHR0cDovL3R3aWtpLmNvcnAueWFob28uY29tL3ZpZXcvVmVzcGEKCj4=</origContents> - <threadUrl>/2014/02/05/jdisc_streaming_websockets</threadUrl> - <level>(NULL)</level> -</document> -<document documenttype="vespacorp" documentid="id:vespacorp:vespacorp::9de81494755bf8ee6940bcdf156081e2"> - <threadId>39e8f15ad422399d1c72d3f178f956c2</threadId> - <contentsHash>d9d95e6364aaac73b3bfb77f377236fc</contentsHash> - <urlPath>/vespa-users/msg/4f12250dae5ee19458973a0afaf6967e</urlPath> - <contents>I don't get any error while deployment the expression using created_at directly I have filed a support BZ 3455702 for this Thanks Yi On Mar 7 2010 at 10:23 AM Jo Kristian Bergum wrote On Mar 5 2010 at 11:47 PM YZ wrote Hi folks I'm trying to confirm the range of value for nativeRank is between 0 and 1 With this rank profile rank-profile native first-phase expression nativeRank I get the following result Question is relevancy showing the ranking score If so why it's 0 Depends on your query what fields were searched and your search definition I did another test this time using rank-profile created_at first-phase expression created_at This should have failed during deploy correct is expression attribute(created_at You probably have quite a few failed blueprint compilation errors in the vespa.log on the search nodes The result is as follows relevancy is still 0 and I would expect the value of created_at which is 1267569885 1267569885 Thanks Y </contents> - <parentId>df5b35d538401b06cfbe803b55856a3e</parentId> - <title>Value of nativeRank</title> - <my_contents>I don't get any error while deployment the expression using created_at directly I have filed a support BZ 3455702 for this Thanks Yi On Mar 7 2010 at 10:23 AM Jo Kristian Bergum wrote On Mar 5 2010 at 11:47 PM YZ wrote Hi folks I'm trying to confirm the range of value for nativeRank is between 0 and 1 With this rank profile rank-profile native first-phase expression nativeRank I get the following result Question is relevancy showing the ranking score If so why it's 0 Depends on your query what fields were searched and your search definition I did another test this time using rank-profile created_at first-phase expression created_at This should have failed during deploy correct is expression attribute(created_at You probably have quite a few failed blueprint compilation errors in the vespa.log on the search nodes The result is as follows relevancy is still 0 and I would expect the value of created_at which is 1267569885 1267569885 Thanks Y </my_contents> - <lastUpdate>1267989987</lastUpdate> - <pagerank>0</pagerank> - <date>1267986312</date> - <headers></headers> - <articleType>email</articleType> - <emailProcessedContents binaryencoding="base64"></emailProcessedContents> - <attachments></attachments> - <documentAbstract></documentAbstract> - <audience>(NULL)</audience> - <docDirId>(NULL)</docDirId> - <visibility>1</visibility> - <headings> - <item></item> - </headings> - <my_title>Value of nativeRank</my_title> - <messageId>4f12250dae5ee19458973a0afaf6967e</messageId> - <origContents binaryencoding="base64">SSBkb24ndCBnZXQgYW55IGVycm9yIHdoaWxlIGRlcGxveW1lbnQgdGhlIGV4cHJlc3Npb24gdXNpbmcgY3JlYXRlZF9hdCBkaXJlY3RseS4KSSBoYXZlIGZpbGVkIGEgc3VwcG9ydCBCWiAzNDU1NzAyIGZvciB0aGlzLgoKVGhhbmtzLAotWWkKCk9uIE1hciA3LCAyMDEwLCBhdCAxMDoyMyBBTSwgSm8gS3Jpc3RpYW4gQmVyZ3VtIHdyb3RlOgoKCk9uIE1hciA1LCAyMDEwLCBhdCAxMTo0NyBQTSwgWWkgWmhhbmcgd3JvdGU6Cgo+IEhpIGZvbGtzLAo+Cj4gICBJJ20gdHJ5aW5nIHRvIGNvbmZpcm0gdGhlIHJhbmdlIG9mIHZhbHVlIGZvciBuYXRpdmVSYW5rIGlzIGJldHdlZW4KPiAwIGFuZCAxLgo+Cj4gICBXaXRoIHRoaXMgcmFuayBwcm9maWxlOgo+ICAgcmFuay1wcm9maWxlIG5hdGl2ZXsKPiAgICAgZmlyc3QtcGhhc2UgeyBleHByZXNzaW9uOiBuYXRpdmVSYW5rIH0KPiAgIH0KPiAgIEkgZ2V0IHRoZSBmb2xsb3dpbmcgcmVzdWx0Ogo+Cj4gPHJlc3VsdCB0b3RhbC1oaXQtY291bnQ9IjIiPgo+IDxoaXQgcmVsZXZhbmN5PSIwIiBzb3VyY2U9InNjMC5udW0wIj4uLi4KPgo+ICAgUXVlc3Rpb246IGlzICJyZWxldmFuY3kiIHNob3dpbmcgdGhlIHJhbmtpbmcgc2NvcmU/IElmIHNvLCB3aHkgaXQncwo+IDA/CgpEZXBlbmRzIG9uIHlvdXIgcXVlcnkgKHdoYXQgZmllbGRzIHdlcmUgc2VhcmNoZWQpIGFuZCB5b3VyIHNlYXJjaApkZWZpbml0aW9uLgoKCj4gICBJIGRpZCBhbm90aGVyIHRlc3QsIHRoaXMgdGltZSB1c2luZwo+ICAgcmFuay1wcm9maWxlIGNyZWF0ZWRfYXR7Cj4gICAgIGZpcnN0LXBoYXNlIHsgIGV4cHJlc3Npb246IGNyZWF0ZWRfYXQgfQo+ICAgfQoKVGhpcyBzaG91bGQgaGF2ZSBmYWlsZWQgZHVyaW5nIGRlcGxveSwgY29ycmVjdCBpcwoKZXhwcmVzc2lvbjogYXR0cmlidXRlKGNyZWF0ZWRfYXQpCgpZb3UgcHJvYmFibHkgaGF2ZSBxdWl0ZSBhIGZldyAiZmFpbGVkIGJsdWVwcmludCBjb21waWxhdGlvbiIgZXJyb3JzIGluCnRoZSB2ZXNwYS5sb2cgb24gdGhlIHNlYXJjaCBub2Rlcz8KCgo+ICAgVGhlIHJlc3VsdCBpcyBhcyBmb2xsb3dzLCByZWxldmFuY3kgaXMgc3RpbGwgMCwgYW5kIEkgd291bGQgZXhwZWN0Cj4gdGhlIHZhbHVlIG9mIGNyZWF0ZWRfYXQsIHdoaWNoIGlzIDEyNjc1Njk4ODUuCj4gPHJlc3VsdCB0b3RhbC1oaXQtY291bnQ9IjIiPgo+IDxoaXQgcmVsZXZhbmN5PSIwIiBzb3VyY2U9InNjMC5udW0wIj4KPiA8ZmllbGQgbmFtZT0iY3JlYXRlZF9hdCI+MTI2NzU2OTg4NTwvZmllbGQ+Cj4KPiBUaGFua3MsCj4gLVlpCj4gPG1lc3NhZ2UtZm9vdGVyLnR4dD4=</origContents> - <threadUrl>/2010/03/05/value_of_nativerank</threadUrl> - <level>(NULL)</level> -</document> -<document documenttype="vespacorp" documentid="id:vespacorp:vespacorp::25ec8cc793e7e49cee2b5584917f6006"> - <threadId>1a60037c3896299b0a90dd5d62034529</threadId> - <contentsHash>7d08e2ecdc2a7d243a748059db0e09ee</contentsHash> - <authors> - <item>bratseth</item> - </authors> - <urlPath>/vespa-users/msg/3eac9311090c9cf7c34f9d8c897f2c4b</urlPath> - <contents> search=incr&restrict=abc RJ skrev Hi Jo We have multiple SDs deployed in one cluster and we normally use search= in our queries How do we use it for differentiating both the cluster name and the sd name For example We have two clusters namely incr and realtime each having SDs abc xyz We need a way to query from incr cluster in abc sd Thanks RJ Jo Kristian Bergum wrote On Mon 2009-08-24 at 16:55 +0530 AS wrote On the serving side how do we indicate the cluster to get results from search= /JKB </contents> - <parentId>00b89ea33262247e728f7534c9eafd0d</parentId> - <title>Need a real time index along with an Incremental index</title> - <my_contents> search=incr&restrict=abc RJ skrev Hi Jo We have multiple SDs deployed in one cluster and we normally use search= in our queries How do we use it for differentiating both the cluster name and the sd name For example We have two clusters namely incr and realtime each having SDs abc xyz We need a way to query from incr cluster in abc sd Thanks RJ Jo Kristian Bergum wrote On Mon 2009-08-24 at 16:55 +0530 AS wrote On the serving side how do we indicate the cluster to get results from search= /JKB </my_contents> - <lastUpdate>1265031079</lastUpdate> - <pagerank>0</pagerank> - <date>1251115178</date> - <headers></headers> - <articleType>email</articleType> - <emailProcessedContents binaryencoding="base64"></emailProcessedContents> - <attachments></attachments> - <documentAbstract></documentAbstract> - <audience>(NULL)</audience> - <docDirId>(NULL)</docDirId> - <visibility>1</visibility> - <headings> - <item></item> - </headings> - <my_title>Need a real time index along with an Incremental index</my_title> - <messageId>3eac9311090c9cf7c34f9d8c897f2c4b</messageId> - <origContents binaryencoding="base64">JnNlYXJjaD1pbmNyJnJlc3RyaWN0PWFiYw0KDQpSYWphdCBKYWluIHNrcmV2Og0KPiBIaSBKbywNCj4NCj4gV2UgaGF2ZSBtdWx0aXBsZSBTRHMgZGVwbG95ZWQgaW4gb25lIGNsdXN0ZXIgYW5kIHdlIG5vcm1hbGx5IHVzZSANCj4gJnNlYXJjaD08c2RuYW1lPiBpbiBvdXIgcXVlcmllcy4gSG93IGRvIHdlIHVzZSBpdCBmb3IgZGlmZmVyZW50aWF0aW5nIA0KPiBib3RoIHRoZSBjbHVzdGVyIG5hbWUgYW5kIHRoZSBzZCBuYW1lPw0KPiBGb3IgZXhhbXBsZSwNCj4gICAgIFdlIGhhdmUgdHdvIGNsdXN0ZXJzIG5hbWVseSAiaW5jciIgYW5kICJyZWFsdGltZSIsIGVhY2ggaGF2aW5nIA0KPiBTRHMsICJhYmMiICYgInh5eiIuIFdlIG5lZWQgYSB3YXkgdG8gcXVlcnkgZnJvbSBpbmNyIGNsdXN0ZXIgaW4gYWJjIHNkLg0KPg0KPiBUaGFua3MhDQo+IFJhamF0IEphaW4NCj4NCj4gSm8gS3Jpc3RpYW4gQmVyZ3VtIHdyb3RlOg0KPj4gT24gTW9uLCAyMDA5LTA4LTI0IGF0IDE2OjU1ICswNTMwLCBBbWl0IFNpbmhhIHdyb3RlOg0KPj4gICANCj4+PiBPbiB0aGUgc2VydmluZyBzaWRlIGhvdyBkbyB3ZSBpbmRpY2F0ZSB0aGUgY2x1c3RlciB0byBnZXQgcmVzdWx0cw0KPj4+IGZyb20uDQo+Pj4gICAgIA0KPj4gJnNlYXJjaD08Y2x1c3Rlcm5hbWU+IA0KPj4NCj4+IC9KS0INCj4+DQo+Pg==</origContents> - <threadUrl>/2009/08/20/partial_update_of_the_entire_document</threadUrl> - <level>(NULL)</level> -</document> -<document documenttype="vespacorp" documentid="id:vespacorp:vespacorp::c146d4c9f6153c4573f9a22fd5b19eb3"> - <threadId>5b3a4526d33f45b09f63d43f476f824a</threadId> - <contentsHash>3700f8d3da3a8963f34f11816ffda2cc</contentsHash> - <authors> - <item>bergum</item> - </authors> - <urlPath>/vespa-users/msg/a0c55f0c7669d097e470182bd7d411cf</urlPath> - <contents>On Mon 2009-05-18 at 15:44 +0800 JLB wrote Hi All Am looking for a documentation on how to setup Vespa 1.1.5 I found this link http://vespa/documentation/setup/howtorun.html and it's not available anymore Am just wondering if you have it somewhere If so can you send it to me Thank you Why do you want to use Vespa 1.1.5 It's legacy and not longer supported Please consider using latest stable vespa release http://vespa Best Jo Kristian J plain text document attachment message-footer.txt Vespa Information http://vespa</contents> - <parentId>acca854c8876330b43143c61a1e1f32c</parentId> - <title>Vespa 1.1.5 Documentation</title> - <my_contents>On Mon 2009-05-18 at 15:44 +0800 JLB wrote Hi All Am looking for a documentation on how to setup Vespa 1.1.5 I found this link http://vespa/documentation/setup/howtorun.html and it's not available anymore Am just wondering if you have it somewhere If so can you send it to me Thank you Why do you want to use Vespa 1.1.5 It's legacy and not longer supported Please consider using latest stable vespa release http://vespa Best Jo Kristian J plain text document attachment message-footer.txt Vespa Information http://vesp </my_contents> - <lastUpdate>1265031079</lastUpdate> - <pagerank>0</pagerank> - <date>1242632220</date> - <headers></headers> - <articleType>email</articleType> - <emailProcessedContents binaryencoding="base64"></emailProcessedContents> - <attachments></attachments> - <documentAbstract></documentAbstract> - <audience>(NULL)</audience> - <docDirId>(NULL)</docDirId> - <visibility>1</visibility> - <headings> - <item></item> - </headings> - <my_title>Vespa 1.1.5 Documentation</my_title> - <messageId>a0c55f0c7669d097e470182bd7d411cf</messageId> - <origContents binaryencoding="base64">T24gTW9uLCAyMDA5LTA1LTE4IGF0IDE1OjQ0ICswODAwLCBKZXJvbWUgTGVtdWVsIEJhc2Egd3JvdGU6Cj4gSGkgQWxsLAo+IAo+IEFtIGxvb2tpbmcgZm9yIGEgZG9jdW1lbnRhdGlvbiBvbiBob3cgdG8gc2V0dXAgVmVzcGEgMS4xLjUsIEkgZm91bmQgdGhpcyAKPiBsaW5rOiAKPiBodHRwOi8vdmVzcGEudHJvbmRoZWltLmNvcnAueWFob28uY29tLzEuMS41L2RvY3VtZW50YXRpb24vc2V0dXAvaG93dG9ydW4uaHRtbCAKPiBhbmQgaXQncyBub3QgYXZhaWxhYmxlIGFueW1vcmUuIEFtIGp1c3Qgd29uZGVyaW5nIGlmIHlvdSBoYXZlIGl0IAo+IHNvbWV3aGVyZS4gSWYgc28sIGNhbiB5b3Ugc2VuZCBpdCB0byBtZS4gVGhhbmsgeW91LgoKV2h5IGRvIHlvdSB3YW50IHRvIHVzZSBWZXNwYSAxLjEuNT8gSXQncyBsZWdhY3kgYW5kIG5vdCBsb25nZXIKc3VwcG9ydGVkLCBQbGVhc2UgY29uc2lkZXIgdXNpbmcgbGF0ZXN0IHN0YWJsZSB2ZXNwYSByZWxlYXNlLiAKaHR0cDovL3Zlc3BhLmNvcnAueWFob28uY29tLwoKQmVzdCwKSm8gS3Jpc3RpYW4gCgoKCj4gCj4gLUplcm9tZQo+IHBsYWluIHRleHQgZG9jdW1lbnQgYXR0YWNobWVudCAobWVzc2FnZS1mb290ZXIudHh0KQo+IFZlc3BhIEluZm9ybWF0aW9uOgo+ICAgICAgaHR0cDovL3Zlc3BhLmNvcnAueWFob28uY29tLwo+ICAgICAgaHR0cDovL3R3aWtpLmNvcnAueWFob28uY29tL3ZpZXcvVmVzcGEKPg==</origContents> - <threadUrl>/2009/05/18/vespa_1_1_5_documentation</threadUrl> - <level>(NULL)</level> -</document> -<document documenttype="vespacorp" documentid="id:vespacorp:vespacorp::0553f1ea3af33d11aa9ab42496d11f78"> - <threadId>cb34c66221f7188a09e6151062c14e16</threadId> - <contentsHash>fba771ebc63a57f191cf783b0a59298f</contentsHash> - <authors> - <item>peng</item> - </authors> - <urlPath>/vespa-users/msg/079588cac2dfd1117bef476409da724b</urlPath> - <contents>There is no rules to write When you enable libyell_poststemmer if a word is not known by dictionary libyell will try to do plural singular stemming according to its builtin rules Peng Original Message From BK mailto:mail@yhost.com Sent Tuesday July 03 2007 3:10 AM To mail@host.com Subject rule based stemming in vespa Hi I came across this document http://VespaStemming on plural singular stemming I want to try rule based plural singular stemming libyell_poststemmer in vespa Can someone point me to the relevant documents on how to write these rules and use them Thanks B </contents> - <parentId>3be4bf063c46c7e3d336dbfb4c58f6e7</parentId> - <title>Rule based stemming in vespa</title> - <my_contents>There is no rules to write When you enable libyell_poststemmer if a word is not known by dictionary libyell will try to do plural singular stemming according to its builtin rules Peng Original Message From BK mailto:mail@yhost.com Sent Tuesday July 03 2007 3:10 AM To mail@host.com Subject rule based stemming in vespa Hi I came across this document http://LocalVespaStemming on plural singular stemming I want to try rule based plural singular stemming libyell_poststemmer in vespa Can someone point me to the relevant documents on how to write these rules and use them Thanks B </my_contents> - <lastUpdate>1265031079</lastUpdate> - <pagerank>0</pagerank> - <date>1184187769</date> - <headers></headers> - <articleType>email</articleType> - <emailProcessedContents binaryencoding="base64"></emailProcessedContents> - <attachments></attachments> - <documentAbstract></documentAbstract> - <audience>(NULL)</audience> - <docDirId>(NULL)</docDirId> - <visibility>1</visibility> - <headings> - <item></item> - </headings> - <my_title>Rule based stemming in vespa</my_title> - <messageId>079588cac2dfd1117bef476409da724b</messageId> - <origContents binaryencoding="base64">VGhlcmUgaXMgbm8gcnVsZXMgdG8gd3JpdGUuIFdoZW4geW91IGVuYWJsZSBsaWJ5ZWxsX3Bvc3RzdGVtbWVyLCBpZiBhIHdvcmQKaXMgbm90IGtub3duIGJ5IGRpY3Rpb25hcnksIGxpYnllbGwgd2lsbCB0cnkgdG8gZG8gcGx1cmFsLT5zaW5ndWxhciBzdGVtbWluZwphY2NvcmRpbmcgdG8gaXRzIGJ1aWx0aW4gcnVsZXMuCgpQZW5nCgogCgotLS0tLU9yaWdpbmFsIE1lc3NhZ2UtLS0tLQpGcm9tOiBCYWxhamkgS2FubmFuIFttYWlsdG86a2JhbGFqaUB5YWhvby1pbmMuY29tXSAKU2VudDogVHVlc2RheSwgSnVseSAwMywgMjAwNyAzOjEwIEFNClRvOiB2ZXNwYS11c2Vyc0B5YWhvby1pbmMuY29tClN1YmplY3Q6IHJ1bGUgYmFzZWQgc3RlbW1pbmcgaW4gdmVzcGEgCgpIaSwKICBJIGNhbWUgYWNyb3NzIHRoaXMgZG9jdW1lbnQKaHR0cDovL3R3aWtpLmNvcnAueWFob28uY29tL3ZpZXcvTG9jYWxlbmcvTG9jYWxWZXNwYVN0ZW1taW5nIG9uIApwbHVyYWwtPnNpbmd1bGFyIHN0ZW1taW5nLiAgIEkgd2FudCB0byB0cnkgcnVsZSBiYXNlZCBwbHVyYWwtPnNpbmd1bGFyIApzdGVtbWluZyAobGlieWVsbF9wb3N0c3RlbW1lcikgaW4gdmVzcGEuICBDYW4gc29tZW9uZSBwb2ludCBtZSB0byB0aGUKcmVsZXZhbnQgZG9jdW1lbnRzIG9uIGhvdyB0byB3cml0ZSB0aGVzZSBydWxlcyBhbmQgdXNlIHRoZW0uCgpUaGFua3MKQmFsYWpp</origContents> - <threadUrl>/2007/07/03/rule_based_stemming_in_vespa</threadUrl> - <level>(NULL)</level> -</document> -</vespafeed> diff --git a/vespa-http-client/src/test/resources/xml-challenge.xml b/vespa-http-client/src/test/resources/xml-challenge.xml deleted file mode 100644 index 55e368732d7..00000000000 --- a/vespa-http-client/src/test/resources/xml-challenge.xml +++ /dev/null @@ -1,6 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<vespafeed> - <document documenttype="biz" transformver="5681" documentid="id:lsbe:biz::21336977"><attrlist><![CDATA[<other_urls><n>2</n><l><m><url>http://www.facebook.com/pages/City-of-Sunnyvale-California/132586463442411</url><URLTYPE>facebook</URLTYPE></m><m><url>http://www.twitter.com/CityofSunnyvale</url><URLTYPE>twitter</URLTYPE></m></l></other_urls><toc>19,22,36,42,48,74</toc><website><m><src>GRID</src><url>http://www.sunnyvale.ca.gov/</url></m></website><neighbor>Downtown|Sunnyvale Town Center</neighbor><woeId>Zip:12797147;DMA:24701119;State:2347563</woeId><consumersubmit><addbyuser>0</addbyuser></consumersubmit>]]></attrlist></document> - -</vespafeed> diff --git a/vespa-http-client/src/test/resources/xml-challenge2.xml b/vespa-http-client/src/test/resources/xml-challenge2.xml deleted file mode 100644 index 7ed0be68dea..00000000000 --- a/vespa-http-client/src/test/resources/xml-challenge2.xml +++ /dev/null @@ -1,5 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<vespafeed> - <document documenttype="biz" transformver="5681" documentid="id:lsbe:biz::21336977"><version_index>1395987733</version_index><attrlist><other_urls><n>2</n><l><m><url>http://www.facebook.com/pages/City-of-Sunnyvale-California/132586463442411</url><URLTYPE>facebook</URLTYPE></m><m><url>http://www.twitter.com/CityofSunnyvale</url><URLTYPE>twitter</URLTYPE></m></l></other_urls><toc>19,22,36,42,48,74</toc><website><m><src>GRID</src><url>http://www.sunnyvale.ca.gov/</url></m></website><neighbor>Downtown|Sunnyvale Town Center</neighbor><woeId>Zip:12797147;DMA:24701119;State:2347563</woeId><consumersubmit><addbyuser>0</addbyuser></consumersubmit></attrlist><dispycat>96929308:City Hall:1|96927047:Government:1</dispycat><ip_hoo5>0|0|0|0|0|0|0|0</ip_hoo5><ip_popularity_features>0.01401179941|0.0780352748154|0.0|0.00020|0.00008</ip_popularity_features><s1><CTS>96925679 96927047 96929308</CTS><S_CNSUMR_SUBMT>0</S_CNSUMR_SUBMT><DR_PLUS4>7619</DR_PLUS4></s1><ll>-122037613;37371072</ll><stat>1 3 4 6 7 8 11 12 13 14 15 16 33 34 36 41 42 43 44 46 48 49 50 51 57 58 59 60 71 72 73 74 75 76 77 80 81 82 83 85 86 87 131 132 134 137 138 139 140 141 143 144 159 181 183 184 192 193 196 197 198 204</stat><uri>21336977</uri><title>sch | sunnyvalecity | sunnyvalecityhall | hall | city | sunnyvale</title><dtitle>Sunnyvale City Hall</dtitle><ip_hoo2>0|0|0|0|0|0|0|0</ip_hoo2><citystate>Sunnyvale CA</citystate><nhelpreview>0</nhelpreview><ll_long>-122037613</ll_long><ip_custombyte>7|57|103</ip_custombyte><prior_rating>0</prior_rating><ip_hoo4>0|0|0|0|0|0|0|0</ip_hoo4><ip_ycatid2>96929308|96927047</ip_ycatid2><ftitle>20</ftitle><ip_ycat_primary_id>96929308|96927047</ip_ycat_primary_id><ip_eid>NAV=17555658|PSOXSOCIALURL=21336977|INFOUSA=102028107</ip_eid><zip>94086</zip><prior_nreview>0</prior_nreview><ip_customweights>780|33|.09102564102564102|49.732394366197184|121739|3060000|717|1860|716|1860|459|1400|457|1400|114|913|-1|-1|-1|-1|0.0862069|0.12267753</ip_customweights><type>POI</type><enhanced>1</enhanced><ip_normtitle>sunnyvale city hall</ip_normtitle><primary_url>http://www.sunnyvale.ca.gov/</primary_url><ip_hoo1>0|0|0|0|0|0|0|0</ip_hoo1><dma>24701119</dma><city>Sunnyvale</city><ip_catkey>city hall|government</ip_catkey><prior_nrating>0</prior_nrating><lcw_ext><item weight="0">0</item></lcw_ext><pop0>0</pop0><ll_lat>37371072</ll_lat><phone>4087307500</phone><dispfeaturef>17</dispfeaturef><ip_ycat_primary>City Hall|Government</ip_ycat_primary><ip_hoo6>0|0|0|0|0|0|0|0</ip_hoo6><ip_ycat_primary_synonyms>county government|marriage licenses|us government|government-office|government relations firms|government agencies|government relations|marriage license|city government departments|city government|federal government|usgovernment|council of governments|governmentoffices|government offices|government officials|government-contract consultants|governments offices</ip_ycat_primary_synonyms><ip_dcat>City Hall|Government</ip_dcat><listing_status>103</listing_status><lcw_pcat><item weight="7071">96929308</item><item weight="7071">96927047</item></lcw_pcat><pop_keyword_portion><item weight="2">3053931</item><item weight="1">99043162</item><item weight="57">-734328434</item><item weight="6">-1164754482</item><item weight="33">-1164681749</item><item weight="10000">-1491073460</item><item weight="5000">1098856921</item><item weight="5000">90458857</item><item weight="10000">-1956199191</item><item weight="10000">-2015626891</item><item weight="10000">38487508</item><item weight="1428">-1384211533</item><item weight="588">-995864441</item><item weight="6666">-350418064</item><item weight="112">-1493944221</item></pop_keyword_portion><stat2>2 3 4 5 7 8 10 11 21 24 31 42 44 45 46 47 48 50 51 53 54 56 57 69 92 112 121 123 128 129 134 154 195 205 232 238 242</stat2><desc> </desc><ip_rating>0</ip_rating><lcw><item weight="5745">96927047</item><item weight="8186">96929308</item></lcw><pop_keyword_certainty><item weight="5425">3053931</item><item weight="5344">99043162</item><item weight="7105">-734328434</item><item weight="5140">-1164754482</item><item weight="6766">-1164681749</item><item weight="10000">-1491073460</item><item weight="9587">1098856921</item><item weight="9587">90458857</item><item weight="10000">-1956199191</item><item weight="10000">-2015626891</item><item weight="10000">38487508</item><item weight="8842">-1384211533</item><item weight="9120">-995864441</item><item weight="9621">-350418064</item><item weight="7429">-1493944221</item></pop_keyword_certainty><q>9</q><isactive>1</isactive><ip_ycat2gc>96929308|96927047</ip_ycat2gc><ip_hoo0>0|0|0|0|0|0|0|0</ip_hoo0><language>en</language><nreview>0</nreview><ip_pycatnames>Government & Community</ip_pycatnames><ip_catkey_click>marriage license</ip_catkey_click><country_code>us</country_code><state>CA</state><ip_hoo3>0|0|0|0|0|0|0|0</ip_hoo3><version_pub>1346889600</version_pub><ip_ycat_primary_cp_desc>CITY GOVERNMENT-EXECUTIVE OFFICES</ip_ycat_primary_cp_desc><crossst>2|All America Way|Charles St</crossst><ip_dcatkey>us government|government relations firms|marriage license|federal government|council of governments|governmentoffices|government officials|government-contract consultants|governments offices|county government|marriage licenses|government-office|government agencies|government relations|city government departments|city government|usgovernment|government offices</ip_dcatkey><dispambiancef>7</dispambiancef><ip_ycatid1>96925679</ip_ycatid1><paid_listing_status>0</paid_listing_status><lcw_norm>0.5640622</lcw_norm><webkeyword>city sunnyvale twitter jobs news contact us about codes policies charter municipal code council policy general plan maps directions map hall external link library public safety parks golf courses swimming pools tennis center transportation area resources community resource guide frequently requested english spanish budget documents data demographics business economic profile learn new resident information sheet this site privacy what hot topics plastic bag ban amendment ballot measures elections firearms retail study issue review committee medical marijuana dispensary horizon downtown redevelopment sustainability consolidation lute update manager updates meetings agendas next meeting february boards commissions latest publications fall activity winter quarterly report current job openings bid on projects open bids around cited as american top potential upcoming events issues workshop feb offices are closed more event calendar government agenda watch making presentation at arts bicycle pedestrian board trustees building appeals heritage preservation housing human recreation personnel planning other agencies county santa clara state california clerk page campaign ethics departments attorney development environmental nova workforce finance technology works living get card search catalog manage your account getting involved volunteering neighborhood associations classes activities out now playing performing shopping dining in centers columbia senior guides unemployed residents infrastructure traffic trees street maintenance garbage recycling smart station extra tags affordable assistance water supply pollution control plant police fire emergency preparedness alarms enforcement animal crime prevention records recruiting permits special residential non one stop permit checks fees encroachment tree removal doing starting facts figures bidding process shop auto row available commercial properties licenses reports links division newsroom recent releases apple occupy president day holiday closures meet local author francisco jimenez reads muslim door arrest made fatal hit run le jazz copper wire theft hits television broadcast schedule social media follow want apply license block party sign up service call tee times facility reservations compost pay my utility bill fines violation graffiti pothole web dispose old medication waste hazardous trash find department places dine volunteer access file claim help improve list something missing from that would make it even useful let know we welcome suggestions friday share comments tuesday strategic featured telephone scam claims be computer has received complaints apparent phone which caller identifies himself representing victim windows system transmitting bad does not these kind calls have if receive type desk officer gary announced will join ranks high tech businesses located square foot town office mathilda mckinley avenues important step forward said entire benefit see yet another large gain project read planned by pg along caribbean drive remove number mostly eucalyptus measure underground gas pipeline young replacement planted median controls during work or go amp presidents facilities including monday observance all parking regulations enforced except where signs specifically exempt holidays collection continue normal posted edition gives tip leading burglars vargas elementary school partnership marathon club kids led streetlight free healthy toddler workshops electronic commission change please note dates do typical due construction chambers scheduled wednesday artists applications hands festival participate may downloading click here download application acclaimed quartet performs gypsy la reinhardt version internationally recognized san perform valentine weekend concert style theatre saturday silicon valley diverse religions cultures been celebrated each year reading choices continues select provocative relevant theme off air channel broadcasts equipment upgrade allowing compliant standards begin broadcasting again no down time so missed vehicle versus accident occurred intersection sequoia reed avenue january driver black struck benjamin lin did major investigation team developed significant leads case identification seizure warrant was issued popular register november networks google divider final eir sales hazard mitigation subscribe notifications feed icon olive ave ca logo can check stay touch used maintained communications questions fine print terms use</webkeyword><ip_provider>NAV|PSOXSOCIALURL|INFOUSA</ip_provider><nrating>0</nrating><ratingfgc>0</ratingfgc><ip_ycat2>96925679</ip_ycat2><spaid>N</spaid><ip_cat>GOVERNMENTOFFICES|OFFICES|HALL|GOVERNMENT|DEPARTMENTS|GOVERNMENTS|CONSULTANTS|OFFICIALS|RELATIONS|USGOVERNMENT|OFFICE|CONTRACT|COUNCIL|CITY|COMMUNITY|FIRMS|AGENCIES|COUNTY</ip_cat><ip_neighborhood>downtownsunnyvaleca|sunnyvaletowncentersunnyvaleca</ip_neighborhood><addrhash>068B74475DB0D415</addrhash><webtext>City of Sunnyvale: Home * Twitter * | * Jobs * | * News * | * eNotify * | * RSS * | * Contact Us Home About The City Codes and Policies City Charter Municipal Code Council Policy General Plan Maps and Directions Map of Sunnyvale City Hall external link Library Public Safety City Parks Golf Courses Swimming Pools Tennis Center Public Transportation Area Resources Community Resource Guide to Frequently Requested Services: English (pdf) | Spanish (pdf) City Budget Budget Documents Data and Demographics Business Demographics Economic Profile (pdf) Learn about Sunnyvale New Resident Guide City Council Information Sheet About This Site About the Site Site Map Privacy Policy Contact Us Whats New Hot Topics Plastic Bag Ban Charter Amendment Ballot Measures City Council Elections Firearms Retail Study Issue Charter Review Committee Medical Marijuana Dispensary Study Horizon 2035 Downtown Redevelopment Onizuka / BRAC Sustainability General Plan Consolidation LUTE Update City Managers Updates Meetings and Agendas Next Council Meeting: February 7 Council Meetings Boards and Commissions Latest Publications Fall Activity Guide Winter 2012 Quarterly Report 2010 New Resident Guide Jobs Current Job Openings Bid on Projects Current Open Bids Around the City Sunnyvale Cited As American City with Top Economic Potential Upcoming Events February 3 - Council Budget/Study Issues Workshop Feb 20 - City Offices are closed More Events on the Community Event Calendar Government City Council About Council Current Council Agenda Councilmembers Council Meeting Agendas Watch Council Meetings Online Making a Presentation at Council Boards and Commissions About Boards and Commissions Current Openings Arts Bicycle and Pedestrian Board of Library Trustees Building Code Appeals Heritage Preservation Housing and Human Services Parks and Recreation Personnel Planning Sustainability Study Issues 2012 Study Issues 2011 Study Issues 2010 Study Issues 2009 Study Issues Other Agencies County of Santa Clara external link State of California external link Codes and Policies City Charter Municipal Code Council Policy General Plan Elections City Clerk Elections Page Campaign Ethics Guide Departments City Attorney City Manager Community Development Community Services Environmental Services NOVA Workforce Services Finance Human Resources Information Technology Public Safety Public Works Sunnyvale Public Library Living Library Library Home Page Get a Library Card Search the Library Catalog Manage Your Library Account Getting Involved Volunteering Neighborhood Associations Recreation Golf Tennis Parks Classes and Activities Swimming Out and About Now Playing at the Performing Arts Center Shopping and Dining in Sunnyvale Community Event Calendar Community Centers Columbia Neighborhood Center Sunnyvale Community Center Sunnyvale Senior Center Resource Guides Resources for Unemployed Residents Community Resource Guide to Frequently Requested Services: English (pdf) | Spanish (pdf) Services City Infrastructure Traffic and Transportation Trees Street Maintenance Garbage and Recycling SMaRT Station Garbage and Recycling Services Extra Garbage Tags Housing Affordable Housing and Community Assistance Water Water Supply Water Pollution Control Plant (WPCP) Public Safety Police Fire Emergency Preparedness Alarms Code Enforcement Animal Control Crime Prevention Public Safety Records Public Safety Recruiting Permits Special Event Permits Residential Permits Non-residential Permits E-Onestop One-Stop Permit Center Permits, Plan Checks and Fees Encroachment Permits Tree Removal Permits Doing Business Sunnyvale for Business Starting a Business in Sunnyvale Facts and Figures Doing Business in Sunnyvale Economic Development Downtown Development Bid on Sunnyvale Projects The Bidding Process Current Open Bids Shop Sunnyvale Sunnyvale Auto Row Business Resources Available Commercial Properties E-One Stop Permit Center Business Licenses Business News and Reports Business Links Building Division Planning Division Newsroom Recent News Releases Apple to Occupy New Downtown Sunnyvale Offices Presidents Day Holiday Closures Meet Sunnyvale Local Author Francisco Jimenez Sunnyvale Reads the Muslim Next Door Arrest Made in Fatal Sunnyvale Hit-And-Run Le Jazz Hot in Sunnyvale February 11 Copper Wire Theft Hits City of Sunnyvale More News Releases Publications Quarterly Report Activity Guide Senior Activity Guide New Resident Guide Campaign Ethics Guide On Television KSUN Broadcast Schedule Social Media Follow Us on Twitter City of Sunnyvale Facebook Page I Want To . Apply For: Job Openings Boards and Commissions Business License Permits Animal License Special Event Permit Block Party Sign Up For: Garbage Service Recycling Service On-Call Garbage/Recycling Service Recreation Classes Golf Tee Times Facility Reservations Compost Workshop Pay For: My Utility Bill Library Fines Report: Code Violation Graffiti Pothole Public Safety Issue Traffic Issue General Issue Web Site Issue Dispose of: Old Medication E-Waste Other Hazardous Waste Extra Trash Find: Building Permits City Hall external link City Parks Sunnyvale Community Center Columbia Neighborhood Center Sunnyvale Senior Center Sunnyvale Public Library Department of Public Safety Places to Shop and Dine Watch: Council Meetings Online Other: Volunteer in the City Access Public Records File a Claim Help Improve This List Is something missing from this list that would make it an even more useful resource? Let us know! We welcome your suggestions. Contact us online. Friday, February 10, 2012 Share Your Comments Next Council Meeting * Tuesday, February 7, 2012 * Council Agendas * Watch the Meeting Online * Watch the Meeting on KSUN-15 * About Council Upcoming Events * February 10 - Strategic Planning Workshop * February 20 - City Holiday - Offices closed * Community Events Calendar Recent News Releases Apple to Occupy New Downtown Sunnyvale Offices Presidents Day Holiday Closures Meet Sunnyvale Local Author Francisco Jimenez Sunnyvale Reads the Muslim Next Door Arrest Made in Fatal Sunnyvale Hit-And-Run Le Jazz Hot in Sunnyvale February 11 More News Releases Featured City News Telephone Scam Claims to be from Sunnyvale Computer Department The City has received complaints of an apparent phone scam in which the caller identifies himself as representing the Sunnyvale Computer Department. The caller claims the victims computers Windows system is transmitting bad data. City of Sunnyvale does not make these kind of calls to the public and does not have a Computer Department. If you receive this type of scam call, report the call to a Public Safety Desk Officer at (408) 730-7110.Learn More Telephone Scam Claims to be from "Sunnyvale Computer Department" Apple to occupy new downtown Sunnyvale offices Sunnyvale City Manager Gary Luebbers has announced that Apple will join the ranks of high-tech businesses located in downtown Sunnyvale. Apple will occupy the new 156,000 square-foot Town Center Office building at Mathilda and McKinley avenues. ?This is an important step forward,? said Luebbers. Our entire community will benefit as we see yet another large gain in the Town Center redevelopment project.? Read more Learn More Apple to occupy new downtown Sunnyvale offices Tree Removal Planned by PG&E along Caribbean Drive PG&E will remove a number of large trees ? mostly eucalyptus ? along Caribbean Drive, February 6 ? 15. This is a safety measure for the underground gas pipeline. Young replacement trees will planted in the median; watch for traffic controls during this work. For more information, call PG&E at (800) 743-5000 or go to http://pge.com/myhome/customerservice/Learn More Tree Removal Planned by PG&amp;E along Caribbean Drive City Closed for Presidents Day Sunnyvale City offices and facilities, including the Sunnyvale Public Library, Community Center, Senior Center and Columbia Neighborhood Center will be closed Monday, February 20, in observance of the Presidents Day holiday. All traffic and parking regulations will be enforced, except for parking where signs specifically exempt holidays. Trash collection will continue on a normal schedule. Learn More City Closed for Presidents Day Latest City Managers Blog Posted In the latest edition of the City Managers Blog, Gary Luebbers gives an update on the 2012 Study Issues, a public tip leading to the arrest of local burglars, the Vargas Elementary School partnership with Public Safety for a Marathon Club for kids, latest street updates and the LED Streetlight project, free healthy toddler workshops at the Library, a new electronic Job Board from NOVA, and more Learn More Latest City Managers Blog Posted Planning Commission February Meeting Schedule Change PLEASE NOTE: The February Planning Commission meeting dates do not follow the typical schedule due to construction in the Council Chambers. The February Planning Commission Meetings are scheduled for: * MONDAY, FEBRUARY 6, 2012 * WEDNESDAY, FEBRUARY 29, 2012 Learn More Planning Commission February Meeting Schedule Change Artists Applications for 2012 Hands on the Arts Festival Now Posted Applications for artists to participate in the 2012 Hands on the Arts Festival in Sunnyvale on May 19 are now available for downloading. Click here to read more and download the application Learn More Artists Applications for 2012 Hands on the Arts Festival Now Posted Le Jazz Hot in Sunnyvale February 11 Acclaimed quartet performs Gypsy jazz la Django Reinhardt Le Jazz Hot, the quartet version of internationally recognized Le Hot Club of San Francisco, will perform a Valentine?s weekend concert of Django Reinhardt-style Gypsy jazz at Sunnyvale Theatre on Saturday, February 11, at 8 p.m. Read more Learn More Le Jazz Hot in Sunnyvale February 11 Sunnyvale Reads The Muslim Next Door Silicon Valley?s diverse religions and cultures have been celebrated each year by reading choices from Silicon Valley Reads, which continues to select provocative topics relevant to Santa Clara County. The theme for Silicon Valley Reads 2012 is ?Muslim and American.? Read more Learn More Sunnyvale Reads "The Muslim Next Door" KSUN-15 Off Air February 8-27 for Ugrades The Citys public access channel KSUN-15, which broadcasts Council and Planning Commission meetings, will be off-air starting February 8 to get an equipment upgrade, allowing the system to be compliant with current broadcast standards. KSUN will begin broadcasting again on February 27. No meetings are planned during this down time so no broadcasts will be missed. Learn More KSUN-15 Off Air February 8-27 for Ugrades Arrest Made in Fatal Hit-and-Run in Sunnyvale A fatal vehicle versus pedestrian accident occurred at the intersection of Sequoia Drive and Reed Avenue in Sunnyvale on January 5. The driver of a black SUV struck 72-year-old Benjamin Lin and did not stop. The Major Accident Investigation Team (MAIT) developed significant leads in the case which led to the identification of the driver and seizure of the vehicle. On January 19, a $500,000 arrest warrant was issued. Read more Learn More Arrest Made in Fatal Hit-and-Run in Sunnyvale Popular Links * Jobs * Sunnyvale Public Library * Register for Classes and Activities * Recycling and Garbage Information * Volunteer * Maps and Directions * Pay Your Utility Bill Online * November 2011 Library Events Calendar in PDF Pay Your Utility Bill Online Utility Bill Online Pay Doing Business * Bid on City Projects * Economic Development * Business Licenses * Apply for Permits Online Social Networks * +1 us on Google+ * City of Sunnyvale Twitter Account * City of Sunnyvale Facebook Page divider Hot Topics * Tennis Center * Plastic Bag Ban Final EIR * Firearms Sales Study Issue * Sunnyvale Works! * Downtown Development * Sustainability * Onizuka / BRAC * Local Hazard Mitigation Plan (LMPH) Subscribe * e-Notifications * City Managers Blog * RSS Feed * RSS Feed icon City of Sunnyvale (408) 730-7500 * Sunnyvale City Hall * 456 W. Olive Ave. * Sunnyvale, CA 94086 * TDD (408) 730-7501 * Map and Directions * City of Sunnyvale Logo Cant Find It? * Or check out the SITE MAP! Stay In Touch * Contact Us * Follow us on Twitter Frequently-Used Links * Jobs with the City * Library * Garbage and Recycling * Downtown Redevelopment * Economic Development * Register for Classes and Activities About the City * Welcome to Sunnyvale * City Charter and Policies * City Council * City Departments * New Resident Guide About the Web Site The City of Sunnyvale Web Site is maintained by the Sunnyvale Communications Office and the Department of Information Technology. Questions? Contact Us. The Fine Print * Privacy Policy * Terms of Use 2010 City of Sunnyvale</webtext><ip_keyword>[ ca ]</ip_keyword><addr>456 W Olive Ave</addr></document> -</vespafeed> diff --git a/vespa-http-client/src/test/resources/xml-challenge3.xml b/vespa-http-client/src/test/resources/xml-challenge3.xml deleted file mode 100644 index be0f789f870..00000000000 --- a/vespa-http-client/src/test/resources/xml-challenge3.xml +++ /dev/null @@ -1,4 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<vespafeed> - <document documenttype="biz" transformver="5681" documentid="id:lsbe:biz::21336977"><attrlist>''</attrlist></document></vespafeed> diff --git a/vespa-maven-plugin/pom.xml b/vespa-maven-plugin/pom.xml index 5984c6a1324..8f46e5ff35f 100644 --- a/vespa-maven-plugin/pom.xml +++ b/vespa-maven-plugin/pom.xml @@ -7,7 +7,7 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa-maven-plugin</artifactId> diff --git a/vespa-osgi-testrunner/pom.xml b/vespa-osgi-testrunner/pom.xml index 4fa651a6930..ebb1240a198 100644 --- a/vespa-osgi-testrunner/pom.xml +++ b/vespa-osgi-testrunner/pom.xml @@ -6,7 +6,7 @@ <parent> <artifactId>parent</artifactId> <groupId>com.yahoo.vespa</groupId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <modelVersion>4.0.0</modelVersion> @@ -23,6 +23,13 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>container-core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>hosted-api</artifactId> <version>${project.version}</version> <scope>compile</scope> @@ -68,6 +75,11 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>org.apache.felix</groupId> + <artifactId>org.apache.felix.framework</artifactId> + <scope>provided</scope> + </dependency> </dependencies> <build> diff --git a/vespa-testrunner-components/pom.xml b/vespa-testrunner-components/pom.xml index 4b4b0cfa92c..332a0eaa82d 100644 --- a/vespa-testrunner-components/pom.xml +++ b/vespa-testrunner-components/pom.xml @@ -11,7 +11,7 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> diff --git a/vespa_feed_perf/pom.xml b/vespa_feed_perf/pom.xml index 31fae41a357..a570cb69c36 100644 --- a/vespa_feed_perf/pom.xml +++ b/vespa_feed_perf/pom.xml @@ -12,11 +12,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa_feed_perf</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java b/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java index 850513fb990..67171de487d 100644 --- a/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java +++ b/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java @@ -2,10 +2,12 @@ package com.yahoo.vespa.feed.perf; import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentTypeManagerConfigurer; import com.yahoo.document.DocumentUpdate; import com.yahoo.document.TestAndSetCondition; import com.yahoo.document.json.JsonFeedReader; @@ -63,8 +65,8 @@ import java.util.stream.Stream; */ public class SimpleFeeder implements ReplyHandler { - private final DocumentTypeManager docTypeMgr = new DocumentTypeManager(); + private final ConfigSubscriber documentTypeConfigSubscriber; private final List<InputStream> inputStreams; private final PrintStream out; private final RPCMessageBus mbus; @@ -359,7 +361,7 @@ public class SimpleFeeder implements ReplyHandler { numMessagesToSend = params.getNumMessagesToSend(); mbus = newMessageBus(docTypeMgr, params); session = newSession(mbus, this, params); - docTypeMgr.configure(params.getConfigId()); + documentTypeConfigSubscriber = DocumentTypeManagerConfigurer.configure(docTypeMgr, params.getConfigId()); benchmarkMode = params.isBenchmarkMode(); destination = (params.getDumpStream() != null) ? createDumper(params) diff --git a/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/SimpleServer.java b/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/SimpleServer.java index 10184b35e4c..a458a59f997 100644 --- a/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/SimpleServer.java +++ b/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/SimpleServer.java @@ -31,8 +31,7 @@ public class SimpleServer { @SuppressWarnings("deprecation") public SimpleServer(String configDir, MessageHandler msgHandler) throws IOException, ListenFailedException { slobrok = new Slobrok(); - documentMgr = new DocumentTypeManager(); - documentMgr.configure("dir:" + configDir); + documentMgr = DocumentTypeManager.fromFile(configDir + "/documentmanager.cfg"); mbus = new MessageBus(new RPCNetwork(new RPCNetworkParams() .setSlobrokConfigId(slobrok.configId()) .setIdentity(new Identity("server"))), @@ -58,7 +57,6 @@ public class SimpleServer { public final void close() { session.destroy(); mbus.destroy(); - documentMgr.shutdown(); slobrok.stop(); } diff --git a/vespa_jersey2/pom.xml b/vespa_jersey2/pom.xml deleted file mode 100644 index 41a079e4ae3..00000000000 --- a/vespa_jersey2/pom.xml +++ /dev/null @@ -1,118 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 - http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> - </parent> - <!-- TODO Vespa 8: Remove this module, or rename to vespa-jackson --> - <artifactId>vespa_jersey2</artifactId> - <version>7-SNAPSHOT</version> - <packaging>pom</packaging> - <dependencies> - <dependency> - <groupId>javax.ws.rs</groupId> - <artifactId>javax.ws.rs-api</artifactId> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.core</groupId> - <artifactId>jersey-client</artifactId> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.datatype</groupId> - <artifactId>jackson-datatype-jdk8</artifactId> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.datatype</groupId> - <artifactId>jackson-datatype-jsr310</artifactId> - </dependency> - - <dependency> - <groupId>org.glassfish.jersey.core</groupId> - <artifactId>jersey-server</artifactId> - <exclusions> - <exclusion> - <groupId>org.glassfish.jersey.media</groupId> - <artifactId>jersey-media-jaxb</artifactId> - </exclusion> - </exclusions> - </dependency> - - <dependency> - <!-- Previously pulled in by jersey-container-servlet-core. - Contains packages imported by hosted user applications, July 2021. --> - <groupId>javax.validation</groupId> - <artifactId>validation-api</artifactId> - </dependency> - - <dependency> - <!-- TODO Vespa 8: remove! Previously pulled in by jersey-media-json-jackson. --> - <groupId>com.fasterxml.jackson.jaxrs</groupId> - <artifactId>jackson-jaxrs-json-provider</artifactId> - <exclusions> - <exclusion> - <!-- Conflicts with javax.activation:javax.activation-api:1.2.0, which is "exported" via jdisc_core. --> - <groupId>jakarta.activation</groupId> - <artifactId>jakarta.activation-api</artifactId> - </exclusion> - <exclusion> - <!-- Conflicts with javax.xml.bind:jaxb-api:2.3, which is "exported" via jdisc_core.--> - <groupId>jakarta.xml.bind</groupId> - <artifactId>jakarta.xml.bind-api</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <!-- TODO Vespa 8: remove! Previously pulled in by jersey-media-json-jackson. --> - <groupId>com.fasterxml.jackson.module</groupId> - <artifactId>jackson-module-jaxb-annotations</artifactId> - <exclusions> - <exclusion> - <!-- Conflicts with javax.activation:javax.activation-api:1.2.0, which is "exported" via jdisc_core. --> - <groupId>jakarta.activation</groupId> - <artifactId>jakarta.activation-api</artifactId> - </exclusion> - <exclusion> - <!-- Conflicts with javax.xml.bind:jaxb-api:2.3, which is "exported" via jdisc_core.--> - <groupId>jakarta.xml.bind</groupId> - <artifactId>jakarta.xml.bind-api</artifactId> - </exclusion> - </exclusions> - </dependency> - - </dependencies> - - <build> - <plugins> - <plugin> - <!-- Explicit for IntelliJ to detect correct language level from parent --> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-dependency-plugin</artifactId> - <executions> - <execution> - <id>copy-dependencies</id> - <phase>package</phase> - <goals> - <goal>copy-dependencies</goal> - </goals> - </execution> - </executions> - <configuration> - <overWriteReleases>false</overWriteReleases> - <overWriteSnapshots>false</overWriteSnapshots> - <excludeTransitive>false</excludeTransitive> - </configuration> - </plugin> - </plugins> - </build> -</project> diff --git a/vespabase/src/common-env.sh b/vespabase/src/common-env.sh index 7268a0a7edc..7b812d40fec 100755 --- a/vespabase/src/common-env.sh +++ b/vespabase/src/common-env.sh @@ -129,6 +129,35 @@ optionally_reduce_base_frequency() { fi } +get_hugepage_size_mb() { + while read -r name size rest + do + if [[ "$name" =~ ^Hugepagesize:$ ]] + then + hugepagesize="$size" + unit="${rest,,}" + break + fi + done < /proc/meminfo + if [[ "$unit" == "kb" ]]; then + hugepage_size_mb=$(($hugepagesize / 1024)) + else + echo "Failed extracting hugepage size from /proc/meminfo. Unknown unit($unit)" + exit 1 + fi + OUT=$hugepage_size_mb +} + +get_jvm_hugepage_settings() { + local heap_mb="$1" + get_hugepage_size_mb + sz_mb=$OUT + if (($sz_mb * 2 < $heap_mb)); then + options=" -XX:+UseTransparentHugePages" + fi + echo "$options" +} + populate_environment export LD_LIBRARY_PATH=$VESPA_HOME/lib64 diff --git a/vespabase/src/rhel-prestart.sh b/vespabase/src/rhel-prestart.sh index dbc77879efe..3bb9ba5bae0 100755 --- a/vespabase/src/rhel-prestart.sh +++ b/vespabase/src/rhel-prestart.sh @@ -103,7 +103,7 @@ fixdir ${VESPA_USER} ${VESPA_GROUP} 755 libexec/vespa/plugins/qrs fixdir ${VESPA_USER} ${VESPA_GROUP} 755 logs fixdir ${VESPA_USER} ${VESPA_GROUP} 755 logs/vespa fixdir ${VESPA_USER} ${VESPA_GROUP} 755 logs/vespa/configserver -fixdir ${VESPA_USER} ${VESPA_GROUP} 755 logs/vespa/qrs +fixdir ${VESPA_USER} ${VESPA_GROUP} 755 logs/vespa/access fixdir ${VESPA_USER} ${VESPA_GROUP} 755 logs/vespa/search fixdir ${VESPA_USER} ${VESPA_GROUP} 755 tmp fixdir ${VESPA_USER} ${VESPA_GROUP} 755 tmp/vespa diff --git a/vespaclient-container-plugin/pom.xml b/vespaclient-container-plugin/pom.xml index c960c2cca44..5c3a04beeb8 100644 --- a/vespaclient-container-plugin/pom.xml +++ b/vespaclient-container-plugin/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespaclient-container-plugin</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> @@ -38,42 +38,6 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespa-http-client</artifactId> - <version>${project.version}</version> - <exclusions> - <!-- Exclude artifacts that are provided by Jdisc container --> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>*</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.datatype</groupId> - <artifactId>*</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.module</groupId> - <artifactId>*</artifactId> - </exclusion> - <exclusion> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </exclusion> - <exclusion> - <groupId>com.yahoo.vespa</groupId> - <artifactId>security-utils</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>*</artifactId> - </exclusion> - <exclusion> - <groupId>org.bouncycastle</groupId> - <artifactId>*</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index d3f8bb40b04..626d4f57c58 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -22,6 +22,7 @@ import com.yahoo.document.TestAndSetCondition; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.document.fieldset.AllFields; import com.yahoo.document.fieldset.DocIdOnly; +import com.yahoo.document.fieldset.DocumentOnly; import com.yahoo.document.idstring.IdIdString; import com.yahoo.document.json.DocumentOperationType; import com.yahoo.document.json.JsonReader; @@ -377,8 +378,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { StorageCluster destination = resolveCluster(Optional.of(requireProperty(request, DESTINATION_CLUSTER)), clusters); VisitorParameters parameters = parseParameters(request, path); parameters.setRemoteDataHandler("[Content:cluster=" + destination.name() + "]"); // Bypass indexing. - // TODO Vespa 8: change to DocumentOnly.NAME - parameters.setFieldSet(AllFields.NAME); + parameters.setFieldSet(DocumentOnly.NAME); return () -> { visitWithRemote(request, parameters, handler); return true; // VisitorSession has its own throttle handling. @@ -713,12 +713,17 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { } } - synchronized void writeSingleDocument(Document document) throws IOException { - boolean tensorShortForm = false; - if (request != null && request.parameters().containsKey("format.tensors")) { - tensorShortForm = request.parameters().get("format.tensors").contains("short"); + private boolean tensorShortForm() { + if (request != null && + request.parameters().containsKey("format.tensors") && + request.parameters().get("format.tensors").contains("long")) { + return false; } - new JsonWriter(json, tensorShortForm).writeFields(document); + return true; // default + } + + synchronized void writeSingleDocument(Document document) throws IOException { + new JsonWriter(json, tensorShortForm()).writeFields(document); } synchronized void writeDocumentsArrayStart() throws IOException { @@ -737,7 +742,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { ByteArrayOutputStream myOut = new ByteArrayOutputStream(1); myOut.write(','); // Prepend rather than append, to avoid double memory copying. try (JsonGenerator myJson = jsonFactory.createGenerator(myOut)) { - new JsonWriter(myJson).write(document); + new JsonWriter(myJson, tensorShortForm()).write(document); } docs.add(myOut); @@ -1102,8 +1107,8 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { throw new IllegalArgumentException("Must set 'cluster' parameter to a valid content cluster id when visiting at a root /document/v1/ level"); VisitorParameters parameters = parseCommonParameters(request, path, cluster); - // TODO Vespa 8: change to DocumentOnly.NAME - parameters.setFieldSet(getProperty(request, FIELD_SET).orElse(path.documentType().map(type -> type + ":[document]").orElse(AllFields.NAME))); + // TODO can the else-case be safely reduced to always be DocumentOnly.NAME? + parameters.setFieldSet(getProperty(request, FIELD_SET).orElse(path.documentType().map(type -> type + ":[document]").orElse(DocumentOnly.NAME))); parameters.setMaxTotalHits(wantedDocumentCount); parameters.visitInconsistentBuckets(true); long timeoutMs = Math.max(1, request.getTimeout(MILLISECONDS) - handlerTimeout.toMillis()); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java index 784575cd001..8ea9234009d 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.http.server; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.ReferencedResource; @@ -14,9 +13,6 @@ import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.Result; import com.yahoo.messagebus.shared.SharedSourceSession; import com.yahoo.net.HostName; -import com.yahoo.vespa.http.client.core.ErrorCode; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.OperationStatus; import com.yahoo.vespaxmlparser.FeedOperation; import com.yahoo.yolean.Exceptions; @@ -240,7 +236,6 @@ class ClientFeederV3 { return message; } - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 private void setMessageParameters(DocumentOperationMessageV3 msg, FeederSettings settings) { msg.getMessage().setContext(new ReplyContext(msg.getOperationId(), feedReplies)); if (settings.traceLevel != null) { @@ -249,9 +244,6 @@ class ClientFeederV3 { if (settings.priority != null) { try { DocumentProtocol.Priority priority = DocumentProtocol.Priority.valueOf(settings.priority); - if (msg.getMessage() instanceof DocumentMessage) { - ((DocumentMessage) msg.getMessage()).setPriority(priority); // TODO: Remove on Vespa 8 - } } catch (IllegalArgumentException i) { log.severe(i.getMessage()); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java index 13a12f707d9..973c154b336 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientState.java @@ -4,14 +4,13 @@ package com.yahoo.vespa.http.server; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.ReferencedResource; import com.yahoo.messagebus.shared.SharedSourceSession; -import com.yahoo.vespa.http.client.core.OperationStatus; import java.util.concurrent.BlockingQueue; /** * The state of a client session, used to save replies when client disconnects. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class ClientState { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Encoder.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Encoder.java index e4781dc3a3f..65fb1223c7d 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Encoder.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Encoder.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; +package com.yahoo.vespa.http.server; /** * Simple encoding scheme to remove space, linefeed, control characters and @@ -10,7 +10,7 @@ package com.yahoo.vespa.http.client.core; * * @author Steinar Knutsen */ -public final class Encoder { +final class Encoder { /** * ISO 646.irv:1991 safe quoting into a StringBuilder instance. @@ -27,12 +27,12 @@ public final class Encoder { encode(c, output); } else { switch (c) { - case '{': - case '}': - encode(c, output); - break; - default: - output.append((char) c); + case '{': + case '}': + encode(c, output); + break; + default: + output.append((char) c); } } } else { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ErrorCode.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ErrorCode.java index 4e739218319..f819ecccbb1 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ErrorCode.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ErrorCode.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; +package com.yahoo.vespa.http.server; /** * Return types for the server. @@ -7,15 +7,15 @@ package com.yahoo.vespa.http.client.core; * @author Einar M R Rosenvinge * @author Steinar Knutsen */ -public enum ErrorCode { +enum ErrorCode { OK(true, true), ERROR(false, false), TRANSIENT_ERROR(false, true), END_OF_FEED(true, true); - private boolean success; - private boolean _transient; + private final boolean success; + private final boolean _transient; ErrorCode(boolean success, boolean _transient) { this.success = success; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java index f99274d3f2b..74665d60a04 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java @@ -15,7 +15,6 @@ import com.yahoo.jdisc.Response; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.messagebus.ReplyHandler; import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.vespa.http.client.core.Headers; import javax.inject.Inject; import java.io.IOException; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java index c8828df6d54..f9ae04623e6 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java @@ -13,7 +13,6 @@ import com.yahoo.jdisc.ReferencedResource; import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.shared.SharedSourceSession; -import com.yahoo.vespa.http.client.core.Headers; import com.yahoo.yolean.Exceptions; import java.util.HashMap; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedParams.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedParams.java new file mode 100644 index 00000000000..6ce20cdec53 --- /dev/null +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedParams.java @@ -0,0 +1,23 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import java.util.concurrent.TimeUnit; + +/** + * Feed level parameters. + * + * @author Einar M R Rosenvinge + */ +public final class FeedParams { + + /** + * Enumeration of data formats that are acceptable by the + * FeedClient methods. + */ + public enum DataFormat { + /** UTF-8-encoded XML. Preamble is not necessary. */ + XML_UTF8, + JSON_UTF8 + } + +} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java index 069ccfd84f0..f96b650748d 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReaderFactory.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.http.server; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.json.JsonFeedReader; import com.yahoo.text.Utf8; -import com.yahoo.vespa.http.client.config.FeedParams; import com.yahoo.vespaxmlparser.FeedReader; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java index 2fbb80d9fcc..1422ec10b08 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java @@ -11,17 +11,11 @@ import com.yahoo.jdisc.Metric; import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.Trace; -import com.yahoo.vespa.http.client.core.ErrorCode; -import com.yahoo.vespa.http.client.core.OperationStatus; import java.util.Map; -import java.util.Optional; -import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; -import static java.util.function.Predicate.not; - /** * Catch message bus replies and make the available to a given session. * diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedResponse.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedResponse.java index 3e2a4a8795f..1da8aded27b 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedResponse.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedResponse.java @@ -2,9 +2,6 @@ package com.yahoo.vespa.http.server; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.ErrorCode; -import com.yahoo.vespa.http.client.core.OperationStatus; import java.io.IOException; import java.io.OutputStream; @@ -15,8 +12,7 @@ import java.util.concurrent.BlockingQueue; * Reads feed responses from a queue and renders them continuously to the * feeder. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @since 5.1 + * @author Steinar Knutsen */ public class FeedResponse extends HttpResponse { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java index 37ccaf0d228..9bb8a58d6f6 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeederSettings.java @@ -3,8 +3,6 @@ package com.yahoo.vespa.http.server; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.messagebus.routing.Route; -import com.yahoo.vespa.http.client.config.FeedParams.DataFormat; -import com.yahoo.vespa.http.client.core.Headers; import java.util.Optional; @@ -18,14 +16,14 @@ public class FeederSettings { private static final Route DEFAULT_ROUTE = Route.parse("default"); public final boolean drain; // TODO: Implement drain=true public final Route route; - public final DataFormat dataFormat; + public final FeedParams.DataFormat dataFormat; public final String priority; public final Integer traceLevel; public FeederSettings(HttpRequest request) { this.drain = Optional.ofNullable(request.getHeader(Headers.DRAIN)).map(Boolean::parseBoolean).orElse(false); this.route = Optional.ofNullable(request.getHeader(Headers.ROUTE)).map(Route::parse).orElse(DEFAULT_ROUTE); - this.dataFormat = Optional.ofNullable(request.getHeader(Headers.DATA_FORMAT)).map(DataFormat::valueOf).orElse(DataFormat.XML_UTF8); + this.dataFormat = Optional.ofNullable(request.getHeader(Headers.DATA_FORMAT)).map(FeedParams.DataFormat::valueOf).orElse(FeedParams.DataFormat.JSON_UTF8); this.priority = request.getHeader(Headers.PRIORITY); this.traceLevel = Optional.ofNullable(request.getHeader(Headers.TRACE_LEVEL)).map(Integer::valueOf).orElse(null); } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Headers.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Headers.java index d41f42ef652..16bff38af4b 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Headers.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Headers.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; +package com.yahoo.vespa.http.server; /** - * Wrapper for shared constants used by both client and server. + * HTTP headers. * * @author Steinar Knutsen */ -public final class Headers { +final class Headers { private Headers() { } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/OperationStatus.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/OperationStatus.java index ee6d96aa600..e771128c2e5 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/OperationStatus.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/OperationStatus.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; +package com.yahoo.vespa.http.server; import com.google.common.base.Splitter; import java.util.Iterator; @@ -9,7 +9,7 @@ import java.util.Iterator; * * @author Steinar Knutsen */ -public final class OperationStatus { +final class OperationStatus { public static final String IS_CONDITION_NOT_MET = "IS-CONDITION-NOT-MET"; public final String message; @@ -60,9 +60,9 @@ public final class OperationStatus { String traceMessage = ""; operationId = Encoder.decode(input.next(), new StringBuilder()) - .toString(); + .toString(); errorCode = ErrorCode.valueOf(Encoder.decode(input.next(), - new StringBuilder()).toString()); + new StringBuilder()).toString()); message = Encoder.decode(input.next(), new StringBuilder()).toString(); // We are backwards compatible, meaning it is ok not to supply the last argument. diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java index aa2651595ef..6dc7f4ab516 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ReplyContext.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.http.server; -import com.yahoo.vespa.http.client.core.OperationStatus; - import java.util.concurrent.BlockingQueue; /** diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java index 4ddc430b35f..c2c6d00fa25 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.http.server; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.vespa.http.client.core.Encoder; import com.yahoo.vespa.http.server.util.ByteLimitedInputStream; import com.yahoo.vespaxmlparser.FeedOperation; import com.yahoo.vespaxmlparser.FeedReader; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/UnknownClientException.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/UnknownClientException.java index 5324b86a98a..513b9355f3e 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/UnknownClientException.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/UnknownClientException.java @@ -2,8 +2,7 @@ package com.yahoo.vespa.http.server; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.5.0 + * @author Einar M R Rosenvinge */ public class UnknownClientException extends RuntimeException { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java index 270ebe7796b..74489c774f0 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStream.java @@ -6,8 +6,6 @@ import java.io.InputStream; /** * @author Einar M R Rosenvinge - * - * @since 5.1.23 */ public class ByteLimitedInputStream extends InputStream { diff --git a/vespaclient-container-plugin/src/test/cfg/music.sd b/vespaclient-container-plugin/src/test/cfg/music.sd index 75a1918de5b..86597379dd3 100644 --- a/vespaclient-container-plugin/src/test/cfg/music.sd +++ b/vespaclient-container-plugin/src/test/cfg/music.sd @@ -2,5 +2,6 @@ search music { document music { field artist type string { } + field embedding type tensor(x[3]) {} } } diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java index 9f3c536a59a..74a86b6a7b7 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java @@ -14,6 +14,7 @@ import com.yahoo.document.FixedBucketSpaces; import com.yahoo.document.TestAndSetCondition; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.restapi.DocumentOperationExecutorConfig; import com.yahoo.document.restapi.resource.DocumentV1ApiHandler.StorageCluster; import com.yahoo.document.update.FieldUpdate; @@ -51,6 +52,7 @@ import com.yahoo.schema.derived.Deriver; import com.yahoo.slime.Inspector; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.SlimeUtils; +import com.yahoo.tensor.Tensor; import com.yahoo.test.ManualClock; import com.yahoo.vdslib.VisitorStatistics; import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; @@ -118,7 +120,9 @@ public class DocumentV1ApiTest { final Document doc3 = new Document(manager.getDocumentType("music"), "id:space:music:g=a:three"); { doc1.setFieldValue("artist", "Tom Waits"); + doc1.setFieldValue("embedding", new TensorFieldValue(Tensor.from("tensor(x[3]):[1,2,3]"))); doc2.setFieldValue("artist", "Asa-Chan & Jun-Ray"); + doc2.setFieldValue("embedding", new TensorFieldValue(Tensor.from("tensor(x[3]):[4,5,6]"))); } final Map<String, StorageCluster> clusters = Map.of("content", new StorageCluster("content", @@ -229,13 +233,15 @@ public class DocumentV1ApiTest { " {" + " \"id\": \"id:space:music::one\"," + " \"fields\": {" + - " \"artist\": \"Tom Waits\"" + + " \"artist\": \"Tom Waits\", " + + " \"embedding\": { \"type\": \"tensor(x[3])\", \"values\": [1.0,2.0,3.0] } " + " }" + " }," + " {" + " \"id\": \"id:space:music:n=1:two\"," + " \"fields\": {" + - " \"artist\": \"Asa-Chan & Jun-Ray\"" + + " \"artist\": \"Asa-Chan & Jun-Ray\", " + + " \"embedding\": { \"type\": \"tensor(x[3])\", \"values\": [4.0,5.0,6.0] } " + " }" + " }," + " {" + @@ -278,13 +284,15 @@ public class DocumentV1ApiTest { " {" + " \"id\": \"id:space:music::one\"," + " \"fields\": {" + - " \"artist\": \"Tom Waits\"" + + " \"artist\": \"Tom Waits\"," + + " \"embedding\": { \"type\": \"tensor(x[3])\", \"values\": [1.0,2.0,3.0] } " + " }" + " }," + " {" + " \"id\": \"id:space:music:n=1:two\"," + " \"fields\": {" + - " \"artist\": \"Asa-Chan & Jun-Ray\"" + + " \"artist\": \"Asa-Chan & Jun-Ray\"," + + " \"embedding\": { \"type\": \"tensor(x[3])\", \"values\": [4.0,5.0,6.0] } " + " }" + " }" + " ]," + @@ -337,7 +345,7 @@ public class DocumentV1ApiTest { // POST with namespace and document type is a restricted visit with a required destination cluster ("destinationCluster") access.expect(parameters -> { assertEquals("[Content:cluster=content]", parameters.getRemoteDataHandler()); - assertEquals("[all]", parameters.fieldSet()); + assertEquals("[document]", parameters.fieldSet()); assertEquals(60_000L, parameters.getSessionTimeoutMs()); parameters.getControlHandler().onDone(VisitorControlHandler.CompletionCode.SUCCESS, "We made it!"); }); @@ -491,12 +499,13 @@ public class DocumentV1ApiTest { parameters.responseHandler().get().handleResponse(new DocumentResponse(0, doc1)); return new Result(); }); - response = driver.sendRequest("http://localhost/document/v1/space/music/docid/one?"); + response = driver.sendRequest("http://localhost/document/v1/space/music/docid/one?format.tensors=long"); assertSameJson("{" + " \"pathId\": \"/document/v1/space/music/docid/one\"," + " \"id\": \"id:space:music::one\"," + " \"fields\": {" + - " \"artist\": \"Tom Waits\"" + + " \"artist\": \"Tom Waits\"," + + " \"embedding\": { \"cells\": [{\"address\":{\"x\":\"0\"},\"value\":1.0},{\"address\":{\"x\":\"1\"},\"value\": 2.0},{\"address\":{\"x\":\"2\"},\"value\": 3.0}]}" + " }" + "}", response.readAll()); assertEquals(200, response.getStatus()); @@ -532,7 +541,8 @@ public class DocumentV1ApiTest { response = driver.sendRequest("http://localhost/document/v1/space/music/number/1/two?condition=test%20it&tracelevel=9", POST, "{" + " \"fields\": {" + - " \"artist\": \"Asa-Chan & Jun-Ray\"" + + " \"artist\": \"Asa-Chan & Jun-Ray\"," + + " \"embedding\": { \"values\": [4.0,5.0,6.0] } " + " }" + "}"); assertSameJson("{" + diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java index a5a8f4cb5bd..dcabc1f338e 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerV3Test.java @@ -17,10 +17,6 @@ import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.shared.SharedSourceSession; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.text.Utf8; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.core.ErrorCode; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.OperationStatus; import org.junit.Test; import org.mockito.stubbing.Answer; diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedReaderFactoryTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedReaderFactoryTestCase.java index 6b0bd1c9518..08a7e82a158 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedReaderFactoryTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedReaderFactoryTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.http.server; import com.yahoo.document.DocumentTypeManager; import com.yahoo.text.Utf8; -import com.yahoo.vespa.http.client.config.FeedParams; import org.junit.Test; import java.io.ByteArrayInputStream; diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java index 513892af213..a009e70fc30 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java @@ -2,7 +2,7 @@ package com.yahoo.vespaxmlparser; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.vespa.http.client.config.FeedParams; +import com.yahoo.vespa.http.server.FeedParams; import com.yahoo.vespa.http.server.FeedReaderFactory; import java.io.InputStream; diff --git a/vespaclient-core/pom.xml b/vespaclient-core/pom.xml index 8f2865a22cf..5afc2b57868 100644 --- a/vespaclient-core/pom.xml +++ b/vespaclient-core/pom.xml @@ -6,11 +6,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespaclient-core</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <dependency> <groupId>junit</groupId> diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessageBusSessionFactory.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessageBusSessionFactory.java index d9b1190aaed..13d6f5d3323 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessageBusSessionFactory.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessageBusSessionFactory.java @@ -19,12 +19,11 @@ public class MessageBusSessionFactory implements SessionFactory { this(processor, null, null); } - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 private MessageBusSessionFactory(MessagePropertyProcessor processor, DocumentmanagerConfig documentmanagerConfig, SlobroksConfig slobroksConfig) { this.processor = processor; - MessageBusParams params = new MessageBusParams(processor.getLoadTypes()); + MessageBusParams params = new MessageBusParams(); params.setTraceLevel(processor.getFeederOptions().getTraceLevel()); RPCNetworkParams rpcNetworkParams = processor.getFeederOptions().getNetworkParams(); if (slobroksConfig != null) // not set: will subscribe diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java index 84fbe63a576..ac1ed248ee9 100644 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java @@ -4,13 +4,10 @@ package com.yahoo.feedapi; import com.yahoo.concurrent.SystemTimer; import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.documentapi.messagebus.loadtypes.LoadType; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.routing.Route; -import com.yahoo.vespa.config.content.LoadTypeConfig; import com.yahoo.vespaclient.config.FeederConfig; import java.util.logging.Level; @@ -20,7 +17,6 @@ import java.util.logging.Logger; * Utility class for assigning properties to messages, either from implicit * config values or from explicit values in requests. */ -@SuppressWarnings("removal") // TODO Vespa 8: remove public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscriber<FeederConfig> { private static final Logger log = Logger.getLogger(MessagePropertyProcessor.class.getName()); @@ -33,23 +29,12 @@ public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscrib private String defaultDocprocChain = null; private boolean defaultAbortOnDocumentError = true; private boolean defaultAbortOnSendError = true; - private final LoadTypeSet loadTypes; // TODO remove on Vespa 8 private boolean configChanged = false; public MessagePropertyProcessor(FeederConfig config) { - loadTypes = new LoadTypeSet(); configure(config); } - /** - * @deprecated load types are deprecated. Use constructor without LoadTypeConfig instead. - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - public MessagePropertyProcessor(FeederConfig config, LoadTypeConfig loadTypeCfg) { - loadTypes = new LoadTypeSet(); - configure(config, loadTypeCfg); - } - public void setRoute(String routeOverride) { defaultRoute = Route.parse(routeOverride); } @@ -60,7 +45,6 @@ public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscrib String priorityParam = null; String abortOnDocErrorParam = null; String abortOnFeedErrorParam = null; - String loadTypeStr = null; String traceStr = null; String createIfNonExistentParam = null; Double totalTimeoutParam = null; @@ -81,7 +65,6 @@ public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscrib traceStr = request.getProperty("tracelevel"); abortOnDocErrorParam = request.getProperty("abortondocumenterror"); abortOnFeedErrorParam = request.getProperty("abortonfeederror"); - loadTypeStr = request.getProperty("loadtype"); createIfNonExistentParam = request.getProperty("createifnonexistent"); } @@ -109,16 +92,7 @@ public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscrib priority = DocumentProtocol.getPriorityByName(priorityParam); } - LoadType loadType = null; - if (loadTypes != null && loadTypeStr != null) { - loadType = loadTypes.getNameMap().get(loadTypeStr); - } - - if (loadType == null) { - loadType = LoadType.DEFAULT; - } - - return new PropertySetter(route, timeout, totalTimeout, priority, loadType, retry, abortOnDocumentError, abortOnFeedError, createIfNonExistent, traceStr != null ? Integer.parseInt(traceStr) : 0); + return new PropertySetter(route, timeout, totalTimeout, priority, retry, abortOnDocumentError, abortOnFeedError, createIfNonExistent, traceStr != null ? Integer.parseInt(traceStr) : 0); } public long getDefaultTimeoutMillis() { return defaultTimeoutMillis; } @@ -135,23 +109,6 @@ public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscrib return feederOptions; } - /** - * @deprecated load types are deprecated. configure without LoadTypeConfig instead. - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - public synchronized void configure(FeederConfig config, LoadTypeConfig loadTypeConfig) { - loadTypes.configure(loadTypeConfig); - configure(config); - } - - /** - * @deprecated load types are deprecated - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - LoadTypeSet getLoadTypes() { - return loadTypes; - } - public synchronized void configure(FeederConfig config) { if (feederOptions != null) { setConfigChanged(true); @@ -191,17 +148,15 @@ public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscrib private boolean abortOnDocumentError; private boolean abortOnFeedError; private boolean createIfNonExistent; - private LoadType loadType; // TODO remove on Vespa 8 private int traceLevel; - PropertySetter(Route route, long timeout, long totalTimeout, DocumentProtocol.Priority priority, LoadType loadType, + PropertySetter(Route route, long timeout, long totalTimeout, DocumentProtocol.Priority priority, boolean retryEnabled, boolean abortOnDocumentError, boolean abortOnFeedError, boolean createIfNonExistent, int traceLevel) { this.route = route; this.timeout = timeout; this.totalTimeout = totalTimeout; this.priority = priority; - this.loadType = loadType; this.retryEnabled = retryEnabled; this.abortOnDocumentError = abortOnDocumentError; this.abortOnFeedError = abortOnFeedError; @@ -245,6 +200,7 @@ public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscrib } @Override + @SuppressWarnings("removal") // TODO Vespa 9: remove public void process(Message msg) { if (route != null) { msg.setRoute(route); @@ -253,11 +209,6 @@ public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscrib msg.setRetryEnabled(retryEnabled); msg.getTrace().setLevel(Math.max(getFeederOptions().getTraceLevel(), traceLevel)); - if (loadType != null) { - ((DocumentMessage) msg).setLoadType(loadType); - ((DocumentMessage) msg).setPriority(loadType.getPriority()); - } - if (priority != null) { ((DocumentMessage) msg).setPriority(priority); } diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java index afe09e06a8f..06f66c83e1b 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java @@ -65,11 +65,10 @@ public final class VespaFeedHandler extends VespaFeedHandlerBase { } private Feeder createFeeder(SimpleFeedAccess sender, HttpRequest request) { - if (Boolean.valueOf(request.getProperty(JSON_INPUT))) { - return new JsonFeeder(getDocumentTypeManager(), sender, getRequestInputStream(request)); - } else { + if ( ! Boolean.valueOf(request.getProperty(JSON_INPUT))) { return new XMLFeeder(getDocumentTypeManager(), sender, getRequestInputStream(request)); } + return new JsonFeeder(getDocumentTypeManager(), sender, getRequestInputStream(request)); } } diff --git a/vespaclient-java/pom.xml b/vespaclient-java/pom.xml index 3a0c52fc462..d0e04d1e692 100644 --- a/vespaclient-java/pom.xml +++ b/vespaclient-java/pom.xml @@ -6,11 +6,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespaclient-java</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <dependency> <groupId>junit</groupId> @@ -33,6 +33,12 @@ <version>${project.version}</version> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-core</artifactId> + <scope>compile</scope> + <version>${project.version}</version> + </dependency> + <dependency> <!-- TODO: remove dependency on container-dev, and instead depend directly on what this module needs! --> <groupId>com.yahoo.vespa</groupId> <artifactId>container-dev</artifactId> diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java index 52f2857c7e5..2ae3ecf113b 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/Arguments.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespafeeder; -import com.yahoo.vespa.config.content.LoadTypeConfig; import com.yahoo.feedapi.DummySessionFactory; import com.yahoo.feedapi.MessageBusSessionFactory; import com.yahoo.feedapi.MessagePropertyProcessor; diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java index d3431818b63..6102f3da77f 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java @@ -17,7 +17,7 @@ public class ClientParameters { public final Iterator<String> documentIds; // Print only the id for retrieved documents public final boolean printIdsOnly; - // Determines which document fields to retrieve. Default is '[all]'. + // Determines which document fields to retrieve. Default is '[document]'. public final String fieldSet; // The Vespa route public final String route; @@ -35,8 +35,6 @@ public class ClientParameters { public final int traceLevel; // Document request priority public final DocumentProtocol.Priority priority; - // Determines the Vespa load type - public final String loadTypeName; // If full documents are printed, they will be printed as JSON (instead of XML) public final boolean jsonOutput; @@ -45,7 +43,7 @@ public class ClientParameters { boolean help, Iterator<String> documentIds, boolean printIdsOnly, String fieldSet, String route, String cluster, String configId, boolean showDocSize, double timeout, boolean noRetry, int traceLevel, - DocumentProtocol.Priority priority, String loadTypeName, boolean jsonOutput) { + DocumentProtocol.Priority priority, boolean jsonOutput) { this.help = help; this.documentIds = documentIds; @@ -59,7 +57,6 @@ public class ClientParameters { this.noRetry = noRetry; this.traceLevel = traceLevel; this.priority = priority; - this.loadTypeName = loadTypeName; this.jsonOutput = jsonOutput; } @@ -76,7 +73,6 @@ public class ClientParameters { private boolean noRetry; private int traceLevel; private DocumentProtocol.Priority priority; - private String loadTypeName; private boolean jsonOutput; public Builder setHelp(boolean help) { @@ -139,11 +135,6 @@ public class ClientParameters { return this; } - public Builder setLoadTypeName(String loadTypeName) { - this.loadTypeName = loadTypeName; - return this; - } - public Builder setJsonOutput(boolean jsonOutput) { this.jsonOutput = jsonOutput; return this; @@ -152,7 +143,7 @@ public class ClientParameters { public ClientParameters build() { return new ClientParameters( help, documentIds, printIdsOnly, fieldSet, route, cluster, configId, - showDocSize, timeout, noRetry, traceLevel, priority, loadTypeName, jsonOutput); + showDocSize, timeout, noRetry, traceLevel, priority, jsonOutput); } } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java index da9f48f44b1..b1bc9c76328 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java @@ -37,7 +37,6 @@ public class CommandLineOptions { public static final String NORETRY_OPTION = "noretry"; public static final String TRACE_OPTION = "trace"; public static final String PRIORITY_OPTION = "priority"; - public static final String LOADTYPE_OPTION = "loadtype"; public static final String JSONOUTPUT_OPTION = "jsonoutput"; public static final String XMLOUTPUT_OPTION = "xmloutput"; @@ -68,10 +67,9 @@ public class CommandLineOptions { .longOpt(PRINTIDS_OPTION) .build()); - // TODO Vespa 8: change to DocumentOnly.NAME options.addOption(Option.builder("f") .hasArg(true) - .desc("Retrieve the specified fields only (see https://docs.vespa.ai/en/documents.html#fieldsets) (default '" + AllFields.NAME + "')") + .desc("Retrieve the specified fields only (see https://docs.vespa.ai/en/documents.html#fieldsets) (default '" + DocumentOnly.NAME + "')") .longOpt(FIELDSET_OPTION) .argName("fieldset").build()); @@ -123,12 +121,6 @@ public class CommandLineOptions { .longOpt(PRIORITY_OPTION) .argName("priority").build()); - options.addOption(Option.builder("l") - .hasArg(true) - .desc("Load type (default \"\").") - .longOpt(LOADTYPE_OPTION) - .argName("loadtype").build()); - options.addOption(Option.builder("j") .hasArg(false) .desc("JSON output (default format)") @@ -163,7 +155,6 @@ public class CommandLineOptions { String route = cl.getOptionValue(ROUTE_OPTION, ""); String configId = cl.getOptionValue(CONFIGID_OPTION, ""); boolean help = cl.hasOption(HELP_OPTION); - String loadtype = cl.getOptionValue(LOADTYPE_OPTION, ""); boolean noRetry = cl.hasOption(NORETRY_OPTION); boolean showDocSize = cl.hasOption(SHOWDOCSIZE_OPTION); boolean jsonOutput = cl.hasOption(JSONOUTPUT_OPTION); @@ -184,8 +175,7 @@ public class CommandLineOptions { if (printIdsOnly) { fieldSet = DocIdOnly.NAME; } else if (fieldSet.isEmpty()) { - // TODO Vespa 8: change to DocumentOnly.NAME - fieldSet = AllFields.NAME; + fieldSet = DocumentOnly.NAME; } if (!cluster.isEmpty() && !route.isEmpty()) { @@ -211,7 +201,6 @@ public class CommandLineOptions { .setFieldSet(fieldSet) .setHelp(help) .setPrintIdsOnly(printIdsOnly) - .setLoadTypeName(loadtype) .setNoRetry(noRetry) .setCluster(cluster) .setRoute(route) diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java index 2454f5c8627..91e3b5205c1 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java @@ -8,8 +8,6 @@ import com.yahoo.documentapi.SyncParameters; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; import com.yahoo.documentapi.messagebus.MessageBusSyncSession; -import com.yahoo.documentapi.messagebus.loadtypes.LoadType; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; import com.yahoo.messagebus.Message; @@ -20,20 +18,18 @@ import com.yahoo.vespaclient.ClusterDef; import com.yahoo.vespaclient.ClusterList; import java.util.Iterator; -import java.util.Map; /** * The document retriever is responsible for retrieving documents using the Document API and printing the result to standard out. * * @author bjorncs */ -@SuppressWarnings("removal") // TODO: Remove on Vespa 8 +@SuppressWarnings("removal") // TODO: Remove on Vespa 9 public class DocumentRetriever { private final ClusterList clusterList; private final DocumentAccessFactory documentAccessFactory; private final ClientParameters params; - private final LoadTypeSet loadTypeSet; // TODO remove on Vespa 8 private MessageBusSyncSession session; private MessageBusDocumentAccess documentAccess; @@ -43,21 +39,6 @@ public class DocumentRetriever { ClientParameters params) { this.clusterList = clusterList; this.documentAccessFactory = documentAccessFactory; - this.loadTypeSet = new LoadTypeSet(); // TODO remove on Vespa 8 - this.params = params; - } - - /** - * @deprecated load types are deprecated. Use constructor without LoadTypeSet instead. - */ - @Deprecated(forRemoval = true) // TODO: Remove on Vespa 8 - public DocumentRetriever(ClusterList clusterList, - DocumentAccessFactory documentAccessFactory, - LoadTypeSet loadTypeSet, - ClientParameters params) { - this.clusterList = clusterList; - this.documentAccessFactory = documentAccessFactory; - this.loadTypeSet = loadTypeSet; this.params = params; } @@ -81,7 +62,6 @@ public class DocumentRetriever { public void retrieveDocuments() throws DocumentRetrieverException { boolean first = true; String route = params.cluster.isEmpty() ? params.route : resolveClusterRoute(params.cluster); - LoadType loadType = params.loadTypeName.isEmpty() ? null : resolveLoadType(params.loadTypeName); MessageBusParams messageBusParams = createMessageBusParams(params.configId, params.timeout, route); documentAccess = documentAccessFactory.createDocumentAccess(messageBusParams); @@ -104,7 +84,7 @@ public class DocumentRetriever { } } String docid = iter.next(); - Message msg = createDocumentRequest(docid, loadType); + Message msg = createDocumentRequest(docid); Reply reply = session.syncSend(msg); printReply(reply); } @@ -133,17 +113,8 @@ public class DocumentRetriever { return clusterDef.getRoute(); } - private LoadType resolveLoadType(String loadTypeName) throws DocumentRetrieverException { - Map<String, LoadType> loadTypesNameMap = loadTypeSet.getNameMap(); - if (!loadTypesNameMap.containsKey(loadTypeName)) { - throw new DocumentRetrieverException(String.format("Loadtype with name '%s' does not exist.\n", loadTypeName)); - } else { - return loadTypesNameMap.get(loadTypeName); - } - } - private MessageBusParams createMessageBusParams(String configId, double timeout, String route) { - MessageBusParams messageBusParams = new MessageBusParams(loadTypeSet); + MessageBusParams messageBusParams = new MessageBusParams(); messageBusParams.setRoute(route); messageBusParams.setProtocolConfigId(configId); messageBusParams.setRoutingConfigId(configId); @@ -155,15 +126,10 @@ public class DocumentRetriever { return messageBusParams; } - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 - private Message createDocumentRequest(String docid, LoadType loadType) { + private Message createDocumentRequest(String docid) { GetDocumentMessage msg = new GetDocumentMessage(new DocumentId(docid), params.fieldSet); - msg.setPriority(params.priority); // TODO: Remove on Vespa 8 + msg.setPriority(params.priority); // TODO: Remove on Vespa 9 msg.setRetryEnabled(!params.noRetry); - - if (loadType != null) { - msg.setLoadType(loadType); - } return msg; } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java index 7596246d16e..f5851041ceb 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/Main.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespaget; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.vespaclient.ClusterList; import java.util.logging.Level; @@ -38,12 +37,10 @@ public class Main { Runtime.getRuntime().addShutdownHook(new Thread(documentRetriever::shutdown)); } - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 private static DocumentRetriever createDocumentRetriever(ClientParameters params) { return new DocumentRetriever( new ClusterList("client"), new DocumentAccessFactory(), - new LoadTypeSet(params.configId), // TODO: Remove on Vespa 8 params ); } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java index 0e64f824b63..42c23c4a961 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java @@ -3,6 +3,7 @@ package com.yahoo.vespavisit; import com.yahoo.document.FixedBucketSpaces; import com.yahoo.document.fieldset.DocIdOnly; +import com.yahoo.document.fieldset.DocumentOnly; import com.yahoo.document.select.parser.ParseException; import com.yahoo.documentapi.ProgressToken; import com.yahoo.documentapi.VisitorControlHandler; @@ -193,7 +194,7 @@ public class VdsVisit { .longOpt("fieldset") .hasArg(true) .argName("fieldset") - .desc("Retrieve the specified fields only (see https://docs.vespa.ai/en/documents.html#fieldsets). Default is [all].") + .desc("Retrieve the specified fields only (see https://docs.vespa.ai/en/documents.html#fieldsets). Default is [document].") .build()); options.addOption(Option.builder() @@ -467,6 +468,8 @@ public class VdsVisit { } if (line.hasOption("l")) { params.fieldSet(line.getOptionValue("l")); + } else { + params.fieldSet(DocumentOnly.NAME); } if (line.hasOption("visitinconsistentbuckets")) { params.visitInconsistentBuckets(true); diff --git a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java index 7f529fb5b86..d6e55a26a39 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/VespaFeederTestCase.java @@ -56,7 +56,7 @@ public class VespaFeederTestCase { assertTrue(arguments.isVerbose()); assertFalse(config.createifnonexistent()); - assertEquals("LOW_1", arguments.getPriority()); + assertEquals("LOW_1", arguments.getPriority()); // TODO remove on Vespa 9 assertEquals("benchmark", arguments.getMode()); assertEquals("foo.xml", arguments.getFiles().get(0)); assertEquals("bar.xml", arguments.getFiles().get(1)); @@ -152,14 +152,14 @@ public class VespaFeederTestCase { } @Test - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 + @SuppressWarnings("removal") // TODO: Remove on Vespa 9 public void feedFile() throws Exception { FeedFixture f = new FeedFixture(); Arguments arguments = new Arguments("--file src/test/files/myfeed.xml --priority LOW_1".split(" "), f.sessionFactory); new VespaFeeder(arguments, f.typeManager).parseFiles(System.in, f.printStream); assertEquals(3, f.sessionFactory.messages.size()); - assertEquals(DocumentProtocol.Priority.LOW_1, ((PutDocumentMessage)f.sessionFactory.messages.get(0)).getPriority()); // TODO: Remove on Vespa 8 + assertEquals(DocumentProtocol.Priority.LOW_1, ((PutDocumentMessage)f.sessionFactory.messages.get(0)).getPriority()); // TODO: Remove on Vespa 9 assertEquals("id:test:news::foo", ((PutDocumentMessage) f.sessionFactory.messages.get(0)).getDocumentPut().getDocument().getId().toString()); DocumentUpdate update = ((UpdateDocumentMessage) f.sessionFactory.messages.get(1)).getDocumentUpdate(); assertEquals("id:test:news::foo", update.getId().toString()); @@ -176,10 +176,10 @@ public class VespaFeederTestCase { assertJsonFeedState(feedFixture); } - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 + @SuppressWarnings("removal") // TODO: Remove on Vespa 9 protected void assertJsonFeedState(FeedFixture feedFixture) { assertEquals(3, feedFixture.sessionFactory.messages.size()); - assertEquals(DocumentProtocol.Priority.LOW_1, ((PutDocumentMessage)feedFixture.sessionFactory.messages.get(0)).getPriority()); // TODO: Remove on Vespa 8 + assertEquals(DocumentProtocol.Priority.LOW_1, ((PutDocumentMessage)feedFixture.sessionFactory.messages.get(0)).getPriority()); // TODO: Remove on Vespa 9 assertEquals("id:test:news::foo", ((PutDocumentMessage) feedFixture.sessionFactory.messages.get(0)).getDocumentPut().getDocument().getId().toString()); DocumentUpdate update = ((UpdateDocumentMessage) feedFixture.sessionFactory.messages.get(1)).getDocumentUpdate(); assertEquals("id:test:news::foo", update.getId().toString()); diff --git a/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java b/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java index b634e899b74..0b828c91f91 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java @@ -56,8 +56,7 @@ public class CommandLineOptionsTest { assertFalse(params.help); assertFalse(params.documentIds.hasNext()); assertFalse(params.printIdsOnly); - // TODO Vespa 8: change to DocumentOnly.NAME - assertEquals(AllFields.NAME, params.fieldSet); + assertEquals(DocumentOnly.NAME, params.fieldSet); assertEquals("default-get", params.route); assertTrue(params.cluster.isEmpty()); assertEquals("client", params.configId); @@ -66,7 +65,6 @@ public class CommandLineOptionsTest { assertFalse(params.noRetry); assertEquals(0, params.traceLevel); assertEquals(DocumentProtocol.Priority.NORMAL_2, params.priority); - assertTrue(params.loadTypeName.isEmpty()); } @Test @@ -80,7 +78,6 @@ public class CommandLineOptionsTest { "--noretry", "--trace", "1", "--priority", Integer.toString(DocumentProtocol.Priority.HIGH_3.getValue()), - "--loadtype", "dummyloadtype", "id:1", "id:2" ); @@ -92,7 +89,6 @@ public class CommandLineOptionsTest { assertTrue(params.noRetry); assertEquals(1, params.traceLevel); assertEquals(DocumentProtocol.Priority.HIGH_3, params.priority); - assertEquals("dummyloadtype", params.loadTypeName); Iterator<String> documentsIds = params.documentIds; assertEquals("id:1", documentsIds.next()); diff --git a/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java b/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java index 30d117ab105..ce555e2c0f5 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespaget/DocumentRetrieverTest.java @@ -5,11 +5,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.document.DataType; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; -import com.yahoo.document.fieldset.AllFields; +import com.yahoo.document.fieldset.DocumentOnly; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusSyncSession; -import com.yahoo.documentapi.messagebus.loadtypes.LoadType; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; @@ -97,11 +95,10 @@ public class DocumentRetrieverTest { .setCluster("") .setRoute("default") .setConfigId("client") - .setFieldSet(AllFields.NAME) + .setFieldSet(DocumentOnly.NAME) .setPrintIdsOnly(false) .setHelp(false) .setShowDocSize(false) - .setLoadTypeName("") .setNoRetry(false) .setTraceLevel(0) .setTimeout(0) @@ -133,30 +130,25 @@ public class DocumentRetrieverTest { } @Test - @SuppressWarnings("removal") // TODO: Remove on Vespa 8 + @SuppressWarnings("removal") // TODO: Remove on Vespa 9 public void testSendSingleMessage() throws DocumentRetrieverException { ClientParameters params = createParameters() .setDocumentIds(asIterator(DOC_ID_1)) .setPriority(DocumentProtocol.Priority.HIGH_1) .setNoRetry(true) - .setLoadTypeName("loadtype") .build(); when(mockedSession.syncSend(any())).thenReturn(createDocumentReply(DOC_ID_1)); - LoadTypeSet loadTypeSet = new LoadTypeSet(); // TODO remove on Vespa 8 - loadTypeSet.addLoadType(1, "loadtype", DocumentProtocol.Priority.HIGH_1); DocumentRetriever documentRetriever = new DocumentRetriever( new ClusterList(), mockedFactory, - loadTypeSet, params); documentRetriever.retrieveDocuments(); verify(mockedSession, times(1)).syncSend(argThat((ArgumentMatcher<GetDocumentMessage>) o -> - o.getPriority().equals(DocumentProtocol.Priority.HIGH_1) && - !o.getRetryEnabled() && - o.getLoadType().equals(new LoadType(1, "loadtype", DocumentProtocol.Priority.HIGH_1)))); // TODO: Remove on Vespa 8 + o.getPriority().equals(DocumentProtocol.Priority.HIGH_1) && // TODO remove on Vespa 9 + !o.getRetryEnabled())); assertContainsDocument(DOC_ID_1); } @@ -221,19 +213,6 @@ public class DocumentRetrieverTest { } @Test - public void testInvalidLoadType() throws DocumentRetrieverException { - exception.expect(DocumentRetrieverException.class); - exception.expectMessage("Loadtype with name 'undefinedloadtype' does not exist.\n"); - - ClientParameters params = createParameters() - .setLoadTypeName("undefinedloadtype") - .build(); - - DocumentRetriever documentRetriever = createDocumentRetriever(params); - documentRetriever.retrieveDocuments(); - } - - @Test public void testClusterLookup() throws DocumentRetrieverException { final String cluster = "storage", expectedRoute = "[Content:cluster=storage]"; diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 654042372cf..81a21af780d 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -1242,7 +1242,6 @@ "public abstract java.lang.String toString()", "public abstract java.lang.String toString(boolean, boolean)", "public abstract java.lang.String toAbbreviatedString()", - "public java.lang.String toShortString()", "public static java.lang.String toStandardString(com.yahoo.tensor.Tensor, boolean, boolean, long)", "public static java.lang.String valueToString(com.yahoo.tensor.Tensor, boolean, long)", "public abstract boolean equals(java.lang.Object)", diff --git a/vespajlib/pom.xml b/vespajlib/pom.xml index ed6ae3678f4..c4d100aa10c 100644 --- a/vespajlib/pom.xml +++ b/vespajlib/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespajlib</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <description> Library for use in Java components of Vespa. Shared code which do not fit anywhere else. diff --git a/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java b/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java index f035e2c6f00..0d06a18b096 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java +++ b/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java @@ -70,12 +70,6 @@ public abstract class AbstractFilteringList<Type, ListType extends AbstractFilte return matching(new HashSet<>(others.asList())::contains); } - /** @deprecated use and(others) */ - @Deprecated // TODO: Remove on Vespa 8 - public ListType concat(ListType others) { - return and(others); - } - /** Returns the union of the two lists. */ public ListType and(ListType others) { return constructor.apply(Stream.concat(items.stream(), others.asList().stream()).collect(toUnmodifiableList()), false); diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/CompletableFutures.java b/vespajlib/src/main/java/com/yahoo/concurrent/CompletableFutures.java index a1235c3821d..2e14b532b35 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/CompletableFutures.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/CompletableFutures.java @@ -1,13 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.concurrent; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.SettableFuture; -import com.yahoo.yolean.UncheckedInterruptedException; - import java.util.ArrayList; import java.util.List; -import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -87,48 +82,4 @@ public class CompletableFutures { return results; }); } - - /** - * Helper for migrating from {@link ListenableFuture} to {@link CompletableFuture} in Vespa public apis - * @deprecated to be removed in Vespa 8 - */ - @SuppressWarnings("unchecked") - @Deprecated(forRemoval = true, since = "7") - public static <V> ListenableFuture<V> toGuavaListenableFuture(CompletableFuture<V> future) { - if (future instanceof ListenableFuture) { - return ((ListenableFuture<V>) future); - } - SettableFuture<V> guavaFuture = SettableFuture.create(); - future.whenComplete((result, error) -> { - if (result != null) guavaFuture.set(result); - else if (error instanceof CancellationException) guavaFuture.setException(error); - else guavaFuture.cancel(true); - }); - return guavaFuture; - } - - /** - * Helper for migrating from {@link ListenableFuture} to {@link CompletableFuture} in Vespa public apis - * @deprecated to be removed in Vespa 8 - */ - @Deprecated(forRemoval = true, since = "7") - public static <V> CompletableFuture<V> toCompletableFuture(ListenableFuture<V> guavaFuture) { - CompletableFuture<V> future = new CompletableFuture<>(); - guavaFuture.addListener( - () -> { - if (guavaFuture.isCancelled()) future.cancel(true); - try { - V value = guavaFuture.get(); - future.complete(value); - } catch (InterruptedException e) { - // Should not happens since listener is invoked after future is complete - throw new UncheckedInterruptedException(e); - } catch (ExecutionException e) { - future.completeExceptionally(e.getCause()); - } - }, - Runnable::run); - return future; - } - } diff --git a/vespajlib/src/main/java/com/yahoo/slime/JsonFormat.java b/vespajlib/src/main/java/com/yahoo/slime/JsonFormat.java index 5abbd136690..f30c3e66aab 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/JsonFormat.java +++ b/vespajlib/src/main/java/com/yahoo/slime/JsonFormat.java @@ -49,13 +49,6 @@ public final class JsonFormat implements SlimeFormat { new Encoder(value, os, indent).encode(); } - @Override - @Deprecated(since = "7", forRemoval = true) - @SuppressWarnings("removal") - public void decode(InputStream is, Slime slime) { - throw new UnsupportedOperationException("Not implemented"); - } - /** Returns the given slime data as UTF-8-encoded JSON */ public static byte[] toJsonBytes(Slime slime) { try { diff --git a/vespajlib/src/main/java/com/yahoo/slime/SlimeFormat.java b/vespajlib/src/main/java/com/yahoo/slime/SlimeFormat.java index 86003f519bc..92512db3da7 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/SlimeFormat.java +++ b/vespajlib/src/main/java/com/yahoo/slime/SlimeFormat.java @@ -2,7 +2,6 @@ package com.yahoo.slime; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; /** @@ -18,14 +17,4 @@ public interface SlimeFormat { */ void encode(OutputStream os, Slime slime) throws IOException; - /** - * Decode an input stream into the provided slime object - * - * @param is The input stream to read from. - * @param slime The slime object to decode into. - * @deprecated use e.g. {@link JsonDecoder} instead - */ - @Deprecated(since = "7", forRemoval = true) - void decode(InputStream is, Slime slime) throws IOException; - } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index 8a84e97fe05..349214ee7f9 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -327,11 +327,6 @@ public interface Tensor { /** Returns an abbreviated string representation of this tensor suitable for human-readable messages */ String toAbbreviatedString(); - // TODO: Remove on Vespa 8 - /** @deprecated use toAbbreviatedString */ - @Deprecated - default String toShortString() { return toAbbreviatedString(); } - /** * Call this from toString in implementations to return this tensor on the * <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form">tensor literal form</a>. diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java index d7758f309db..b683519988c 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java @@ -109,11 +109,15 @@ public class JsonFormat { private static void encodeValues(IndexedTensor tensor, Cursor cursor, long[] indexes, int dimension) { DimensionSizes sizes = tensor.dimensionSizes(); - for (indexes[dimension] = 0; indexes[dimension] < sizes.size(dimension); ++indexes[dimension]) { - if (dimension < (sizes.dimensions() - 1)) { - encodeValues(tensor, cursor.addArray(), indexes, dimension + 1); - } else { - cursor.addDouble(tensor.get(indexes)); + if (indexes.length == 0) { + cursor.addDouble(tensor.get(0)); + } else { + for (indexes[dimension] = 0; indexes[dimension] < sizes.size(dimension); ++indexes[dimension]) { + if (dimension < (sizes.dimensions() - 1)) { + encodeValues(tensor, cursor.addArray(), indexes, dimension + 1); + } else { + cursor.addDouble(tensor.get(indexes)); + } } } } diff --git a/vespajlib/src/main/java/com/yahoo/vespa/VersionTagger.java b/vespajlib/src/main/java/com/yahoo/vespa/VersionTagger.java index 7d2c780bcdb..925cdffe5a5 100644 --- a/vespajlib/src/main/java/com/yahoo/vespa/VersionTagger.java +++ b/vespajlib/src/main/java/com/yahoo/vespa/VersionTagger.java @@ -57,12 +57,12 @@ public class VersionTagger { // Use default values map.put("V_TAG", "NOTAG"); map.put("V_TAG_DATE", "NOTAG"); - map.put("V_TAG_PKG", "6.9999.0"); + map.put("V_TAG_PKG", "8.9999.0"); map.put("V_TAG_ARCH", "NOTAG"); map.put("V_TAG_SYSTEM", "NOTAG"); map.put("V_TAG_SYSTEM_REV", "NOTAG"); map.put("V_TAG_BUILDER", "NOTAG"); - map.put("V_TAG_COMPONENT", "6.9999.0"); + map.put("V_TAG_COMPONENT", "8.9999.0"); map.put("V_TAG_COMMIT_SHA", "badc0ffe"); map.put("V_TAG_COMMIT_DATE", "0"); } catch (IOException e) { diff --git a/vespajlib/src/main/java/net/jpountz/lz4/package-info.java b/vespajlib/src/main/java/net/jpountz/lz4/package-info.java deleted file mode 100644 index 478f666dab4..00000000000 --- a/vespajlib/src/main/java/net/jpountz/lz4/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 1, minor = 7, micro = 1)) -package net.jpountz.lz4; -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; diff --git a/vespajlib/src/main/java/net/jpountz/util/package-info.java b/vespajlib/src/main/java/net/jpountz/util/package-info.java deleted file mode 100644 index 66089785785..00000000000 --- a/vespajlib/src/main/java/net/jpountz/util/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 1, minor = 7, micro = 1)) -package net.jpountz.util; -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; diff --git a/vespajlib/src/main/java/net/jpountz/xxhash/package-info.java b/vespajlib/src/main/java/net/jpountz/xxhash/package-info.java deleted file mode 100644 index e9ceccb9a34..00000000000 --- a/vespajlib/src/main/java/net/jpountz/xxhash/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 1, minor = 7, micro = 1)) -package net.jpountz.xxhash; -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; diff --git a/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java index 9465f081990..1d8c39a7778 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/slime/JsonFormatTestCase.java @@ -226,12 +226,6 @@ public class JsonFormatTestCase { assertEquals(str, a.asString()); } - @Test(expected = UnsupportedOperationException.class) - @SuppressWarnings("removal") - public void testThatDecodeIsNotImplemented() { - new JsonFormat(true).decode(null, null); - } - private void verifyEncoding(Slime slime, String expected) { verifyEncoding(slime, expected, true); } diff --git a/vespalog/abi-spec.json b/vespalog/abi-spec.json index 996cc0259a0..b0aa26db686 100644 --- a/vespalog/abi-spec.json +++ b/vespalog/abi-spec.json @@ -1,20 +1,4 @@ { - "com.yahoo.log.FileLogTarget": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.log.LogTarget" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.io.File)", - "public synchronized java.io.OutputStream open()", - "public synchronized void close()", - "public java.lang.String toString()" - ], - "fields": [] - }, "com.yahoo.log.InvalidLogFormatException": { "superClass": "java.lang.Exception", "interfaces": [], @@ -28,36 +12,6 @@ ], "fields": [] }, - "com.yahoo.log.LevelController": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract boolean shouldLog(java.util.logging.Level)", - "public abstract java.lang.String getOnOffString()", - "public abstract void checkBack()", - "public abstract java.util.logging.Level getLevelLimit()" - ], - "fields": [] - }, - "com.yahoo.log.LevelControllerRepo": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract com.yahoo.log.LevelController getLevelController(java.lang.String)", - "public abstract void close()" - ], - "fields": [] - }, "com.yahoo.log.LogFileDb": { "superClass": "java.lang.Object", "interfaces": [], @@ -121,12 +75,9 @@ "methods": [ "public static com.yahoo.log.LogMessage of(java.time.Instant, java.lang.String, long, long, java.lang.String, java.lang.String, java.util.logging.Level, java.lang.String)", "public java.time.Instant getTimestamp()", - "public long getTime()", - "public long getTimeInSeconds()", "public java.lang.String getHost()", "public long getProcessId()", "public java.util.OptionalLong getThreadId()", - "public java.lang.String getThreadProcess()", "public java.lang.String getService()", "public java.lang.String getComponent()", "public java.util.logging.Level getLevel()", @@ -164,380 +115,10 @@ ], "methods": [ "public void <init>()", - "public static java.util.Timer getTaskRunner()", "public static void clearHandlers()", "public static void initVespaLogging(java.lang.String)", - "public static com.yahoo.log.VespaLogHandler getLogHandler()", "public static void cleanup()" ], "fields": [] - }, - "com.yahoo.log.LogTarget": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract java.io.OutputStream open()", - "public abstract void close()" - ], - "fields": [] - }, - "com.yahoo.log.RejectFilter": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public boolean shouldReject(java.lang.String)", - "public void addRejectedMessage(java.lang.String)", - "public static com.yahoo.log.RejectFilter createDefaultRejectFilter()" - ], - "fields": [] - }, - "com.yahoo.log.StderrLogTarget": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.log.LogTarget" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public java.io.OutputStream open()", - "public void close()" - ], - "fields": [] - }, - "com.yahoo.log.StdoutLogTarget": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.log.LogTarget" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public java.io.OutputStream open()", - "public void close()" - ], - "fields": [] - }, - "com.yahoo.log.Util": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public static java.lang.String getHostName()", - "public static java.lang.String getPID()" - ], - "fields": [] - }, - "com.yahoo.log.VespaFormat": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public static java.lang.String escape(java.lang.String)", - "public static void formatTime(long, java.lang.StringBuilder)", - "public static java.lang.String formatTime(java.time.Instant)", - "public static java.lang.String format(java.lang.String, java.lang.String, java.lang.String, long, java.lang.String, java.lang.String, java.lang.String, java.lang.Throwable)", - "public static void formatException(java.lang.Throwable, java.lang.StringBuilder)" - ], - "fields": [] - }, - "com.yahoo.log.VespaFormatter": { - "superClass": "java.util.logging.SimpleFormatter", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(java.lang.String, java.lang.String)", - "public static java.lang.String unEscape(java.lang.String)", - "public java.lang.String format(java.util.logging.LogRecord)", - "public void setServiceName(java.lang.String)", - "public java.lang.String getServiceName()", - "public static java.lang.String toMessageString(java.lang.Throwable)" - ], - "fields": [ - "public static final java.lang.String serviceNameUnsetValue" - ] - }, - "com.yahoo.log.VespaLevelControllerRepo": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.log.LevelControllerRepo" - ], - "attributes": [ - "public" - ], - "methods": [ - "public com.yahoo.log.LevelController getLevelController(java.lang.String)", - "public void close()" - ], - "fields": [] - }, - "com.yahoo.log.event.Collection": { - "superClass": "com.yahoo.log.event.Event", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(long, java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.log.event.Count": { - "superClass": "com.yahoo.log.event.Event", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(java.lang.String, double)", - "public com.yahoo.log.event.Event setValue(java.lang.String, java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.log.event.CountGroup": { - "superClass": "com.yahoo.log.event.Event", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(java.lang.String, java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.log.event.Crash": { - "superClass": "com.yahoo.log.event.Event", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(java.lang.String, int, int)" - ], - "fields": [] - }, - "com.yahoo.log.event.Event": { - "superClass": "java.lang.Object", - "interfaces": [ - "java.io.Serializable" - ], - "attributes": [ - "public", - "abstract" - ], - "methods": [ - "protected void <init>()", - "public com.yahoo.log.event.Event setValue(java.lang.String, java.lang.String)", - "public java.lang.String getValue(java.lang.String)", - "public void setTime(long)", - "public long getTime()", - "public com.yahoo.log.event.Event setVersion(int)", - "public int getVersion()", - "public java.lang.String getValuePossiblyQuote(java.lang.String)", - "public java.lang.String getName()", - "public java.lang.String toString()", - "public static com.yahoo.log.event.Event parse(java.lang.String)", - "public static final void starting(java.lang.String)", - "public static final void started(java.lang.String)", - "public static final void stopping(java.lang.String, java.lang.String)", - "public static final void stopped(java.lang.String, int, int)", - "public static final void reloading(java.lang.String)", - "public static final void reloaded(java.lang.String)", - "public static final void count(java.lang.String, long)", - "public static final void value(java.lang.String, double)", - "public static final void histogram(java.lang.String, java.lang.String, java.lang.String)", - "public static final void valueGroup(java.lang.String, java.lang.String)", - "public static final void countGroup(java.lang.String, java.lang.String)", - "public static final void progress(java.lang.String, long, long)", - "public static final void state(java.lang.String, java.lang.String)", - "public static final void crash(java.lang.String, int, int)" - ], - "fields": [] - }, - "com.yahoo.log.event.Histogram": { - "superClass": "com.yahoo.log.event.Event", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(java.lang.String, java.lang.String, java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.log.event.MalformedEventException": { - "superClass": "java.lang.Exception", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.Throwable)", - "public void <init>(java.lang.String)", - "public void <init>()" - ], - "fields": [] - }, - "com.yahoo.log.event.Progress": { - "superClass": "com.yahoo.log.event.Event", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(java.lang.String, java.lang.String, java.lang.String)", - "public void <init>(java.lang.String, double, double)", - "public void <init>(java.lang.String, java.lang.String)", - "public void <init>(java.lang.String, double)" - ], - "fields": [] - }, - "com.yahoo.log.event.Reloaded": { - "superClass": "com.yahoo.log.event.Event", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.log.event.Reloading": { - "superClass": "com.yahoo.log.event.Event", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.log.event.Started": { - "superClass": "com.yahoo.log.event.Event", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.log.event.Starting": { - "superClass": "com.yahoo.log.event.Event", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.log.event.State": { - "superClass": "com.yahoo.log.event.Event", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(java.lang.String, java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.log.event.Stopped": { - "superClass": "com.yahoo.log.event.Event", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(java.lang.String, int, int)" - ], - "fields": [] - }, - "com.yahoo.log.event.Stopping": { - "superClass": "com.yahoo.log.event.Event", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(java.lang.String, java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.log.event.Unknown": { - "superClass": "com.yahoo.log.event.Event", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public com.yahoo.log.event.Unknown setName(java.lang.String)", - "public java.lang.String getName()" - ], - "fields": [] - }, - "com.yahoo.log.event.Value": { - "superClass": "com.yahoo.log.event.Event", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(java.lang.String, double)" - ], - "fields": [] - }, - "com.yahoo.log.event.ValueGroup": { - "superClass": "com.yahoo.log.event.Event", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(java.lang.String, java.lang.String)" - ], - "fields": [] } }
\ No newline at end of file diff --git a/vespalog/pom.xml b/vespalog/pom.xml index 1fa007d6029..c18a319adac 100644 --- a/vespalog/pom.xml +++ b/vespalog/pom.xml @@ -5,11 +5,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespalog</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <name>Vespa Log</name> <packaging>container-plugin</packaging> @@ -53,14 +53,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <!-- TODO Vespa 8: remove configuration. - Included to allow 'removal' warnings for classes in its own module --> - <configuration> - <compilerArgs> - <arg>-Xlint:all,-serial,-try,-processing,-removal</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> diff --git a/vespalog/src/main/java/com/yahoo/log/DefaultLevelController.java b/vespalog/src/main/java/com/yahoo/log/DefaultLevelController.java index b90c63b185e..5f6251d291b 100644 --- a/vespalog/src/main/java/com/yahoo/log/DefaultLevelController.java +++ b/vespalog/src/main/java/com/yahoo/log/DefaultLevelController.java @@ -7,10 +7,9 @@ import java.util.logging.Level; /** * a levelcontroller that just implements a simple default * (possibly controlled by a system property or environment) - * @deprecated Should only be used internally in the log library + * Should only be used internally in the log library */ -@SuppressWarnings("removal") -@Deprecated(since = "7", forRemoval = true) +@SuppressWarnings("deprecation") class DefaultLevelController implements LevelController { private String levelstring; private Level levelLimit = LogLevel.EVENT; diff --git a/vespalog/src/main/java/com/yahoo/log/FileLogTarget.java b/vespalog/src/main/java/com/yahoo/log/FileLogTarget.java index b4752c23b73..1fe00fbd6f5 100644 --- a/vespalog/src/main/java/com/yahoo/log/FileLogTarget.java +++ b/vespalog/src/main/java/com/yahoo/log/FileLogTarget.java @@ -6,11 +6,9 @@ import java.io.*; /** * @author Ulf Lilleengen * @since 5.1 - * @deprecated Should only be used internally in the log library + * Should only be used internally in the log library */ -@SuppressWarnings("removal") -@Deprecated(since = "7", forRemoval = true) -public class FileLogTarget implements LogTarget { +class FileLogTarget implements LogTarget { private final File file; private FileOutputStream fileOutputStream; diff --git a/vespalog/src/main/java/com/yahoo/log/LevelController.java b/vespalog/src/main/java/com/yahoo/log/LevelController.java index c710d5c60eb..cd65c3b5337 100644 --- a/vespalog/src/main/java/com/yahoo/log/LevelController.java +++ b/vespalog/src/main/java/com/yahoo/log/LevelController.java @@ -10,10 +10,9 @@ import java.util.logging.Level; * * @author arnej27959 * - * @deprecated Should only be used internally in the log library + * Should only be used internally in the log library */ -@Deprecated(since = "7", forRemoval = true) -public interface LevelController { +interface LevelController { /** * should we actually publish a log message with the given Level now? diff --git a/vespalog/src/main/java/com/yahoo/log/LevelControllerRepo.java b/vespalog/src/main/java/com/yahoo/log/LevelControllerRepo.java index ab70ecf2c96..b48cd2ae7fd 100644 --- a/vespalog/src/main/java/com/yahoo/log/LevelControllerRepo.java +++ b/vespalog/src/main/java/com/yahoo/log/LevelControllerRepo.java @@ -7,11 +7,9 @@ package com.yahoo.log; * * @author Ulf Lilleengen * @since 5.1 - * @deprecated Should only be used internally in the log library + * Should only be used internally in the log library */ -@SuppressWarnings("removal") -@Deprecated(since = "7", forRemoval = true) -public interface LevelControllerRepo { +interface LevelControllerRepo { /** * Return the level controller for a given component. * @param component The component name string. diff --git a/vespalog/src/main/java/com/yahoo/log/LogMessage.java b/vespalog/src/main/java/com/yahoo/log/LogMessage.java index 878e041231c..b09cb10cd4a 100644 --- a/vespalog/src/main/java/com/yahoo/log/LogMessage.java +++ b/vespalog/src/main/java/com/yahoo/log/LogMessage.java @@ -70,24 +70,9 @@ public class LogMessage } public Instant getTimestamp() {return time;} - /** - * @deprecated Use {@link #getTimestamp()} - */ - @Deprecated(since = "7", forRemoval = true) - public long getTime () {return time.toEpochMilli();} - /** - * @deprecated Use {@link #getTimestamp()} - */ - @Deprecated(since = "7", forRemoval = true) - public long getTimeInSeconds () {return time.getEpochSecond();} public String getHost () {return host;} public long getProcessId() {return processId;} public OptionalLong getThreadId() {return threadId > 0 ? OptionalLong.of(threadId) : OptionalLong.empty();} - /** - * @deprecated Use {@link #getProcessId()} / {@link #getThreadId()} - */ - @Deprecated(since = "7", forRemoval = true) - public String getThreadProcess () {return VespaFormat.formatThreadProcess(processId, threadId);} public String getService () {return service;} public String getComponent () {return component;} public Level getLevel () {return level;} diff --git a/vespalog/src/main/java/com/yahoo/log/LogSetup.java b/vespalog/src/main/java/com/yahoo/log/LogSetup.java index d285382849e..e000684db7f 100644 --- a/vespalog/src/main/java/com/yahoo/log/LogSetup.java +++ b/vespalog/src/main/java/com/yahoo/log/LogSetup.java @@ -20,17 +20,11 @@ import java.util.logging.Logger; * @author Bjorn Borud * @author arnej27959 */ -@SuppressWarnings("removal") public class LogSetup { private static final Timer taskRunner = new Timer(true); - /** - * A global task thread - * @deprecated Just construct a java.util.Timer instead - **/ - @Deprecated(since = "7", forRemoval = true) - public static Timer getTaskRunner() { return taskRunner; } + static Timer getTaskRunner() { return taskRunner; } /** The log handler used by this */ private static VespaLogHandler logHandler; @@ -168,12 +162,7 @@ public class LogSetup { Logger.getLogger("").addHandler(logHandler); } - /** - * Returns the log handler set up by this class - * @deprecated Should only be used internally in the log library - */ - @Deprecated(since = "7", forRemoval = true) - public static VespaLogHandler getLogHandler() { + static VespaLogHandler getLogHandler() { return logHandler; } diff --git a/vespalog/src/main/java/com/yahoo/log/LogTarget.java b/vespalog/src/main/java/com/yahoo/log/LogTarget.java index ef5974259c8..45ac126e9ac 100644 --- a/vespalog/src/main/java/com/yahoo/log/LogTarget.java +++ b/vespalog/src/main/java/com/yahoo/log/LogTarget.java @@ -6,10 +6,9 @@ import java.io.OutputStream; /** * @author Ulf Lilleengen * @since 5.1 - * @deprecated Should only be used internally in the log library + * Should only be used internally in the log library */ -@Deprecated(since = "7", forRemoval = true) -public interface LogTarget { +interface LogTarget { /** * Opens an output stream for the target. If already open, the stream should be reopened. * @return a new outputstream for the log target. diff --git a/vespalog/src/main/java/com/yahoo/log/LogUtil.java b/vespalog/src/main/java/com/yahoo/log/LogUtil.java index 2424aa9060f..694cb3cf45c 100644 --- a/vespalog/src/main/java/com/yahoo/log/LogUtil.java +++ b/vespalog/src/main/java/com/yahoo/log/LogUtil.java @@ -4,9 +4,8 @@ package com.yahoo.log; /** * @author Ulf Lilleengen * @since 5.1 - * @deprecated Should only be used internally in the log library + * Should only be used internally in the log library */ -@Deprecated(since = "7", forRemoval = true) class LogUtil { static boolean empty(String s) { return (s == null || s.equals("")); diff --git a/vespalog/src/main/java/com/yahoo/log/MappedLevelController.java b/vespalog/src/main/java/com/yahoo/log/MappedLevelController.java index b7e18c83997..49ba78da56b 100644 --- a/vespalog/src/main/java/com/yahoo/log/MappedLevelController.java +++ b/vespalog/src/main/java/com/yahoo/log/MappedLevelController.java @@ -9,10 +9,9 @@ import java.util.logging.Level; /** * a level controller that does lookup in a file via a memory-mapped * buffer for realtime logging control. - * @deprecated Should only be used internally in the log library + * Should only be used internally in the log library */ -@SuppressWarnings("removal") -@Deprecated(since = "7", forRemoval = true) +@SuppressWarnings("deprecation") class MappedLevelController implements LevelController { private static final int ONVAL = 0x20204f4e; // equals " ON" in file private static final int OFFVAL = 0x204f4646; // equals " OFF" in file diff --git a/vespalog/src/main/java/com/yahoo/log/MappedLevelControllerRepo.java b/vespalog/src/main/java/com/yahoo/log/MappedLevelControllerRepo.java index 97c8aa8f70f..10b4d11c695 100644 --- a/vespalog/src/main/java/com/yahoo/log/MappedLevelControllerRepo.java +++ b/vespalog/src/main/java/com/yahoo/log/MappedLevelControllerRepo.java @@ -12,10 +12,8 @@ import java.util.Map; * * @author Ulf Lilleengen * @since 5.1 - * @deprecated Should only be used internally in the log library + * Should only be used internally in the log library */ -@SuppressWarnings("removal") -@Deprecated(since = "7", forRemoval = true) class MappedLevelControllerRepo { private final Map<String, LevelController> levelControllerMap = new HashMap<>(); private final MappedByteBuffer mapBuf; diff --git a/vespalog/src/main/java/com/yahoo/log/RejectFilter.java b/vespalog/src/main/java/com/yahoo/log/RejectFilter.java index b328073b20b..79010bbe7cb 100644 --- a/vespalog/src/main/java/com/yahoo/log/RejectFilter.java +++ b/vespalog/src/main/java/com/yahoo/log/RejectFilter.java @@ -9,10 +9,9 @@ import java.util.List; * * @author Ulf Lilleengen * @since 5.1 - * @deprecated Should only be used internally in the log library + * Should only be used internally in the log library */ -@Deprecated(since = "7", forRemoval = true) -public class RejectFilter { +class RejectFilter { private final List<String> rejectedMessages = new ArrayList<>(); public boolean shouldReject(String message) { diff --git a/vespalog/src/main/java/com/yahoo/log/StderrLogTarget.java b/vespalog/src/main/java/com/yahoo/log/StderrLogTarget.java index b1c245642ff..f4a22d972db 100644 --- a/vespalog/src/main/java/com/yahoo/log/StderrLogTarget.java +++ b/vespalog/src/main/java/com/yahoo/log/StderrLogTarget.java @@ -6,11 +6,9 @@ import java.io.OutputStream; /** * @author Ulf Lilleengen * @since 5.1 - * @deprecated Should only be used internally in the log library + * Should only be used internally in the log library */ -@SuppressWarnings("removal") -@Deprecated(since = "7", forRemoval = true) -public class StderrLogTarget implements LogTarget { +class StderrLogTarget implements LogTarget { @Override public OutputStream open() { diff --git a/vespalog/src/main/java/com/yahoo/log/StdoutLogTarget.java b/vespalog/src/main/java/com/yahoo/log/StdoutLogTarget.java index e036e534e00..61e4713ce46 100644 --- a/vespalog/src/main/java/com/yahoo/log/StdoutLogTarget.java +++ b/vespalog/src/main/java/com/yahoo/log/StdoutLogTarget.java @@ -6,11 +6,9 @@ import java.io.OutputStream; /** * @author Ulf Lilleengen * @since 5.1 - * @deprecated Should only be used internally in the log library + * Should only be used internally in the log library */ -@SuppressWarnings("removal") -@Deprecated(since = "7", forRemoval = true) -public class StdoutLogTarget implements LogTarget { +class StdoutLogTarget implements LogTarget { @Override public OutputStream open() { diff --git a/vespalog/src/main/java/com/yahoo/log/UncloseableOutputStream.java b/vespalog/src/main/java/com/yahoo/log/UncloseableOutputStream.java index 61e278ef93a..6172bf8d632 100644 --- a/vespalog/src/main/java/com/yahoo/log/UncloseableOutputStream.java +++ b/vespalog/src/main/java/com/yahoo/log/UncloseableOutputStream.java @@ -7,9 +7,8 @@ import java.io.OutputStream; /** * @author Simon Thoresen Hult * @since 5.1.14 - * @deprecated Should only be used internally in the log library + * Should only be used internally in the log library */ -@Deprecated(since = "7", forRemoval = true) class UncloseableOutputStream extends OutputStream { private final OutputStream out; diff --git a/vespalog/src/main/java/com/yahoo/log/Util.java b/vespalog/src/main/java/com/yahoo/log/Util.java index 7a7ae5221c4..4b6665a2f41 100644 --- a/vespalog/src/main/java/com/yahoo/log/Util.java +++ b/vespalog/src/main/java/com/yahoo/log/Util.java @@ -9,10 +9,9 @@ import static com.yahoo.vespa.defaults.Defaults.getDefaults; * @author arnej27959 * @author bjorncs * - * @deprecated Should only be used internally in the log library + * Should only be used internally in the log library */ -@Deprecated(since = "7", forRemoval = true) -public class Util { +class Util { public static String getHostName () { return getDefaults().vespaHostname(); diff --git a/vespalog/src/main/java/com/yahoo/log/VespaFormat.java b/vespalog/src/main/java/com/yahoo/log/VespaFormat.java index 0ce3668223d..646459161f5 100644 --- a/vespalog/src/main/java/com/yahoo/log/VespaFormat.java +++ b/vespalog/src/main/java/com/yahoo/log/VespaFormat.java @@ -14,11 +14,9 @@ import java.util.regex.Pattern; * @author arnej27959 * @author bjorncs * - * @deprecated Should only be used internally in the log library + * Should only be used internally in the log library */ -@SuppressWarnings("removal") -@Deprecated(since = "7", forRemoval = true) -public class VespaFormat { +class VespaFormat { private static final Pattern special = Pattern.compile("[\r\\\n\\\t\\\\]+"); private static final Pattern newLine = Pattern.compile("\n"); @@ -91,108 +89,10 @@ public class VespaFormat { } - /** - * @deprecated Use {@link #formatTime(Instant)} ()} - */ - @Deprecated(since = "7", forRemoval = true) - public static void formatTime (long time, StringBuilder sbuffer) { - sbuffer.append(formatTime(Instant.ofEpochMilli(time))); - } - public static String formatTime(Instant instant) { return String.format("%d.%06d", instant.getEpochSecond(), instant.getNano() / 1000); } - @Deprecated(since = "7", forRemoval = true) // Unused - this is not the format used by the Vespa log handler - public static String format(String levelName, - String component, - String componentPrefix, - long millis, - String threadId, - String serviceName, - String formattedMessage, - Throwable t) - { - StringBuilder sbuf = new StringBuilder(300); // initial guess - - // format the time - formatTime(millis, sbuf); - sbuf.append("\t"); - - sbuf.append(hostname).append("\t"); - - sbuf.append(processID); - if (threadId != null) { - sbuf.append("/").append(threadId); - } - sbuf.append("\t"); - - sbuf.append(serviceName).append("\t"); - - if (component == null && componentPrefix == null) { - sbuf.append("-"); - } else if (component == null) { - sbuf.append(componentPrefix); - } else if (componentPrefix == null) { - sbuf.append(".").append(component); - } else { - sbuf.append(componentPrefix).append(".").append(component); - } - sbuf.append("\t"); - - sbuf.append(levelName).append("\t"); - - sbuf.append(escape(formattedMessage)); - if (t != null) { - formatException(t, sbuf); - } - sbuf.append("\n"); - return sbuf.toString(); - } - - /** - * Format throwable into given StringBuffer. - * - * @param t The Throwable we want to format - * @param sbuf The stringbuffer into which we wish to - * format the Throwable - */ - @Deprecated(since = "7", forRemoval = true) // Unused - this is not the format used by the Vespa log handler - public static void formatException (Throwable t, StringBuilder sbuf) { - Throwable last = t; - int depth = 0; - while (last != null) { - sbuf.append("\\nmsg=\""); - sbuf.append(escape(last.getMessage())); - sbuf.append("\"\\nname=\""); - sbuf.append(escape(last.getClass().getName())); - sbuf.append("\"\\nstack=\"\\n"); - - // loop through stack frames and format them - StackTraceElement[] st = last.getStackTrace(); - int stopAt = Math.min(st.length, 15); - boolean first = true; - for (int i = 0; i < stopAt; i++) { - if (first) { - first = false; - } else { - sbuf.append("\\n"); - } - sbuf.append(escape(st[i].toString())); - } - - // tell the reader if we chopped off part of the stacktrace - if (stopAt < st.length) { - sbuf.append("\\n[...]"); - } - sbuf.append("\\n\""); - - last = last.getCause(); - depth++; - } - sbuf.append(" nesting=").append(depth); - } - static String formatThreadProcess(long processId, long threadId) { if (threadId == 0) { return Long.toString(processId); diff --git a/vespalog/src/main/java/com/yahoo/log/VespaFormatter.java b/vespalog/src/main/java/com/yahoo/log/VespaFormatter.java index caa68cf3019..140e83f5a51 100644 --- a/vespalog/src/main/java/com/yahoo/log/VespaFormatter.java +++ b/vespalog/src/main/java/com/yahoo/log/VespaFormatter.java @@ -19,10 +19,8 @@ import java.util.regex.Pattern; * @author Bjorn Borud * @author arnej27959 * - * @deprecated Should only be used internally in the log library */ -@Deprecated(since = "7", forRemoval = true) -public class VespaFormatter extends SimpleFormatter { +class VespaFormatter extends SimpleFormatter { private static final Pattern backSlash = Pattern.compile("\\\\"); @@ -92,6 +90,7 @@ public class VespaFormatter extends SimpleFormatter { return s; } + @SuppressWarnings("deprecation") public String format(LogRecord r) { StringBuilder sbuf = new StringBuilder(300); // initial guess diff --git a/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java b/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java index 0163a68bf4c..6f6d29d3a8f 100644 --- a/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java +++ b/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java @@ -17,11 +17,9 @@ import static java.nio.charset.StandardCharsets.US_ASCII; /** * @author Ulf Lilleengen * @since 5.1 - * @deprecated Should only be used internally in the log library + * Should only be used internally in the log library */ -@SuppressWarnings("removal") -@Deprecated(since = "7", forRemoval = true) -public class VespaLevelControllerRepo implements LevelControllerRepo { +class VespaLevelControllerRepo implements LevelControllerRepo { private RandomAccessFile ctlFile; private FileOutputStream ctlFileAppender; diff --git a/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java b/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java index d41ded6a9b4..95cc1e40876 100644 --- a/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java +++ b/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java @@ -10,8 +10,7 @@ import java.util.logging.StreamHandler; * @author Bjorn Borud * @author arnej27959 */ -@SuppressWarnings("removal") -@Deprecated(since = "7", forRemoval = true) +@SuppressWarnings("deprecation") class VespaLogHandler extends StreamHandler { private final LogTarget logTarget; diff --git a/vespalog/src/main/java/com/yahoo/log/event/Collection.java b/vespalog/src/main/java/com/yahoo/log/event/Collection.java deleted file mode 100644 index 331fdf33d9f..00000000000 --- a/vespalog/src/main/java/com/yahoo/log/event/Collection.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.log.event; - -/** - * - * @author Bjorn Borud - */ -@Deprecated(forRemoval = true, since = "7") -public class Collection extends Event { - public Collection () { - } - - public Collection (long collectionId, String name) { - setValue("collectionId", Long.toString(collectionId)); - setValue("name", name); - } -} diff --git a/vespalog/src/main/java/com/yahoo/log/event/Count.java b/vespalog/src/main/java/com/yahoo/log/event/Count.java index 284d23e6999..bc411628776 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Count.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Count.java @@ -5,8 +5,7 @@ package com.yahoo.log.event; * * @author Bjorn Borud */ -@Deprecated(forRemoval = true, since = "7") -public class Count extends Event { +class Count extends Event { public Count () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/CountGroup.java b/vespalog/src/main/java/com/yahoo/log/event/CountGroup.java deleted file mode 100755 index fc52c577bda..00000000000 --- a/vespalog/src/main/java/com/yahoo/log/event/CountGroup.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.log.event; - -@Deprecated(forRemoval = true, since = "7") -public class CountGroup extends Event { - public CountGroup () { - } - - public CountGroup (String name, String values) { - init(name, values); - } - - private void init (String name, String counts) { - setValue("name", name); - setValue("values", counts); - } -} diff --git a/vespalog/src/main/java/com/yahoo/log/event/Crash.java b/vespalog/src/main/java/com/yahoo/log/event/Crash.java index bcfaed4cce7..93efc64b061 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Crash.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Crash.java @@ -5,8 +5,7 @@ package com.yahoo.log.event; * * @author Bjorn Borud */ -@Deprecated(forRemoval = true, since = "7") -public class Crash extends Event { +class Crash extends Event { public Crash () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/Event.java b/vespalog/src/main/java/com/yahoo/log/event/Event.java index 288fd1f56dc..925b90dbf1d 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Event.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Event.java @@ -29,9 +29,8 @@ import java.util.regex.Pattern; * * @author Bjorn Borud * - * @deprecated Only for internal Vespa usage + * Only for internal Vespa usage */ -@Deprecated(forRemoval = true, since = "7") public abstract class Event implements Serializable { private static Logger log = Logger.getLogger(Event.class.getName()); @@ -426,20 +425,6 @@ public abstract class Event implements Serializable { } /** - * Static method for logging the <b>reloading</b> event. - */ - public static final void reloading (String name) { - log(getCallerLogger(), new Reloading(name)); - } - - /** - * Static method for logging the <b>reloaded</b> event. - */ - public static final void reloaded (String name) { - log(getCallerLogger(), new Reloaded(name)); - } - - /** * Static method for logging the <b>count</b> event. */ public static final void count (String name, long value) { @@ -454,29 +439,6 @@ public abstract class Event implements Serializable { } /** - * Static method for logging the <b>histogram</b> event. - */ - public static final void histogram (String name, String value, - String representation) { - log(getCallerLogger(), new Histogram(name, value, - representation)); - } - - /** - * Static method for logging a set of <b>value</b> events. - */ - public static final void valueGroup (String name, String value) { - log(getCallerLogger(), new ValueGroup(name, value)); - } - - /** - * Static method for logging a set of <b>count</b> events. - */ - public static final void countGroup (String name, String value) { - log(getCallerLogger(), new CountGroup(name, value)); - } - - /** * Static method for logging the <b>progress</b> event. */ public static final void progress (String name, long value, long total) { diff --git a/vespalog/src/main/java/com/yahoo/log/event/Histogram.java b/vespalog/src/main/java/com/yahoo/log/event/Histogram.java deleted file mode 100755 index 08875063a4e..00000000000 --- a/vespalog/src/main/java/com/yahoo/log/event/Histogram.java +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.log.event; - -@Deprecated(forRemoval = true, since = "7") -public class Histogram extends Event { - public Histogram () { - } - - public Histogram (String name, String value, String representation) { - init(name, value, representation); - } - - private void init (String name, String value, String representation) { - setValue("name", name); - setValue("counts", value); - setValue("representation", representation); - } -} diff --git a/vespalog/src/main/java/com/yahoo/log/event/MalformedEventException.java b/vespalog/src/main/java/com/yahoo/log/event/MalformedEventException.java index 6283832ff6d..a9f9774fd9f 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/MalformedEventException.java +++ b/vespalog/src/main/java/com/yahoo/log/event/MalformedEventException.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.log.event; -@Deprecated(forRemoval = true, since = "7") public class MalformedEventException extends Exception { public MalformedEventException (Throwable cause) { super(cause); diff --git a/vespalog/src/main/java/com/yahoo/log/event/Progress.java b/vespalog/src/main/java/com/yahoo/log/event/Progress.java index 2e54e72cb17..14ef9efe520 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Progress.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Progress.java @@ -5,8 +5,7 @@ package com.yahoo.log.event; * * @author Bjorn Borud */ -@Deprecated(forRemoval = true, since = "7") -public class Progress extends Event { +class Progress extends Event { public Progress () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/Reloaded.java b/vespalog/src/main/java/com/yahoo/log/event/Reloaded.java deleted file mode 100644 index 517c28f5f2e..00000000000 --- a/vespalog/src/main/java/com/yahoo/log/event/Reloaded.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.log.event; - -/** - * - * @author Bjorn Borud - */ -@Deprecated(forRemoval = true, since = "7") -public class Reloaded extends Event { - public Reloaded () { - } - - public Reloaded (String name) { - setValue("name", name); - } -} diff --git a/vespalog/src/main/java/com/yahoo/log/event/Reloading.java b/vespalog/src/main/java/com/yahoo/log/event/Reloading.java deleted file mode 100644 index ae537b84316..00000000000 --- a/vespalog/src/main/java/com/yahoo/log/event/Reloading.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.log.event; - -/** - * - * @author Bjorn Borud - */ -@Deprecated(forRemoval = true, since = "7") -public class Reloading extends Event { - public Reloading () { - } - - public Reloading (String name) { - setValue("name", name); - } -} diff --git a/vespalog/src/main/java/com/yahoo/log/event/Started.java b/vespalog/src/main/java/com/yahoo/log/event/Started.java index c3d90174893..af1e1bef7f7 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Started.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Started.java @@ -5,8 +5,7 @@ package com.yahoo.log.event; * * @author Bjorn Borud */ -@Deprecated(forRemoval = true, since = "7") -public class Started extends Event { +class Started extends Event { public Started () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/Starting.java b/vespalog/src/main/java/com/yahoo/log/event/Starting.java index 4f4af50a729..9a37a4b504e 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Starting.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Starting.java @@ -5,8 +5,7 @@ package com.yahoo.log.event; * * @author Bjorn Borud */ -@Deprecated(forRemoval = true, since = "7") -public class Starting extends Event { +class Starting extends Event { public Starting () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/State.java b/vespalog/src/main/java/com/yahoo/log/event/State.java index 106f4031fee..fd90ac3b95b 100755 --- a/vespalog/src/main/java/com/yahoo/log/event/State.java +++ b/vespalog/src/main/java/com/yahoo/log/event/State.java @@ -1,8 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.log.event; -@Deprecated(forRemoval = true, since = "7") -public class State extends Event { +class State extends Event { public State () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/Stopped.java b/vespalog/src/main/java/com/yahoo/log/event/Stopped.java index c6e95d877c4..15c7aee395e 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Stopped.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Stopped.java @@ -5,8 +5,7 @@ package com.yahoo.log.event; * * @author Bjorn Borud */ -@Deprecated(forRemoval = true, since = "7") -public class Stopped extends Event { +class Stopped extends Event { public Stopped () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/Stopping.java b/vespalog/src/main/java/com/yahoo/log/event/Stopping.java index daec0123e80..65693dc4a5b 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Stopping.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Stopping.java @@ -5,8 +5,7 @@ package com.yahoo.log.event; * * @author Bjorn Borud */ -@Deprecated(forRemoval = true, since = "7") -public class Stopping extends Event { +class Stopping extends Event { public Stopping () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/Unknown.java b/vespalog/src/main/java/com/yahoo/log/event/Unknown.java index b82cfebbaaa..388f8c5f363 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Unknown.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Unknown.java @@ -5,8 +5,7 @@ package com.yahoo.log.event; * * @author Bjorn Borud */ -@Deprecated(forRemoval = true, since = "7") -public class Unknown extends Event { +class Unknown extends Event { public Unknown() { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/Value.java b/vespalog/src/main/java/com/yahoo/log/event/Value.java index d0d1f6424c7..2598a34ba14 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/Value.java +++ b/vespalog/src/main/java/com/yahoo/log/event/Value.java @@ -5,8 +5,7 @@ package com.yahoo.log.event; * * @author Bjorn Borud */ -@Deprecated(forRemoval = true, since = "7") -public class Value extends Event { +class Value extends Event { public Value () { } diff --git a/vespalog/src/main/java/com/yahoo/log/event/ValueGroup.java b/vespalog/src/main/java/com/yahoo/log/event/ValueGroup.java deleted file mode 100755 index 49835ca8ed4..00000000000 --- a/vespalog/src/main/java/com/yahoo/log/event/ValueGroup.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.log.event; - -@Deprecated(forRemoval = true, since = "7") -public class ValueGroup extends Event { - public ValueGroup () { - } - - public ValueGroup (String name, String values) { - init(name, values); - } - - private void init (String name, String value) { - setValue("name", name); - setValue("values", value); - } -} diff --git a/vespalog/src/main/java/com/yahoo/log/event/package-info.java b/vespalog/src/main/java/com/yahoo/log/event/package-info.java index f2cb20e3ac8..775df301fb4 100644 --- a/vespalog/src/main/java/com/yahoo/log/event/package-info.java +++ b/vespalog/src/main/java/com/yahoo/log/event/package-info.java @@ -1,7 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage -@PublicApi // TODO Vespa 8: remove package com.yahoo.log.event; -import com.yahoo.api.annotations.PublicApi; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespalog/src/test/java/com/yahoo/log/LogSetupTestCase.java b/vespalog/src/test/java/com/yahoo/log/LogSetupTestCase.java index 354892e7d37..0d2c2adf920 100644 --- a/vespalog/src/test/java/com/yahoo/log/LogSetupTestCase.java +++ b/vespalog/src/test/java/com/yahoo/log/LogSetupTestCase.java @@ -61,7 +61,7 @@ public class LogSetupTestCase { + hostname + "\t" + pid - + "/" + zookeeperLogRecord.getThreadID() + "\t-\t.org.apache.zookeeper.server.NIOServerCnxn" + + "/" + zookeeperLogRecord.getLongThreadID() + "\t-\t.org.apache.zookeeper.server.NIOServerCnxn" + "\twarning\tzookeeper log record"; zookeeperLogRecordError = new LogRecord(Level.SEVERE, "zookeeper error"); diff --git a/vespalog/src/test/java/com/yahoo/log/TestUtil.java b/vespalog/src/test/java/com/yahoo/log/TestUtil.java new file mode 100644 index 00000000000..ffb9b2e08de --- /dev/null +++ b/vespalog/src/test/java/com/yahoo/log/TestUtil.java @@ -0,0 +1,15 @@ +package com.yahoo.log; + + +import java.util.logging.LogRecord; + +/** + * @author gjoranv + */ +public class TestUtil { + + public static String formatWithVespaFormatter(LogRecord record) { + return new VespaFormatter().format(record); + } + +} diff --git a/vespalog/src/test/java/com/yahoo/log/VespaFormatterTestCase.java b/vespalog/src/test/java/com/yahoo/log/VespaFormatterTestCase.java index 95a06ab2535..eb8656aaea9 100644 --- a/vespalog/src/test/java/com/yahoo/log/VespaFormatterTestCase.java +++ b/vespalog/src/test/java/com/yahoo/log/VespaFormatterTestCase.java @@ -41,7 +41,7 @@ public class VespaFormatterTestCase { testRecord1 = new LogRecord(Level.INFO, "this is a test"); testRecord1.setInstant(Instant.ofEpochMilli(1098709021843L)); - testRecord1.setThreadID(123); + testRecord1.setLongThreadID(123L); expected1 = "1098709021.843000\t" + hostname + "\t" @@ -64,7 +64,7 @@ public class VespaFormatterTestCase { testRecord2 = new LogRecord(Level.INFO, "this is a test"); testRecord2.setInstant(Instant.ofEpochMilli(1098709021843L)); - testRecord2.setThreadID(123); + testRecord2.setLongThreadID(123L); testRecord2.setLoggerName("org.foo"); expected3 = "1098709021.843000\t" @@ -110,7 +110,7 @@ public class VespaFormatterTestCase { LogRecord testRecord = new LogRecord(Level.INFO, "this {1} is {0} test"); testRecord.setInstant(Instant.ofEpochMilli(1098709021843L)); - testRecord.setThreadID(123); + testRecord.setLongThreadID(123L); testRecord.setLoggerName("org.foo"); Object[] params = { "a small", "message" }; testRecord.setParameters(params); @@ -228,45 +228,4 @@ public class VespaFormatterTestCase { assertTrue(formatter.format(r).contains("meldingen her")); } - /** - * This test doesn't really do that much. It is just here - * to ensure this doesn't crash. XXX TODO: make this test - * actually test something more than just the non-generation - * of runtime errors. -bb - */ - @Test - @SuppressWarnings("removal") // for VespaFormat.formatException - public void testExceptionFormatting () { - StringBuilder sb = new StringBuilder(128); - Exception e = new Exception("testing", new Exception("nested")); - VespaFormat.formatException(e, sb); - } - - - @Test - @SuppressWarnings("removal") - public void testGeneralFormat() { // for VespaFormat.format - String[] expected = new String[] { - "54.321000", - "hostname", - "26019/UnitTest-Thread-37", - "UnitTestRunner", - "com.UnitTest", - "INFO", - "Just check it looks OK\\nmsg=\"boom\"\\nname=\"java.lang.Throwable\"\\nstack=\"\\n" + this.getClass().getName() // Clover rewrites class names, get the current one to avoid test failure - }; - String formatted = VespaFormat.format("INFO", - "UnitTest", "com", 54321L, - "UnitTest-Thread-37", "UnitTestRunner", - "Just check it looks OK", - new Throwable("boom")); - String[] split = formatted.split("\t"); - assertEquals(expected[0], split[0]); - assertEquals(expected[2].split("/")[1], split[2].split("/")[1]); - assertEquals(expected[3], split[3]); - assertEquals(expected[4], split[4]); - assertEquals(expected[5], split[5]); - assertEquals(expected[6], split[6].substring(0, expected[6].length())); - assertEquals(expected.length, split.length); - } } diff --git a/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java b/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java index 77d7fa66bb6..64aec98eca2 100644 --- a/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java +++ b/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java @@ -60,7 +60,7 @@ public class VespaLogHandlerTestCase { + "\t" + pid + "/" - + record1.getThreadID() + + record1.getLongThreadID() + "\tmy-test-config-id\tTST\tinfo\tThis is a test"; record2 = new LogRecord(Level.FINE, "This is a test too"); @@ -70,7 +70,7 @@ public class VespaLogHandlerTestCase { + hostname + "\t" + pid - + "/" + record2.getThreadID() + "\tmy-test-config-id\tTST.com.yahoo.log.test\tdebug\tThis is a test too"; + + "/" + record2.getLongThreadID() + "\tmy-test-config-id\tTST.com.yahoo.log.test\tdebug\tThis is a test too"; record3 = new LogRecord(Level.WARNING, "another test"); record3.setLoggerName("com.yahoo.log.test"); @@ -79,7 +79,7 @@ public class VespaLogHandlerTestCase { + hostname + "\t" + pid - + "/" + record3.getThreadID() + "\tmy-test-config-id\tTST.com.yahoo.log.test" + + "/" + record3.getLongThreadID() + "\tmy-test-config-id\tTST.com.yahoo.log.test" + "\twarning\tanother test"; record4 = new LogRecord(Level.WARNING, "unicode \u00E6\u00F8\u00E5 test \u7881 unicode"); @@ -89,7 +89,7 @@ public class VespaLogHandlerTestCase { + hostname + "\t" + pid - + "/" + record4.getThreadID() + "\tmy-test-config-id\tTST.com.yahoo.log.test" + + "/" + record4.getLongThreadID() + "\tmy-test-config-id\tTST.com.yahoo.log.test" + "\twarning\tunicode \u00E6\u00F8\u00E5 test \u7881 unicode"; } diff --git a/vespalog/src/test/java/com/yahoo/log/event/EventTestCase.java b/vespalog/src/test/java/com/yahoo/log/event/EventTestCase.java index 045c7cd874f..bbda100cd3a 100644 --- a/vespalog/src/test/java/com/yahoo/log/event/EventTestCase.java +++ b/vespalog/src/test/java/com/yahoo/log/event/EventTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.log.event; import java.util.logging.Logger; -import com.yahoo.log.VespaFormatter; +import com.yahoo.log.TestUtil; import org.junit.Before; import org.junit.Test; @@ -23,13 +23,10 @@ public class EventTestCase { Crash crashEvent; Progress progressEvent1; Progress progressEvent2; - Reloading reloadingEvent; - Reloaded reloadedEvent; Starting startingEvent; Started startedEvent; Stopping stoppingEvent; Stopped stoppedEvent; - Collection collectionEvent; Unknown unknownEvent; @Before @@ -41,13 +38,10 @@ public class EventTestCase { crashEvent = new Crash("appname", 1234, 11); progressEvent1 = new Progress("thename", 1); progressEvent2 = new Progress("thename", 1.0, 2.0); - reloadingEvent = new Reloading("thefilewereloading"); - reloadedEvent = new Reloaded("thefilewereloaded"); startingEvent = new Starting("startingName"); startedEvent = new Started("startedName"); stoppingEvent = new Stopping("stoppingName", "because we want to"); stoppedEvent = new Stopped("stoppedName", 1234, 1); - collectionEvent=new Collection(123456, "thename"); unknownEvent = new Unknown(); } @@ -61,13 +55,10 @@ public class EventTestCase { assertNotNull(crashEvent); assertNotNull(progressEvent1); assertNotNull(progressEvent2); - assertNotNull(reloadingEvent); - assertNotNull(reloadedEvent); assertNotNull(startingEvent); assertNotNull(startedEvent); assertNotNull(stoppingEvent); assertNotNull(stoppedEvent); - assertNotNull(collectionEvent); assertNotNull(unknownEvent); } @@ -88,12 +79,6 @@ public class EventTestCase { assertEquals("progress/1 name=thename value=1.0 total=2.0", progressEvent2.toString()); - assertEquals("reloaded/1 name=thefilewereloaded", - reloadedEvent.toString()); - - assertEquals("reloading/1 name=thefilewereloading", - reloadingEvent.toString()); - assertEquals("started/1 name=startedName", startedEvent.toString()); @@ -103,9 +88,6 @@ public class EventTestCase { assertEquals("stopping/1 name=stoppingName why=\"because we want to\"", stoppingEvent.toString()); - assertEquals("collection/1 collectionId=123456 name=thename", - collectionEvent.toString()); - assertEquals("stopped/1 name=stoppedName pid=1234 exitcode=1", stoppedEvent.toString()); } @@ -124,8 +106,6 @@ public class EventTestCase { parseEvent(crashEvent); parseEvent(progressEvent1); parseEvent(progressEvent2); - parseEvent(reloadingEvent); - parseEvent(reloadedEvent); parseEvent(startingEvent); parseEvent(startedEvent); parseEvent(stoppingEvent); @@ -152,22 +132,19 @@ public class EventTestCase { SingleHandler sh = new SingleHandler(); assertNull(sh.lastRecord()); - VespaFormatter formatter = new VespaFormatter(); Logger log = Logger.getLogger(EventTestCase.class.getName()); synchronized(log) { log.setUseParentHandlers(false); log.addHandler(sh); Event.starting("mintest"); - assertTrue(formatter.format(sh.lastRecord()). - indexOf("\t.com.yahoo.log.event.EventTestCase\tevent\tstarting/1 name=mintest") > -1); + assertTrue(TestUtil.formatWithVespaFormatter(sh.lastRecord()) + .contains("\t.com.yahoo.log.event.EventTestCase\tevent\tstarting/1 name=mintest")); Event.starting("startingName"); Event.started("startedName"); Event.stopping("stoppingName", "whyParam"); Event.stopped("stoppedName", 1, 2); - Event.reloading("reloadingName"); - Event.reloaded("reloadedName"); Event.count("countName", 1); Event.progress("progressName", 1, 2); Event.crash("crashName", 1, 2); @@ -176,7 +153,7 @@ public class EventTestCase { @Test public void testFunnyEvent () { - String funnyEvent = "collection/1 collectionId=1111111111 name=\"syncrows\" params=\"column=0 badrow=1 goodrow=0\""; + String funnyEvent = "value/1 collectionId=1111111111 name=\"syncrows\" params=\"column=0 badrow=1 goodrow=0\""; try { Event e = Event.parse(funnyEvent); } diff --git a/vespalog/src/test/logtest.cpp b/vespalog/src/test/logtest.cpp index bdbb5af0e94..8b012a92f5d 100644 --- a/vespalog/src/test/logtest.cpp +++ b/vespalog/src/test/logtest.cpp @@ -15,8 +15,6 @@ main(int, char **argv) EV_STARTING("logtest"); LOG(info, "Starting up, called as %s", argv[0]); EV_STARTED("logtest"); - EV_RELOADING("something"); - EV_RELOADED("something"); EV_CRASH("something", getpid(), SIGTERM); EV_PROGRESS("batch-index", 7, 100); EV_PROGRESS("unbounded-batch-index", 9); diff --git a/vespalog/src/test/simple/logtest.cpp b/vespalog/src/test/simple/logtest.cpp index 9e3d99cf0ed..b543f1f755d 100644 --- a/vespalog/src/test/simple/logtest.cpp +++ b/vespalog/src/test/simple/logtest.cpp @@ -15,8 +15,6 @@ main(int, char **argv) EV_STARTING("logtest"); LOG(info, "Starting up, called as %s", argv[0]); EV_STARTED("logtest"); - EV_RELOADING("something"); - EV_RELOADED("something"); EV_CRASH("something", getpid(), SIGTERM); EV_PROGRESS("batch-index", 7, 100); EV_PROGRESS("unbounded-batch-index", 9); diff --git a/vespalog/src/vespa/log/log.h b/vespalog/src/vespa/log/log.h index 48cc4211ffc..5b73aef5083 100644 --- a/vespalog/src/vespa/log/log.h +++ b/vespalog/src/vespa/log/log.h @@ -98,20 +98,6 @@ do { \ } \ } while (false) -#define EV_RELOADING(name) \ -do { \ - if (logger.wants(ns_log::Logger::event)) { \ - logger.doEventReloading(name); \ - } \ -} while (false) - -#define EV_RELOADED(name) \ -do { \ - if (logger.wants(ns_log::Logger::event)) { \ - logger.doEventReloaded(name); \ - } \ -} while (false) - #define EV_CRASH(name,pid,signal) \ do { \ if (logger.wants(ns_log::Logger::event)) { \ diff --git a/yolean/abi-spec.json b/yolean/abi-spec.json index 8c10006e7e1..5085f67811c 100644 --- a/yolean/abi-spec.json +++ b/yolean/abi-spec.json @@ -109,72 +109,6 @@ ], "fields": [] }, - "com.yahoo.yolean.chain.Chain": { - "superClass": "java.lang.Object", - "interfaces": [ - "java.lang.Iterable" - ], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public varargs void <init>(java.lang.String, java.lang.Object[])", - "public void <init>(java.lang.String, java.util.List)", - "public java.lang.String id()", - "public boolean isEmpty()", - "public java.util.Iterator iterator()", - "public java.lang.String toString()", - "public int hashCode()", - "public boolean equals(java.lang.Object)", - "public boolean equals(com.yahoo.yolean.chain.Chain)" - ], - "fields": [] - }, - "com.yahoo.yolean.chain.ChainBuilder": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(java.lang.String)", - "public final varargs com.yahoo.yolean.chain.ChainBuilder add(java.lang.Object, com.yahoo.yolean.chain.Dependencies[])", - "public com.yahoo.yolean.chain.Chain build()" - ], - "fields": [] - }, - "com.yahoo.yolean.chain.ChainCycleException": { - "superClass": "java.lang.RuntimeException", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.util.List)", - "public java.util.List components()" - ], - "fields": [] - }, - "com.yahoo.yolean.chain.Dependencies": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public static varargs com.yahoo.yolean.chain.Dependencies before(java.lang.Object[])", - "public static varargs com.yahoo.yolean.chain.Dependencies before(java.lang.Class[])", - "public static varargs com.yahoo.yolean.chain.Dependencies before(java.lang.String[])", - "public static varargs com.yahoo.yolean.chain.Dependencies after(java.lang.Object[])", - "public static varargs com.yahoo.yolean.chain.Dependencies after(java.lang.Class[])", - "public static varargs com.yahoo.yolean.chain.Dependencies after(java.lang.String[])", - "public static varargs com.yahoo.yolean.chain.Dependencies provides(java.lang.String[])", - "public static com.yahoo.yolean.chain.Dependencies emptyDependencies()" - ], - "fields": [] - }, "com.yahoo.yolean.chain.Provides": { "superClass": "java.lang.Object", "interfaces": [ @@ -200,7 +134,6 @@ "public" ], "methods": [ - "public void <init>(com.yahoo.yolean.concurrent.ResourceFactory)", "public void <init>(java.util.function.Supplier)", "public void preallocate(int)", "public final java.lang.Object alloc()", @@ -265,20 +198,6 @@ ], "fields": [] }, - "com.yahoo.yolean.concurrent.ResourceFactory": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "abstract" - ], - "methods": [ - "public void <init>()", - "public abstract java.lang.Object create()", - "public final java.util.function.Supplier asSupplier()" - ], - "fields": [] - }, "com.yahoo.yolean.concurrent.ResourcePool": { "superClass": "java.lang.Object", "interfaces": [ @@ -289,7 +208,6 @@ "final" ], "methods": [ - "public void <init>(com.yahoo.yolean.concurrent.ResourceFactory)", "public void <init>(java.util.function.Supplier)", "public java.lang.Object alloc()", "public void free(java.lang.Object)", diff --git a/yolean/pom.xml b/yolean/pom.xml index 68313e3e1a8..a767e6dfcdd 100644 --- a/yolean/pom.xml +++ b/yolean/pom.xml @@ -5,11 +5,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>yolean</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/yolean/src/main/java/com/yahoo/yolean/chain/Chain.java b/yolean/src/main/java/com/yahoo/yolean/chain/Chain.java deleted file mode 100644 index 516e3786f28..00000000000 --- a/yolean/src/main/java/com/yahoo/yolean/chain/Chain.java +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yolean.chain; - -import com.google.common.collect.ImmutableList; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; - -import static java.util.Objects.requireNonNull; - -/** - * An immutable and ordered list of components - * - * @author Tony Vaagenes - * @deprecated Will be removed in Vespa 8 with no replacement. - */ -@Deprecated -public final class Chain<T> implements Iterable<T> { - - private final String id; - private final Collection<T> components; - - @SafeVarargs - @SuppressWarnings("varargs") - public Chain(String id, T... components) { - this(id, Arrays.asList(components)); - } - - public Chain(String id, List<? extends T> components) { - requireNonNull(id, "id must be non-null."); - requireNonNull(components, "components must be non-null"); - - this.components = ImmutableList.copyOf(components); - this.id = id; - } - - public String id() { - return id; - } - - public boolean isEmpty() { - return components.isEmpty(); - } - - @Override - public Iterator<T> iterator() { - return components.iterator(); - } - - @Override - public String toString() { - StringBuilder b = new StringBuilder(); - b.append("chain '").append(id).append("'{"); - boolean first = true; - for (T component : components) { - if (!first) { - b.append("->"); - } else { - first = false; - } - b.append(" ").append(component.getClass().getSimpleName()).append(" "); - } - b.append("}"); - return b.toString(); - } - - @Override - public int hashCode() { - return id.hashCode(); - } - - @Override - public boolean equals(Object other) { - return other instanceof Chain && equals((Chain<?>)other); - } - - public boolean equals(Chain<?> other) { - return id.equals(other.id) && componentsIdentical(components, other.components); - } - - private boolean componentsIdentical(Collection<T> components1, Collection<?> components2) { - if (components1.size() != components2.size()) { - return false; - } - Iterator<T> iterator1 = components1.iterator(); - Iterator<?> iterator2 = components2.iterator(); - while (iterator1.hasNext()) { - T c1 = iterator1.next(); - Object c2 = iterator2.next(); - - if (c1 != c2) { - return false; - } - } - return true; - } - -} diff --git a/yolean/src/main/java/com/yahoo/yolean/chain/ChainBuilder.java b/yolean/src/main/java/com/yahoo/yolean/chain/ChainBuilder.java deleted file mode 100644 index cb79c05ddab..00000000000 --- a/yolean/src/main/java/com/yahoo/yolean/chain/ChainBuilder.java +++ /dev/null @@ -1,250 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yolean.chain; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.IdentityHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * @author Tony Vaagenes - * @author gjoranv - * @deprecated Will be removed in Vespa 8 with no replacement. - */ -@Deprecated -public final class ChainBuilder<T> { - - private final String chainId; - private final List<T> components = new ArrayList<>(); - private final IdentityHashMap<T, Dependencies<T>> dependencies = new IdentityHashMap<>(); - - public ChainBuilder(String chainId) { - this.chainId = chainId; - } - - @SafeVarargs - @SuppressWarnings("varargs") - public final ChainBuilder<T> add(T component, Dependencies<? extends T>... dependenciesList) { - if (dependencies.containsKey(component)) { - throw new IllegalArgumentException("The same component cannot be added twice: " + component); - } - components.add(component); - - List<Dependencies<? extends T>> allDependencies = - Dependencies.allOf(dependenciesList, Dependencies.getAnnotatedDependencies(component)); - dependencies.put(component, Dependencies.union(allDependencies)); - - return this; - } - - public Chain<T> build() { - DirectedGraph graph = buildGraph(); - List<Vertex> sortedVertices = graph.topologicalSort(); - return new Chain<>(chainId, components(sortedVertices)); - } - - private DirectedGraph buildGraph() { - DirectedGraph graph = new DirectedGraph(); - List<ComponentVertex<T>> vertices = createVertices(); - - addVertices(graph, vertices); - addEdges(graph, vertices, dependencies); - - return graph; - } - - private List<ComponentVertex<T>> createVertices() { - List<ComponentVertex<T>> vertices = new ArrayList<>(); - for (T component : components) { - vertices.add(new ComponentVertex<>(component)); - } - return vertices; - } - - @SuppressWarnings("unchecked") - private List<T> components(List<Vertex> sortedVertices) { - List<T> result = new ArrayList<>(); - for (Vertex vertex : sortedVertices) { - if (vertex instanceof ComponentVertex) { - result.add((T)((ComponentVertex)vertex).component); - } - } - return result; - } - - // TODO: create subclasses Beginning/EdingVertex instead? We could then create the correct class in createVertices, - // TODO: and call the same method in DirGraph for all types. - private void addVertices(DirectedGraph graph, List<ComponentVertex<T>> vertices) { - for (ComponentVertex<T> v : vertices) { - if (isBeginningVertex(v)) { - graph.addBeginningVertex(v); - } else if (isEndingVertex(v)) { - graph.addEndingVertex(v); - } else { - graph.addVertex(v); - } - } - } - - private boolean isBeginningVertex(ComponentVertex<T> v) { - return dependencies.get(v.component).before.providedNames.contains("*"); - } - - private boolean isEndingVertex(ComponentVertex<T> v) { - return dependencies.get(v.component).after.providedNames.contains("*"); - } - - private static <T> void addEdges(DirectedGraph graph, - List<ComponentVertex<T>> vertices, - IdentityHashMap<T, Dependencies<T>> dependencies) { - addBeforeInstanceEdges(graph, vertices, dependencies); - addAfterInstanceEdges(graph, vertices, dependencies); - addBeforeClassEdges(graph, vertices, dependencies); - addAfterClassEdges(graph, vertices, dependencies); - addBeforeProvidedEdges(graph, vertices, dependencies); - addAfterProvidedEdges(graph, vertices, dependencies); - } - - // NOTE: When reading 'beforeVertex' below, think that 'vertex' should be _before_ beforeVertex. - - private static <T> void addBeforeClassEdges(DirectedGraph graph, - List<ComponentVertex<T>> vertices, - IdentityHashMap<T, Dependencies<T>> dependencies) { - for (ComponentVertex<T> vertex : vertices) { - for (Class<? extends T> beforeClass : dependencies.get(vertex.component).before.classes) { - for (Vertex beforeVertex : componentsWithClass(vertices, beforeClass)) { - graph.addEdge(vertex, beforeVertex); - } - } - } - } - - private static <T> void addAfterClassEdges(DirectedGraph graph, - List<ComponentVertex<T>> vertices, - IdentityHashMap<T, Dependencies<T>> dependencies) { - for (ComponentVertex<T> vertex : vertices) { - for (Class<? extends T> afterClass : dependencies.get(vertex.component).after.classes) { - for (Vertex afterVertex : componentsWithClass(vertices, afterClass)) { - graph.addEdge(afterVertex, vertex); - } - } - } - } - - private static <T> List<Vertex> componentsWithClass(List<ComponentVertex<T>> vertices, - Class<? extends T> beforeClass) { - List<Vertex> result = new ArrayList<>(); - for (ComponentVertex<T> vertex : vertices) { - if (beforeClass.isInstance(vertex.component)) { - result.add(vertex); - } - } - return result; - } - - private static <T> void addBeforeInstanceEdges(DirectedGraph graph, - List<ComponentVertex<T>> vertices, - IdentityHashMap<T, Dependencies<T>> dependencies) { - IdentityHashMap<T, Vertex> componentToVertex = getComponentToVertex(vertices); - for (ComponentVertex<T> vertex : vertices) { - for (T before : dependencies.get(vertex.component).before.instances) { - Vertex beforeVertex = componentToVertex.get(before); - if (beforeVertex != null) { - graph.addEdge(vertex, beforeVertex); - } - } - } - } - - private static <T> void addAfterInstanceEdges(DirectedGraph graph, - List<ComponentVertex<T>> vertices, - IdentityHashMap<T, Dependencies<T>> dependencies) { - IdentityHashMap<T, Vertex> componentToVertex = getComponentToVertex(vertices); - for (ComponentVertex<T> vertex : vertices) { - for (T after : dependencies.get(vertex.component).after.instances) { - Vertex afterVertex = componentToVertex.get(after); - if (afterVertex != null) { - graph.addEdge(afterVertex, vertex); - } - } - } - } - - private static <T> IdentityHashMap<T, Vertex> getComponentToVertex(List<ComponentVertex<T>> vertices) { - IdentityHashMap<T, Vertex> result = new IdentityHashMap<>(); - for (ComponentVertex<T> vertex : vertices) { - result.put(vertex.component, vertex); - } - return result; - } - - private static <T> void addBeforeProvidedEdges(DirectedGraph graph, - List<ComponentVertex<T>> vertices, - IdentityHashMap<T, Dependencies<T>> dependencies) { - Map<String, Set<Vertex>> providedNames = getProvidedNames(vertices, dependencies); - for (ComponentVertex<T> vertex : vertices) { - for (String name : dependencies.get(vertex.component).before.providedNames) { - for (Vertex beforeVertex : emptyIfNull(providedNames.get(name))) { - graph.addEdge(vertex, beforeVertex); - } - } - } - } - - private static <T> void addAfterProvidedEdges(DirectedGraph graph, - List<ComponentVertex<T>> vertices, - IdentityHashMap<T, Dependencies<T>> dependencies) { - Map<String, Set<Vertex>> providedNames = getProvidedNames(vertices, dependencies); - for (ComponentVertex<T> vertex : vertices) { - for (String name : dependencies.get(vertex.component).after.providedNames) { - for (Vertex afterVertex : emptyIfNull(providedNames.get(name))) { - graph.addEdge(afterVertex, vertex); - } - } - } - } - - private static <T> Map<String, Set<Vertex>> getProvidedNames(List<ComponentVertex<T>> vertices, - IdentityHashMap<T, Dependencies<T>> dependencies) { - Map<String, Set<Vertex>> result = new HashMap<>(); - for (ComponentVertex<T> vertex : vertices) { - for (String providedName : dependencies.get(vertex.component).provided) { - getIdentitySet(result, providedName).add(vertex); - } - addClassName(result, vertex); - } - return result; - } - - private static void addClassName(Map<String, Set<Vertex>> providedNamesToVertex, ComponentVertex<?> vertex) { - String className = vertex.component.getClass().getName(); - getIdentitySet(providedNamesToVertex, className).add(vertex); - } - - private static <T> Set<T> getIdentitySet(Map<String, Set<T>> map, String key) { - Set<T> result = map.get(key); - if (result == null) { - result = Collections.newSetFromMap(new IdentityHashMap<T, Boolean>()); - map.put(key, result); - } - return result; - } - - private static <T> Set<T> emptyIfNull(Set<T> set) { - return set != null ? - set : - Collections.<T>emptySet(); - } - - private static class ComponentVertex<T> implements Vertex { - - final T component; - - private ComponentVertex(T component) { - this.component = component; - } - } -} diff --git a/yolean/src/main/java/com/yahoo/yolean/chain/ChainCycleException.java b/yolean/src/main/java/com/yahoo/yolean/chain/ChainCycleException.java deleted file mode 100644 index 6d8e14d696d..00000000000 --- a/yolean/src/main/java/com/yahoo/yolean/chain/ChainCycleException.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yolean.chain; - -import com.google.common.collect.ImmutableList; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * @author Tony Vaagenes - * @deprecated Will be removed in Vespa 8 with no replacement. - */ -@Deprecated -public class ChainCycleException extends RuntimeException { - - private final List<?> components; - - public ChainCycleException(List<?> components) { - this.components = ImmutableList.copyOf(components); - } - - public List<?> components() { - return components; - } - -} diff --git a/yolean/src/main/java/com/yahoo/yolean/chain/Dependencies.java b/yolean/src/main/java/com/yahoo/yolean/chain/Dependencies.java deleted file mode 100644 index 57f74190aff..00000000000 --- a/yolean/src/main/java/com/yahoo/yolean/chain/Dependencies.java +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yolean.chain; - -import java.lang.annotation.Annotation; -import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; - -/** - * @author Tony Vaagenes - * @author gjoranv - * @deprecated Will be removed in Vespa 8 with no replacement. - */ -@Deprecated -public class Dependencies<T> { - - final Order<T> before; - final Order<T> after; - final List<String> provided; - - private Dependencies(Order<T> before, Order<T> after, String[] provided) { - this.before = before; - this.after = after; - this.provided = copyList(provided); - } - - @SafeVarargs - @SuppressWarnings("varargs") - public static <T> Dependencies<T> before(T... components) { - return new Dependencies<>(new Order<>(components, null, null), Order.<T>emptyOrder(), null); - } - - @SafeVarargs - @SuppressWarnings("varargs") - public static <T> Dependencies<T> before(Class<? extends T>... classes) { - return new Dependencies<>(new Order<>(null, classes, null), Order.<T>emptyOrder(), null); - } - - public static <T> Dependencies<T> before(String... providedNames) { - return new Dependencies<>(new Order<T>(null, null, providedNames), Order.<T>emptyOrder(), null); - } - - @SafeVarargs - @SuppressWarnings("varargs") - public static <T> Dependencies<T> after(T... components) { - return new Dependencies<>(Order.<T>emptyOrder(), new Order<>(components, null, null), null); - } - - @SafeVarargs - @SuppressWarnings("varargs") - public static <T> Dependencies<T> after(Class<? extends T>... classes) { - return new Dependencies<>(Order.<T>emptyOrder(), new Order<>(null, classes, null), null); - } - - public static <T> Dependencies<T> after(String... providedNames) { - return new Dependencies<>(Order.<T>emptyOrder(), new Order<T>(null, null, providedNames), null); - } - - public static <T> Dependencies<T> provides(String... names) { - return new Dependencies<>(Order.<T>emptyOrder(), Order.<T>emptyOrder(), names); - } - - public static <T> Dependencies<T> emptyDependencies() { - return new Dependencies<>(Order.<T>emptyOrder(), Order.<T>emptyOrder(), null); - } - - @SuppressWarnings("unchecked") - static <T> Dependencies<T> union(List<Dependencies<? extends T>> dependenciesList) { - if (dependenciesList.size() > 1) { - Dependencies<T> result = emptyDependencies(); - for (Dependencies<? extends T> dependencies : dependenciesList) { - result = result.union(dependencies); - } - return result; - } else if (dependenciesList.size() == 0) { - return emptyDependencies(); - } else { - return (Dependencies<T>)dependenciesList.get(0); // Dependencies<T> is covariant for T, the cast is valid. - } - } - - private Dependencies<T> union(Dependencies<? extends T> other) { - List<String> lst = listUnion(provided, other.provided); - return new Dependencies<>(before.union(other.before), - after.union(other.after), - lst.toArray(new String[lst.size()])); - } - - private static <T> List<T> listUnion(List<? extends T> list1, List<? extends T> list2) { - List<T> union = new ArrayList<>(list1); - union.removeAll(list2); - union.addAll(list2); - return union; - } - - static <T> Dependencies<T> getAnnotatedDependencies(T component) { - return new Dependencies<>( - new Order<T>(null, null, getSymbols(component, Before.class)), - new Order<T>(null, null, getSymbols(component, After.class)), - getProvidedSymbols(component)); - } - - private static <T> String[] getProvidedSymbols(T component) { - List<String> lst = allOf(getSymbols(component, Provides.class), component.getClass().getName()); - return lst.toArray(new String[lst.size()]); - } - - @SafeVarargs - @SuppressWarnings("varargs") - static <T> List<T> allOf(List<T> elements, T... otherElements) { - List<T> result = new ArrayList<>(elements); - result.addAll(Arrays.asList(otherElements)); - return result; - } - - @SafeVarargs - @SuppressWarnings("varargs") - static <T> List<T> allOf(T[] elements, T... otherElements) { - return allOf(Arrays.asList(elements), otherElements); - } - - private static <T> List<String> getSymbols(T component, Class<? extends Annotation> annotationClass) { - List<String> result = new ArrayList<>(); - - result.addAll(annotationSymbols(component, annotationClass)); - return result; - } - - private static <T> Collection<String> annotationSymbols(T component, Class<? extends Annotation> annotationClass) { - try { - List<String> values = new ArrayList<>(); - - Class<?> clazz = component.getClass(); - while (clazz != null) { - Annotation annotation = clazz.getAnnotation(annotationClass); - if (annotation != null) { - values.addAll(Arrays.asList((String[])annotationClass.getMethod("value").invoke(annotation))); - } - clazz = clazz.getSuperclass(); - } - return values; - } catch (IllegalAccessException | NoSuchMethodException | InvocationTargetException e) { - throw new RuntimeException(e); - } - } - - private static <U> List<U> copyList(List<U> list) { - return list == null ? - Collections.<U>emptyList() : - new ArrayList<>(list); - } - - private static <U> List<U> copyList(U[] array) { - return array == null ? - Collections.<U>emptyList() : - new ArrayList<>(Arrays.<U>asList(array)); - } - - static final class Order<T> { - - final List<T> instances; - final List<Class<? extends T>> classes; - final List<String> providedNames; - - private Order(T[] instances, Class<? extends T>[] classes, String[] providedNames) { - this.instances = copyList(instances); - this.classes = copyList(classes); - this.providedNames = copyList(providedNames); - } - - private Order(List<T> instances, List<Class<? extends T>> classes, List<String> providedNames) { - this.instances = copyList(instances); - this.classes = copyList(classes); - this.providedNames = copyList(providedNames); - } - - // TODO: unit test - private Order<T> union(Order<? extends T> other) { - return new Order<>( - listUnion(instances, other.instances), - listUnion(classes, other.classes), - listUnion(providedNames, other.providedNames)); - } - - // TODO: try to make it possible to use 'null' Order in Dependencies instead. - private static <U> Order<U> emptyOrder() { - return new Order<>((U[])null, null, null); - } - } - -} diff --git a/yolean/src/main/java/com/yahoo/yolean/chain/DirectedGraph.java b/yolean/src/main/java/com/yahoo/yolean/chain/DirectedGraph.java deleted file mode 100644 index 0a481042af4..00000000000 --- a/yolean/src/main/java/com/yahoo/yolean/chain/DirectedGraph.java +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yolean.chain; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.IdentityHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * TODO: prioritize vertices in edge map. - * - * @author Tony Vaagenes - */ -@Deprecated -class DirectedGraph { - - private IdentityHashMap<Vertex, List<Vertex>> incommingEdges = new IdentityHashMap<>(); - private List<Vertex> vertices = new ArrayList<>(); - private List<Vertex> beginningVertices = new ArrayList<>(); - private List<Vertex> endingVertices = new ArrayList<>(); - - public void addVertex(Vertex vertex) { - vertices.add(vertex); - } - - public void addBeginningVertex(Vertex vertex) { - beginningVertices.add(vertex); - } - - public void addEndingVertex(Vertex vertex) { - endingVertices.add(vertex); - } - - public void addEdge(Vertex start, Vertex end) { - get(incommingEdges, end).add(start); - } - - private static List<Vertex> get(Map<Vertex, List<Vertex>> edgeMap, Vertex key) { - List<Vertex> value = edgeMap.get(key); - return value == null ? - addEmptyList(edgeMap, key) : - value; - } - - private static List<Vertex> addEmptyList(Map<Vertex, List<Vertex>> edgeMap, Vertex key) { - List<Vertex> value = new ArrayList<>(); - edgeMap.put(key, value); - return value; - } - - public List<Vertex> topologicalSort() { - EnumeratedIdentitySet<Vertex> visitedVertices = new EnumeratedIdentitySet<>(); - - for (Vertex v : beginningVertices) { - topologicalSortVisit(v, visitedVertices); - } - - warnIfVisitedEndVertices(visitedVertices); - - for (Vertex v : vertices) { - topologicalSortVisit(v, visitedVertices); - } - - // TODO: review this! - for (Vertex v : endingVertices) { - topologicalSortVisit(v, visitedVertices); - } - - return visitedVertices.insertionOrderedList(); - } - - private void warnIfVisitedEndVertices(EnumeratedIdentitySet<Vertex> visitedVertices) { - //TVT: - } - - private void topologicalSortVisit(Vertex vertex, Set<Vertex> visitedVertices) { - topologicalSortVisitImpl(vertex, visitedVertices, new EnumeratedIdentitySet<Vertex>()); - } - - private void topologicalSortVisitImpl(Vertex vertex, Set<Vertex> visitedVertices, - EnumeratedIdentitySet<Vertex> cycleDetector) { - if (cycleDetector.contains(vertex)) { - throw new ChainCycleException(cycleDetector.insertionOrderedList()); - } - - if (visitedVertices.contains(vertex)) { - return; - } - - cycleDetector.add(vertex); - - for (Vertex endVertex : emptyIfNull(incommingEdges.get(vertex))) { - topologicalSortVisitImpl(endVertex, visitedVertices, cycleDetector); - } - - visitedVertices.add(vertex); - cycleDetector.remove(vertex); - } - - private <T> List<T> emptyIfNull(List<T> list) { - return list == null ? - Collections.<T>emptyList() : - list; - } - -} diff --git a/yolean/src/main/java/com/yahoo/yolean/chain/EnumeratedIdentitySet.java b/yolean/src/main/java/com/yahoo/yolean/chain/EnumeratedIdentitySet.java deleted file mode 100644 index 220696e6c5f..00000000000 --- a/yolean/src/main/java/com/yahoo/yolean/chain/EnumeratedIdentitySet.java +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yolean.chain; - -import java.lang.reflect.Array; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.IdentityHashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; - -/** - * A set using identity comparison. - * Keeps track of insertion order, which is available by calling insertionOrderedList. - * - * @author Tony Vaagenes - */ -@Deprecated -class EnumeratedIdentitySet<T> implements Set<T> { - - private int counter = 0; - private final Map<T, Integer> set = new IdentityHashMap<>(); - - public EnumeratedIdentitySet(Collection<? extends T> collection) { - addAll(collection); - } - - public EnumeratedIdentitySet() { - // empty - } - - @Override - public int size() { - return set.size(); - } - - @Override - public boolean isEmpty() { - return set.isEmpty(); - } - - @Override - public boolean contains(Object o) { - return set.containsKey(o); - } - - @Override - public Iterator<T> iterator() { - return set.keySet().iterator(); - } - - @Override - public Object[] toArray() { - return set.keySet().toArray(); - } - - @Override - public <T1> T1[] toArray(T1[] a) { - return set.keySet().toArray(a); - } - - @Override - public boolean add(T t) { - if (set.containsKey(t)) { - return false; - } else { - set.put(t, counter++); - return true; - } - } - - @Override - public boolean remove(Object o) { - return set.remove(o) != null; - } - - @Override - public boolean containsAll(Collection<?> c) { - return set.keySet().containsAll(c); - } - - @Override - public boolean addAll(Collection<? extends T> collection) { - boolean changed = false; - - for (T t : collection) { - changed |= add(t); - } - - return changed; - } - - @Override - public boolean retainAll(Collection<?> collection) { - return set.keySet().retainAll(collection); - } - - @Override - public boolean removeAll(Collection<?> collection) { - boolean changed = false; - - for (Object o : collection) { - changed |= remove(o); - } - - return changed; - } - - @Override - public void clear() { - set.clear(); - counter = 0; - } - - public List<T> insertionOrderedList() { - if (set.isEmpty()) { - counter = 0; - return Collections.emptyList(); - } - - if (counter >= set.size() * 2 + 20) { - renumber(); - } - - return getKeysSortedByValue(set, counter); - } - - private static <KEY> List<KEY> getKeysSortedByValue(Map<KEY, Integer> set, int maxValue) { - @SuppressWarnings("unchecked") - KEY[] result = (KEY[])Array.newInstance(headKey(set).getClass(), maxValue); - - for (Map.Entry<KEY, Integer> entry : set.entrySet()) { - result[entry.getValue()] = entry.getKey(); - } - - return removeNulls(result); - } - - private static <T> T headKey(Map<T, ?> map) { - return map.entrySet().iterator().next().getKey(); - } - - static <T> List<T> removeNulls(T[] list) { - int insertionSpot = 0; - for (int i = 0; i < list.length; i++) { - T element = list[i]; - if (element != null) { - list[insertionSpot] = element; - insertionSpot++; - } - } - return Arrays.asList(list).subList(0, insertionSpot); - } - - //only for testing - List<Integer> numbers() { - return new ArrayList<>(set.values()); - } - - private void renumber() { - SortedMap<Integer, T> invertedSet = invertedSortedMap(set); - - int i = 0; - for (Map.Entry<Integer, T> entry : invertedSet.entrySet()) { - set.put(entry.getValue(), i++); - } - counter = i; - } - - private static <K, V> SortedMap<V, K> invertedSortedMap(Map<K, V> map) { - SortedMap<V, K> result = new TreeMap<>(); - - for (Map.Entry<K, V> entry : map.entrySet()) { - result.put(entry.getValue(), entry.getKey()); - } - - return result; - } - -} diff --git a/yolean/src/main/java/com/yahoo/yolean/chain/Vertex.java b/yolean/src/main/java/com/yahoo/yolean/chain/Vertex.java deleted file mode 100644 index 5a506406ebf..00000000000 --- a/yolean/src/main/java/com/yahoo/yolean/chain/Vertex.java +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yolean.chain; - -/** - * @author Tony Vaagenes - */ -@Deprecated -interface Vertex { - -} diff --git a/yolean/src/main/java/com/yahoo/yolean/concurrent/ConcurrentResourcePool.java b/yolean/src/main/java/com/yahoo/yolean/concurrent/ConcurrentResourcePool.java index bdd059f3e17..0e91a44bf5d 100644 --- a/yolean/src/main/java/com/yahoo/yolean/concurrent/ConcurrentResourcePool.java +++ b/yolean/src/main/java/com/yahoo/yolean/concurrent/ConcurrentResourcePool.java @@ -18,12 +18,6 @@ public class ConcurrentResourcePool<T> implements Iterable<T> { private final Queue<T> pool = new ConcurrentLinkedQueue<>(); private final Supplier<T> factory; - /** @deprecated Use {@link ConcurrentResourcePool(Supplier)} instead */ - @Deprecated(forRemoval = true, since = "7") - @SuppressWarnings("removal") - public ConcurrentResourcePool(ResourceFactory<T> factory) { - this.factory = factory.asSupplier(); - } public ConcurrentResourcePool(Supplier<T> factory) { this.factory = factory; diff --git a/yolean/src/main/java/com/yahoo/yolean/concurrent/ResourceFactory.java b/yolean/src/main/java/com/yahoo/yolean/concurrent/ResourceFactory.java deleted file mode 100644 index cc9acf69684..00000000000 --- a/yolean/src/main/java/com/yahoo/yolean/concurrent/ResourceFactory.java +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.yolean.concurrent; - -import java.util.function.Supplier; - -/** - * @author baldersheim - */ -@Deprecated(forRemoval = true, since = "7") -public abstract class ResourceFactory<T> { - - public abstract T create(); - - public final Supplier<T> asSupplier() { - return () -> create(); - } - -} diff --git a/yolean/src/main/java/com/yahoo/yolean/concurrent/ResourcePool.java b/yolean/src/main/java/com/yahoo/yolean/concurrent/ResourcePool.java index 895fa890beb..ffc761ad625 100644 --- a/yolean/src/main/java/com/yahoo/yolean/concurrent/ResourcePool.java +++ b/yolean/src/main/java/com/yahoo/yolean/concurrent/ResourcePool.java @@ -18,12 +18,6 @@ public final class ResourcePool<T> implements Iterable<T> { private final Deque<T> pool = new ArrayDeque<>(); private final Supplier<T> factory; - /** @deprecated Use {@link ResourcePool( Supplier )} instead */ - @Deprecated(forRemoval = true, since = "7") - @SuppressWarnings("removal") - public ResourcePool(ResourceFactory<T> factory) { - this(factory.asSupplier()); - } public ResourcePool(Supplier<T> factory) { this.factory = factory; diff --git a/zkfacade/pom.xml b/zkfacade/pom.xml index 86a60702c26..0360b9ce91f 100644 --- a/zkfacade/pom.xml +++ b/zkfacade/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>zkfacade</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <dependency> <groupId>junit</groupId> @@ -34,6 +34,12 @@ <artifactId>vespajlib</artifactId> <version>${project.version}</version> <scope>provided</scope> + <exclusions> + <exclusion> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java index 3e1198a4e3e..859dcc2f93f 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java @@ -114,7 +114,7 @@ public class Curator extends AbstractComponent implements VespaCurator, AutoClos .connectionTimeoutMs((int) ZK_CONNECTION_TIMEOUT.toMillis()) .connectString(connectionSpec.local()) .zookeeperFactory(new VespaZooKeeperFactory(createClientConfig(clientConfigFile))) - .dontUseContainerParents() // TODO: Remove when we know ZooKeeper 3.5 works fine, consider waiting until Vespa 8 + .dontUseContainerParents() // TODO: Consider changing this in Vespa 9 .build(), juteMaxBuffer); } diff --git a/zookeeper-client-common/pom.xml b/zookeeper-client-common/pom.xml index 7e23e0cb1c8..5eafd040e79 100644 --- a/zookeeper-client-common/pom.xml +++ b/zookeeper-client-common/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>zookeeper-client-common</artifactId> <packaging>jar</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <!-- provided --> diff --git a/zookeeper-command-line-client/pom.xml b/zookeeper-command-line-client/pom.xml index 5dfc5ab22be..a0c268e0f95 100644 --- a/zookeeper-command-line-client/pom.xml +++ b/zookeeper-command-line-client/pom.xml @@ -5,11 +5,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>zookeeper-command-line-client</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <dependency> <groupId>org.apache.zookeeper</groupId> diff --git a/zookeeper-server/pom.xml b/zookeeper-server/pom.xml index 34f4c0e9191..b9cc4478bc2 100644 --- a/zookeeper-server/pom.xml +++ b/zookeeper-server/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>zookeeper-server-parent</artifactId> <packaging>pom</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <modules> <module>zookeeper-server-common</module> <module>zookeeper-server</module> @@ -23,6 +23,12 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>security-utils</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> </dependencies> <build> <plugins> diff --git a/zookeeper-server/zookeeper-server-3.7.1/pom.xml b/zookeeper-server/zookeeper-server-3.7.1/pom.xml index ed677ef8601..ea0e61075f1 100644 --- a/zookeeper-server/zookeeper-server-3.7.1/pom.xml +++ b/zookeeper-server/zookeeper-server-3.7.1/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>zookeeper-server-parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <artifactId>zookeeper-server-3.7.1</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <properties> <zookeeper.version>3.7.1</zookeeper.version> </properties> diff --git a/zookeeper-server/zookeeper-server-common/pom.xml b/zookeeper-server/zookeeper-server-common/pom.xml index 82494e7bd14..db377fa8e8f 100644 --- a/zookeeper-server/zookeeper-server-common/pom.xml +++ b/zookeeper-server/zookeeper-server-common/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>zookeeper-server-parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <artifactId>zookeeper-server-common</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <dependencies> <dependency> <groupId>junit</groupId> diff --git a/zookeeper-server/zookeeper-server/pom.xml b/zookeeper-server/zookeeper-server/pom.xml index 01cd666d5ab..72cb21b2362 100644 --- a/zookeeper-server/zookeeper-server/pom.xml +++ b/zookeeper-server/zookeeper-server/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>zookeeper-server-parent</artifactId> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> <artifactId>zookeeper-server-3.7.0</artifactId> <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> + <version>8-SNAPSHOT</version> <properties> <zookeeper.version>3.7.0</zookeeper.version> </properties> |