diff options
| author | Geir Storli <geirstorli@yahoo.no> | 2019-01-21 16:06:53 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2019-01-21 16:06:53 +0100 |
| commit | 8372a883c5a5fa100f88fc9b80824359b5bb70cd (patch) | |
| tree | 660ce3ef61f060b33bd97eac1185715606b8e2bd | |
| parent | 3722c1cdd91fce30d1c2538b2a8749d9321e194b (diff) | |
| parent | eb0b1134a66507e3bd8f09793c22cd824d01dff5 (diff) | |
Merge pull request #8198 from vespa-engine/7
7 MERGEOK
1031 files changed, 9027 insertions, 36940 deletions
diff --git a/.travis.yml b/.travis.yml index c1f970abad4..54c98daca89 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ cache: branches: only: - - master + - 7 before_cache: - sudo rm -rf $HOME/.m2/repository/com/yahoo/vespa @@ -1 +1 @@ -6.240.0 +7.0.0 diff --git a/abi-check-plugin/pom.xml b/abi-check-plugin/pom.xml index 5b12c82f378..6931e0b9b2d 100644 --- a/abi-check-plugin/pom.xml +++ b/abi-check-plugin/pom.xml @@ -6,22 +6,18 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>abi-check-plugin</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>maven-plugin</packaging> <name>${project.artifactId}</name> <description>Maven Plugin for ensuring ABI stability.</description> - <prerequisites> - <maven>2.2.1</maven> - </prerequisites> <dependencies> <dependency> <groupId>org.apache.maven</groupId> <artifactId>maven-plugin-api</artifactId> - <version>3.5.0</version> </dependency> <dependency> <groupId>org.apache.maven</groupId> @@ -35,7 +31,6 @@ <dependency> <groupId>org.ow2.asm</groupId> <artifactId>asm</artifactId> - <version>7.0</version> </dependency> <dependency> <groupId>com.google.guava</groupId> @@ -76,6 +71,25 @@ <artifactId>clover-maven-plugin</artifactId> <version>4.3.1</version> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <executions> + <execution> + <id>enforce-maven</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <requireMavenVersion> + <version>[3.5, )</version> + </requireMavenVersion> + </rules> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> </project> diff --git a/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/AnnotationCollector.java b/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/AnnotationCollector.java index 5be66a4fbdd..cc2b37f13c9 100644 --- a/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/AnnotationCollector.java +++ b/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/AnnotationCollector.java @@ -12,7 +12,7 @@ public class AnnotationCollector extends ClassVisitor { private final Set<String> annotations = new HashSet<>(); public AnnotationCollector() { - super(Opcodes.ASM6); + super(Opcodes.ASM7); } @Override diff --git a/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/PublicSignatureCollector.java b/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/PublicSignatureCollector.java index d90b3c1a8dd..8bd7f6f7927 100644 --- a/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/PublicSignatureCollector.java +++ b/abi-check-plugin/src/main/java/com/yahoo/abicheck/collector/PublicSignatureCollector.java @@ -26,7 +26,7 @@ public class PublicSignatureCollector extends ClassVisitor { private Set<String> currentFields; public PublicSignatureCollector() { - super(Opcodes.ASM6); + super(Opcodes.ASM7); } private static boolean testBit(long access, long mask) { diff --git a/annotations/pom.xml b/annotations/pom.xml index da3d69199bd..df4b000372a 100644 --- a/annotations/pom.xml +++ b/annotations/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>annotations</artifactId> <packaging>jar</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>annotations</name> <description>Public API annotations</description> <repositories> diff --git a/application-deploy-plugin/pom.xml b/application-deploy-plugin/pom.xml index 5497466bfec..9eb0d61339f 100644 --- a/application-deploy-plugin/pom.xml +++ b/application-deploy-plugin/pom.xml @@ -6,11 +6,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>application-deploy-plugin</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>maven-plugin</packaging> <description>Maven Plugin for deploying a vespa application package to a cloud config server</description> <prerequisites> diff --git a/application-model/pom.xml b/application-model/pom.xml index b4a3949fdb4..ebea834aec3 100644 --- a/application-model/pom.xml +++ b/application-model/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>application-model</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>${project.artifactId}</name> <description>Model classes for Vespa application entities such as host name, application id etc.</description> <dependencies> diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceType.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceType.java index 07f2cd604b4..f01d3c3b3fb 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceType.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ServiceType.java @@ -9,6 +9,7 @@ import java.util.Objects; * @author bjorncs */ public class ServiceType { + // Common service types. public static final ServiceType CONTAINER = new ServiceType("container"); public static final ServiceType SLOBROK = new ServiceType("slobrok"); diff --git a/application-preprocessor/pom.xml b/application-preprocessor/pom.xml index e6371073271..b3b85a776fe 100644 --- a/application-preprocessor/pom.xml +++ b/application-preprocessor/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>application-preprocessor</artifactId> <packaging>jar</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/application/pom.xml b/application/pom.xml index 9e6334836af..f9f3e8394af 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>application</artifactId> <packaging>jar</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <description>Runs an application directly from services.xml</description> <dependencies> <dependency> @@ -129,6 +129,12 @@ <groupId>xerces</groupId> <artifactId>xercesImpl</artifactId> </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + <scope>test</scope> + </dependency> + <!-- These dependencies are necessary in test classpath when using jdisc_http_filters --> <dependency> diff --git a/application/src/main/java/com/yahoo/application/container/Search.java b/application/src/main/java/com/yahoo/application/container/Search.java index 116068105cb..16db6ef4cc8 100644 --- a/application/src/main/java/com/yahoo/application/container/Search.java +++ b/application/src/main/java/com/yahoo/application/container/Search.java @@ -47,7 +47,6 @@ public final class Search extends ProcessingBase<Query, Result, Searcher> { Renderer<Result> renderer, ByteArrayOutputStream stream) throws IOException { Result result = process(chainSpec, request); - result.getTemplating().setRenderer(renderer); // TODO: Remove on Vespa 7 return HttpSearchResponse.waitableRender(result, result.getQuery(), renderer, stream); } diff --git a/application/src/main/java/com/yahoo/application/container/handler/Headers.java b/application/src/main/java/com/yahoo/application/container/handler/Headers.java index 8086960df25..ed3018eb430 100644 --- a/application/src/main/java/com/yahoo/application/container/handler/Headers.java +++ b/application/src/main/java/com/yahoo/application/container/handler/Headers.java @@ -155,7 +155,7 @@ public class Headers implements Map<String, List<String>> { /** * <p>Adds all the entries of the given map to this. This is the same as calling {@link #add(String, List)} for each - * entry in <tt>values</tt>.</p> + * entry in <code>values</code>.</p> * * @param values The values to be added to this. */ @@ -169,8 +169,8 @@ public class Headers implements Map<String, List<String>> { * * @param key The key of the entry to put. * @param value The value to put. - * @return The previous value associated with <tt>key</tt>, or <tt>null</tt> if there was no mapping for - * <tt>key</tt>. + * @return The previous value associated with <code>key</code>, or <code>null</code> if there was no mapping for + * <code>key</code>. */ public List<String> put(String key, String value) { return h.put(key, value); diff --git a/application/src/test/java/com/yahoo/application/container/JDiscContainerSearchTest.java b/application/src/test/java/com/yahoo/application/container/JDiscContainerSearchTest.java index 729439e7e5a..b7445d13a17 100644 --- a/application/src/test/java/com/yahoo/application/container/JDiscContainerSearchTest.java +++ b/application/src/test/java/com/yahoo/application/container/JDiscContainerSearchTest.java @@ -23,7 +23,7 @@ public class JDiscContainerSearchTest { try (JDisc container = containerWithSearch(searcherId)) { byte[] rendered = container.search().processAndRender(ComponentSpecification.fromString("mychain"), - ComponentSpecification.fromString("DefaultRenderer"), new Query("")); + ComponentSpecification.fromString("XmlRenderer"), new Query("")); String renderedAsString = new String(rendered, "utf-8"); assertThat(renderedAsString, containsString(searcherId)); } diff --git a/application/src/test/java/com/yahoo/application/container/JDiscTest.java b/application/src/test/java/com/yahoo/application/container/JDiscTest.java index 7963c2dd165..86a96d04848 100644 --- a/application/src/test/java/com/yahoo/application/container/JDiscTest.java +++ b/application/src/test/java/com/yahoo/application/container/JDiscTest.java @@ -152,7 +152,6 @@ public class JDiscTest { " field title type string {\n" + // " indexing: summary | index # How this field should be indexed\n" + // " weight: 75 # Ranking importancy of this field, used by the built in nativeRank feature\n" + // - " header\n" + // " }\n" + // "}\n"; diff --git a/application/src/test/resources/test.sd b/application/src/test/resources/test.sd index 85600ef9546..81d10391294 100644 --- a/application/src/test/resources/test.sd +++ b/application/src/test/resources/test.sd @@ -4,19 +4,15 @@ search test { document test { field title type string { - # index-to: title, default weight: 30 stemming: none bolding: on - body } field body type string { - # index-to: body, default weight: 6 stemming: none summary: dynamic - body } } diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml index 16e18e99e1b..3274599f70f 100644 --- a/athenz-identity-provider-service/pom.xml +++ b/athenz-identity-provider-service/pom.xml @@ -10,7 +10,7 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <dependencies> @@ -55,6 +55,8 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + + <!-- COMPILE --> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java index 94df93aaea7..1a7224fdc71 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java @@ -30,7 +30,6 @@ import java.security.PublicKey; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; -import java.util.Arrays; import java.util.Optional; import java.util.UUID; import java.util.concurrent.Executors; @@ -112,12 +111,6 @@ public class ConfigserverSslContextFactoryProvider extends AbstractComponent imp AthenzProviderServiceConfig.Zones zoneConfig) { SslContextFactory factory = new SslContextFactory(); - // Allow safe TLS_RSA* ciphers - String[] excludedCiphersWithoutTlsRsaExclusion = Arrays.stream(factory.getExcludeCipherSuites()) - .filter(cipher -> !cipher.equals("^TLS_RSA_.*$")) - .toArray(String[]::new); - factory.setExcludeCipherSuites(excludedCiphersWithoutTlsRsaExclusion); - factory.setWantClientAuth(true); KeyStore trustStore = diff --git a/bundle-plugin-test/pom.xml b/bundle-plugin-test/pom.xml index 4b36435529e..3ba9bf7b4b6 100644 --- a/bundle-plugin-test/pom.xml +++ b/bundle-plugin-test/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <groupId>com.yahoo.vespa</groupId> <artifactId>bundle-plugin-test</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> diff --git a/bundle-plugin-test/src/main/java/com/yahoo/test/SimpleSearcher2.java b/bundle-plugin-test/src/main/java/com/yahoo/test/SimpleSearcher2.java index c92f029ade6..3220171de13 100644 --- a/bundle-plugin-test/src/main/java/com/yahoo/test/SimpleSearcher2.java +++ b/bundle-plugin-test/src/main/java/com/yahoo/test/SimpleSearcher2.java @@ -3,10 +3,26 @@ package com.yahoo.test; /** - * A searcher adding a new hit. - * TODO: add annotation check - * @author Joe Developer + * Test class for integration tests. Used to verify that nested classes are visited. + * + * Do not replace the fully qualified class names with imports! */ public class SimpleSearcher2 extends SimpleSearcher { public void dummy() {} + + private class InnerClassProcessor extends com.yahoo.processing.Processor { + @Override + public com.yahoo.processing.Response process(com.yahoo.processing.Request request, + com.yahoo.processing.execution.Execution execution) { + return null; + } + } + + private static class NestedStaticClass { + private com.yahoo.metrics.simple.Counter counter; + + @com.google.inject.Inject + public NestedStaticClass() { } + } + } diff --git a/bundle-plugin-test/src/test/java/com/yahoo/BundleIT.java b/bundle-plugin-test/src/test/java/com/yahoo/BundleIT.java index 38ca08ecff1..5b4fdda06f7 100644 --- a/bundle-plugin-test/src/test/java/com/yahoo/BundleIT.java +++ b/bundle-plugin-test/src/test/java/com/yahoo/BundleIT.java @@ -4,7 +4,6 @@ package com.yahoo; import com.yahoo.osgi.maven.ProjectBundleClassPaths; import com.yahoo.vespa.config.VespaVersion; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import java.io.File; @@ -81,9 +80,15 @@ public class BundleIT { @Test public void require_that_manifest_contains_inferred_imports() { String importPackage = mainAttributes.getValue("Import-Package"); + + // From SimpleSearcher assertThat(importPackage, containsString("com.yahoo.prelude.hitfield")); - //Not available from jdisc at the moment, scope temporarily changed to compile. - //assertThat(importPackage, containsString("org.json")); + assertThat(importPackage, containsString("org.json")); + + // From SimpleSearcher2 + assertThat(importPackage, containsString("com.yahoo.processing")); + assertThat(importPackage, containsString("com.yahoo.metrics.simple")); + assertThat(importPackage, containsString("com.google.inject")); } @Test @@ -104,20 +109,22 @@ public class BundleIT { } @Test + // TODO: use another jar than jrt, which now pulls in a lot of dependencies that pollute the manifest of the + // generated bundle. (It's compile scoped in pom.xml to be added to the bundle-cp.) public void require_that_manifest_contains_bundle_class_path() { String bundleClassPath = mainAttributes.getValue("Bundle-ClassPath"); assertThat(bundleClassPath, containsString(".,")); // If bundle-plugin-test is compiled in a mvn command that also built jrt, // the jrt artifact is jrt.jar, otherwise the installed and versioned artifact - // is used: jrt-6-SNAPSHOT.jar. + // is used: jrt-7-SNAPSHOT.jar. assertThat(bundleClassPath, anyOf( - containsString("dependencies/jrt-6-SNAPSHOT.jar"), + containsString("dependencies/jrt-7-SNAPSHOT.jar"), containsString("dependencies/jrt.jar"))); } @Test public void require_that_component_jar_file_contains_compile_artifacts() { - ZipEntry versionedEntry = jarFile.getEntry("dependencies/jrt-6-SNAPSHOT.jar"); + ZipEntry versionedEntry = jarFile.getEntry("dependencies/jrt-7-SNAPSHOT.jar"); ZipEntry unversionedEntry = jarFile.getEntry("dependencies/jrt.jar"); if (versionedEntry == null) { assertNotNull(unversionedEntry); diff --git a/bundle-plugin/pom.xml b/bundle-plugin/pom.xml index a5cb3c9fbcc..e3fc62262d0 100644 --- a/bundle-plugin/pom.xml +++ b/bundle-plugin/pom.xml @@ -6,11 +6,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>bundle-plugin</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>maven-plugin</packaging> <name>${project.artifactId}</name> <description>Maven Plugin for creating OSGi bundles for the JDisc Container.</description> diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Analyze.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Analyze.java index c59f8559405..13a306a0c1c 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Analyze.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/Analyze.java @@ -58,7 +58,7 @@ public class Analyze { } static AnnotationVisitor visitAnnotationDefault(ImportCollector collector) { - return new AnnotationVisitor(Opcodes.ASM6) { + return new AnnotationVisitor(Opcodes.ASM7) { @Override public void visit(String name, Object value) { } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java index d9519fd7986..0225c43e4b8 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java @@ -28,7 +28,7 @@ class AnalyzeClassVisitor extends ClassVisitor implements ImportCollector { private Optional<ExportPackageAnnotation> exportPackageAnnotation = Optional.empty(); AnalyzeClassVisitor() { - super(Opcodes.ASM6); + super(Opcodes.ASM7); } @Override @@ -98,7 +98,7 @@ class AnalyzeClassVisitor extends ClassVisitor implements ImportCollector { } private AnnotationVisitor visitExportPackage() { - return new AnnotationVisitor(Opcodes.ASM6) { + return new AnnotationVisitor(Opcodes.ASM7) { private int major = defaultVersionValue("major"); private int minor = defaultVersionValue("minor"); private int micro = defaultVersionValue("micro"); diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeFieldVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeFieldVisitor.java index ea10b6ef0aa..1966c602479 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeFieldVisitor.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeFieldVisitor.java @@ -18,7 +18,7 @@ public class AnalyzeFieldVisitor extends FieldVisitor implements ImportCollector private final Set<String> imports = new HashSet<>(); public AnalyzeFieldVisitor(AnalyzeClassVisitor analyzeClassVisitor) { - super(Opcodes.ASM6); + super(Opcodes.ASM7); this.analyzeClassVisitor = analyzeClassVisitor; } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java index b1a92f9c10b..6a769b26295 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodVisitor.java @@ -24,7 +24,7 @@ class AnalyzeMethodVisitor extends MethodVisitor implements ImportCollector { private final AnalyzeClassVisitor analyzeClassVisitor; AnalyzeMethodVisitor(AnalyzeClassVisitor analyzeClassVisitor) { - super(Opcodes.ASM6); + super(Opcodes.ASM7); this.analyzeClassVisitor = analyzeClassVisitor; } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeSignatureVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeSignatureVisitor.java index 0f5fcf89f6a..c20d7d47de0 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeSignatureVisitor.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeSignatureVisitor.java @@ -18,7 +18,7 @@ class AnalyzeSignatureVisitor extends SignatureVisitor implements ImportCollecto private Set<String> imports = new HashSet<>(); AnalyzeSignatureVisitor(AnalyzeClassVisitor analyzeClassVisitor) { - super(Opcodes.ASM6); + super(Opcodes.ASM7); this.analyzeClassVisitor = analyzeClassVisitor; } diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateBundleClassPathMappingsMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateBundleClassPathMappingsMojo.java index b2abf13695f..4d5b0e327d5 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateBundleClassPathMappingsMojo.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateBundleClassPathMappingsMojo.java @@ -45,15 +45,15 @@ public class GenerateBundleClassPathMappingsMojo extends AbstractMojo { "bundleSymbolicName": "bundle-plugin-test", "classPathElements": [ "/Users/tonyv/Repos/vespa/bundle-plugin-test/target/classes", - "/Users/tonyv/.m2/repository/com/yahoo/vespa/jrt/6-SNAPSHOT/jrt-6-SNAPSHOT.jar", - "/Users/tonyv/.m2/repository/com/yahoo/vespa/annotations/6-SNAPSHOT/annotations-6-SNAPSHOT.jar" + "/Users/tonyv/.m2/repository/com/yahoo/vespa/jrt/7-SNAPSHOT/jrt-7-SNAPSHOT.jar", + "/Users/tonyv/.m2/repository/com/yahoo/vespa/annotations/7-SNAPSHOT/annotations-7-SNAPSHOT.jar" ] }, "providedDependencies": [ { "bundleSymbolicName": "jrt", "classPathElements": [ - "/Users/tonyv/.m2/repository/com/yahoo/vespa/jrt/6-SNAPSHOT/jrt-6-SNAPSHOT.jar" + "/Users/tonyv/.m2/repository/com/yahoo/vespa/jrt/7-SNAPSHOT/jrt-7-SNAPSHOT.jar" ] } ] diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java index 973854aa59a..9c9957ae718 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java @@ -65,7 +65,7 @@ public class GenerateOsgiManifestMojo extends AbstractMojo { @Parameter(alias = "Bundle-Version", defaultValue = "${project.version}") private String bundleVersion = null; - // TODO Vespa 7: default should be ${project.groupId}.${project.artifactId} + // TODO: default should include groupId, but that will require a lot of changes both by us and users. @Parameter(alias = "Bundle-SymbolicName", defaultValue = "${project.artifactId}") private String bundleSymbolicName = null; diff --git a/chain/pom.xml b/chain/pom.xml index cf82376b857..8f76598c51e 100755 --- a/chain/pom.xml +++ b/chain/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>chain</artifactId> <packaging>jar</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>com.google.code.findbugs</groupId> diff --git a/clustercontroller-apps/pom.xml b/clustercontroller-apps/pom.xml index ece2d66c8b7..acd9656db38 100644 --- a/clustercontroller-apps/pom.xml +++ b/clustercontroller-apps/pom.xml @@ -5,11 +5,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>clustercontroller-apps</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> diff --git a/clustercontroller-apputil/pom.xml b/clustercontroller-apputil/pom.xml index f1dbd57a275..20c0c0df8d2 100644 --- a/clustercontroller-apputil/pom.xml +++ b/clustercontroller-apputil/pom.xml @@ -5,11 +5,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>clustercontroller-apputil</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> diff --git a/clustercontroller-core/pom.xml b/clustercontroller-core/pom.xml index d10c8c8739b..6bfa308bb10 100644 --- a/clustercontroller-core/pom.xml +++ b/clustercontroller-core/pom.xml @@ -5,11 +5,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>clustercontroller-core</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/clustercontroller-standalone/pom.xml b/clustercontroller-standalone/pom.xml index 94b1b3d25c6..1f6daca62b2 100644 --- a/clustercontroller-standalone/pom.xml +++ b/clustercontroller-standalone/pom.xml @@ -5,11 +5,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>clustercontroller-standalone</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>jar</packaging> <dependencies> <dependency> diff --git a/clustercontroller-utils/pom.xml b/clustercontroller-utils/pom.xml index e176ce9d1e5..0fed0c67eb5 100644 --- a/clustercontroller-utils/pom.xml +++ b/clustercontroller-utils/pom.xml @@ -6,11 +6,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>clustercontroller-utils</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> diff --git a/component/abi-spec.json b/component/abi-spec.json index e71171c2a18..8e4442ea199 100644 --- a/component/abi-spec.json +++ b/component/abi-spec.json @@ -8,8 +8,8 @@ "public" ], "methods": [ - "protected void <init>(com.yahoo.component.ComponentId)", "protected void <init>()", + "protected void <init>(com.yahoo.component.ComponentId)", "public final void initId(com.yahoo.component.ComponentId)", "public final com.yahoo.component.ComponentId getId()", "public final boolean hasInitializedId()", @@ -201,35 +201,6 @@ "public static final com.yahoo.component.Version currentVersion" ] }, - "com.yahoo.component.provider.ComponentClass$ComponentConstructor": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.reflect.Constructor)", - "public java.lang.reflect.Constructor getConstructor()", - "public boolean preferredTo(com.yahoo.component.provider.ComponentClass$ComponentConstructor)" - ], - "fields": [ - "public final boolean isLegal", - "public final boolean hasComponentId" - ] - }, - "com.yahoo.component.provider.ComponentClass": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.Class)", - "public com.yahoo.component.AbstractComponent createComponent(com.yahoo.component.ComponentId, java.util.Map, java.lang.String)", - "public com.yahoo.component.provider.ComponentClass$ComponentConstructor getPreferredConstructor()" - ], - "fields": [] - }, "com.yahoo.component.provider.ComponentRegistry": { "superClass": "java.lang.Object", "interfaces": [], diff --git a/component/pom.xml b/component/pom.xml index a5a0ef3379b..c4fd5220d97 100755 --- a/component/pom.xml +++ b/component/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>component</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>junit</groupId> diff --git a/component/src/main/java/com/yahoo/component/AbstractComponent.java b/component/src/main/java/com/yahoo/component/AbstractComponent.java index 41d4c0dc6e9..7fc828becb5 100644 --- a/component/src/main/java/com/yahoo/component/AbstractComponent.java +++ b/component/src/main/java/com/yahoo/component/AbstractComponent.java @@ -6,8 +6,8 @@ import com.yahoo.collections.MethodCache; import java.lang.reflect.Method; /** - * Superclass of components. You must use this instead of subclassing Component if your component - * needs to be called on destruction. + * Superclass of destructible components. Container components to be created via dependency injection + * do not have to extend this class unless they need to implement {@link #deconstruct}. * * @author bratseth */ @@ -24,7 +24,17 @@ public class AbstractComponent implements Component { protected final boolean isDeconstructable; /** + * Creates a new component which is invalid until {@link #initId} is called on it. + * The dependency injection framework (DI) will always set the id, so components to be created + * via DI do not have to implement other constructors, and should not set the id themselves. + */ + protected AbstractComponent() { + isDeconstructable = setIsDeconstructable(); + } + + /** * Creates a new component with an id. + * Only for testing and components that are not created via dependency injection. * * @throws NullPointerException if the given id is null */ @@ -33,11 +43,6 @@ public class AbstractComponent implements Component { isDeconstructable = setIsDeconstructable(); } - /** Creates a new component which is invalid until {@link #initId} is called on it. */ - protected AbstractComponent() { - isDeconstructable = setIsDeconstructable(); - } - /** Initializes this. Always called from a constructor or the framework. Do not call. */ public final void initId(ComponentId id) { if (this.id != null && !this.id.equals(id)) diff --git a/component/src/main/java/com/yahoo/component/Component.java b/component/src/main/java/com/yahoo/component/Component.java index 9740ef9d490..c34044e279a 100644 --- a/component/src/main/java/com/yahoo/component/Component.java +++ b/component/src/main/java/com/yahoo/component/Component.java @@ -3,9 +3,10 @@ package com.yahoo.component; /** - * A named, versioned, identifiable component. - * Components can by default be ordered by their id order. Their identity is defined by the id. - * Prefer extending AbstractComponent instead of implementing this interface directly. + * <p>A named, versioned, identifiable component. + * Components can by default be ordered by their id order. Their identity is defined by the id.</p> + * + * <p>Container components to be created via dependency injection do not need to implement this interface.</p> * * @author bratseth */ diff --git a/component/src/main/java/com/yahoo/component/provider/ComponentClass.java b/component/src/main/java/com/yahoo/component/provider/ComponentClass.java deleted file mode 100644 index 25c27128786..00000000000 --- a/component/src/main/java/com/yahoo/component/provider/ComponentClass.java +++ /dev/null @@ -1,254 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.component.provider; - -import com.yahoo.component.AbstractComponent; -import com.yahoo.component.ComponentId; -import com.yahoo.config.ConfigInstance; -import com.yahoo.vespa.config.ConfigKey; - -import java.lang.annotation.Annotation; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Type; -import java.util.*; -import java.util.logging.Logger; - -/** - * Encapsulates the class of a component to be created, along with the constructor that will be used. - * - * @author gjoranv - * @author bratseth - */ -public class ComponentClass<T extends AbstractComponent> { - - private static Logger log = Logger.getLogger(ComponentClass.class.getName()); - - private final Class<T> clazz; - private final ComponentConstructor<T> constructor; - - public ComponentClass(Class<T> clazz) { - this.clazz = clazz; - constructor = findPreferredConstructor(); - if (! constructor.isLegal) { - throw new IllegalArgumentException("Class '" + clazz.getName() + "' must have at least one public " + - "constructor with an optional component ID followed by an optional FileAcquirer and " + - "zero or more config arguments: " + - clazz.getSimpleName() + "([ComponentId] [ConfigInstance ...])"); - } - } - - /** - * Create an instance of this ComponentClass with the given configId. The configs needed by the component - * must exist in the provided set of {@link com.yahoo.config.ConfigInstance}s. - * - * @param id The id of the component to create, never null. - * @param availableConfigs The set of available config instances. - * @param configId The config ID of the component, nullable. - * @return A new instance of the class represented by this ComponentClass. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public T createComponent(ComponentId id, Map<ConfigKey, ConfigInstance> availableConfigs, String configId) { - if (configId == null) { - configId = System.getProperty("config.id"); - } - - boolean hasId = false; - List<Object> params = new LinkedList<>(); - for (Class cc : constructor.parameters) { - if (cc.equals(ComponentId.class)) { - params.add(id); - hasId = true; - } else if (cc.getSuperclass().equals(ConfigInstance.class)) { - ConfigKey key = new ConfigKey(cc, configId); - if ((availableConfigs == null) || ! availableConfigs.containsKey(key)) { - throw new IllegalStateException - ("Could not resolve config instance '" + key + "' required to instantiate " + clazz); - } - params.add(availableConfigs.get(key)); - } - } - T component = construct(params.toArray()); - - if (hasId && component.hasInitializedId() && !id.equals(component.getId())) { - log.warning("Component with id '" + id + "' tried to set illegal component id: '" + component.getId() + - "', or the component takes ComponentId as a constructor arg without calling super(id)."); - } - // Enforce correct id - see bug #4036397 - component.initId(id); - - return component; - } - - public ComponentConstructor<T> getPreferredConstructor() { - return constructor; - } - - /** - * Creates an instance of this class. Due to the error-prone Object varargs, this method must be used with - * caution, and never from outside this class. - * - * @param arguments the arguments to the constructor - * @return The new instance. - * @throws RuntimeException if construction fails for some reason - */ - private T construct(Object... arguments) { - String args = Arrays.toString(arguments); - try { - return constructor.getConstructor().newInstance(arguments); - } catch (InstantiationException e) { - throw new RuntimeException("Exception while instantiating " + clazz + " from " + args,e); - } catch (IllegalAccessException e) { - throw new RuntimeException("Could not access " + constructor + " of " + clazz); - } catch (InvocationTargetException e) { - throw new RuntimeException("Exception while executing constructor of " + clazz + " with " + args,e); - } catch (IllegalArgumentException e) { - log.warning(clazz.getName() + " expected ctor arguments:"); - for (@SuppressWarnings("rawtypes") Class expectedArg : constructor.getConstructor().getParameterTypes()) - log.warning(" " + expectedArg + " - " + System.identityHashCode(expectedArg)); - - log.warning(clazz.getName() + " actual ctor arguments: "); - for (Object actualArg : arguments) - log.warning(" " + actualArg.getClass() + " - " + System.identityHashCode(actualArg.getClass())); - throw new RuntimeException("Exception while executing constructor of " + clazz + " with " + args,e); - } - - } - - /** - * Returns the preferred constructor of the given class, or null if no satisfactory constructor is present. - * The preferred constructor is always the one with the most arguments of type T extends ConfigInstance. - * - * @return The preferred constructor. - */ - @SuppressWarnings("unchecked") - private ComponentConstructor<T> findPreferredConstructor() { - @SuppressWarnings("rawtypes") - Constructor[] constructors = clazz.getConstructors(); - if (constructors.length < 1) { - throw new RuntimeException("Class has no public constructors: " + clazz.getName()); - } - ComponentConstructor<T> best = new ComponentConstructor<T>(constructors[0]); - for (int i = 1; i < constructors.length; i++) { - Constructor<T> c = constructors[i]; - ComponentConstructor<T> cc = new ComponentConstructor<>(c); - if (cc.preferredTo(best)) { - best = cc; - } - } - return best; - } - - /** - * Encapsulates a constructor for a ComponentClass. Immutable. - */ - public static class ComponentConstructor<T> { - - // The legal argument classes (except '? extends ConfigInstance' of course) - @SuppressWarnings("rawtypes") - private static final Set<Class> legalArgs = Collections.singleton((Class) ComponentId.class); - - private final Constructor<T> constructor; - - @SuppressWarnings("rawtypes") - private final Class[] parameters; - private final List<Class<? extends ConfigInstance>> configArgs; - - public final boolean isLegal; - public final boolean hasComponentId; - - public ComponentConstructor(Constructor<T> c) { - constructor = c; - parameters = c.getParameterTypes(); - - isLegal = isLegal(parameters); - hasComponentId = hasComponentId(parameters); - configArgs = findConfigArgs(parameters); - } - - public Constructor<T> getConstructor() { - return constructor; - } - - /** - * Returns true if this constructor is preferred to the other, or if they are equivalent. - * False otherwise. - * @param other The other constructor. - * @return true if this constructor is preferred to the other, false otherwise. - */ - public boolean preferredTo(ComponentConstructor<T> other) { - if (this.isLegal && ! other.isLegal) - return true; - else if (! this.isLegal && other.isLegal) - return false; - - // Both are legal - if (this.parameters.length > other.parameters.length) - return true; - else if (this.parameters.length < other.parameters.length) - return false; - - // Equal number of args - if (this.configArgs.size() > other.configArgs.size()) - return true; - else if (this.configArgs.size() < other.configArgs.size()) - return false; - - // Equal number of args and config args, prefer ComponentId - if (this.hasComponentId && ! other.hasComponentId) - return true; - else if (! this.hasComponentId && other.hasComponentId) - return false; - - // Equivalent - return true; - } - - @SuppressWarnings("rawtypes") - private static boolean isLegal(Class[] args) { - Set<Class> used = new HashSet<>(); - for (Class cl : args) { - if (legalArgs.contains(cl)) { - if (used.contains(cl)) { - return false; - } - if (cl.equals(String.class) || cl.equals(ComponentId.class)) { - // Only one of these are allowed, so mark both as used. - used.add(String.class); - used.add(ComponentId.class); - } else { - used.add(cl); - } - } else { - // Must be a config arg - Class superclass = cl.getSuperclass(); - if ((superclass == null) || !superclass.equals(com.yahoo.config.ConfigInstance.class)) - return false; - } - } - return true; - } - - @SuppressWarnings("rawtypes") - private static boolean hasComponentId(Class[] args) { - for (Class cl : args) { - if (cl.equals(ComponentId.class)) - return true; - } - return false; - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - private static List<Class<? extends ConfigInstance>> findConfigArgs(Class[] args) { - List<Class<? extends ConfigInstance>> configs = new ArrayList<>(); - for (Class cl : args) { - Class superclass = cl.getSuperclass(); - if ((superclass != null) && superclass.equals(ConfigInstance.class)) { - configs.add(cl); - } - } - return configs; - } - - } // class ComponentConstructor - -} diff --git a/config-application-package/pom.xml b/config-application-package/pom.xml index 416e59d3633..2cce4c01243 100644 --- a/config-application-package/pom.xml +++ b/config-application-package/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-application-package</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 5ec30f71e7b..b2a1507b424 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -609,13 +609,11 @@ public class FilesApplicationPackage implements ApplicationPackage { /** * Reads a ranking expression from file to a string and returns it. * - * @param name the name of the file to return, either absolute or + * @param name the name of the file to return, * relative to the search definition directory in the application package * @return the content of a ranking expression file * @throws IllegalArgumentException if the file was not found or could not be read */ - // TODO: A note on absolute paths: We don't want to support this and it should be removed on 6.0 - // Currently one system test (basicmlr) depends on it. @Override public Reader getRankingExpression(String name) { try { @@ -628,8 +626,9 @@ public class FilesApplicationPackage implements ApplicationPackage { private File expressionFileNameToFile(String name) { File expressionFile = new File(name); - if (expressionFile.isAbsolute()) return expressionFile; - + if (expressionFile.isAbsolute()) { + throw new IllegalArgumentException("Absolute path to ranking expression file is not allowed: " + name); + } File sdDir = new File(appDir, ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()); return new File(sdDir, name); } diff --git a/config-bundle/pom.xml b/config-bundle/pom.xml index 61d1065bee4..2147798791f 100644 --- a/config-bundle/pom.xml +++ b/config-bundle/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-bundle</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/config-class-plugin/pom.xml b/config-class-plugin/pom.xml index b7a09d8c7c7..e0ea70bd4f0 100644 --- a/config-class-plugin/pom.xml +++ b/config-class-plugin/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-class-plugin</artifactId> <packaging>maven-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>Vespa ConfigGen Plugin</name> <dependencies> <dependency> diff --git a/config-lib/abi-spec.json b/config-lib/abi-spec.json index 1497d0ab15d..0a6a673e39e 100644 --- a/config-lib/abi-spec.json +++ b/config-lib/abi-spec.json @@ -213,8 +213,7 @@ "public" ], "methods": [ - "public void <init>(java.util.List)", - "public void <init>(java.util.List, com.yahoo.config.InnerNode)" + "public void <init>(java.util.List)" ], "fields": [] }, diff --git a/config-lib/pom.xml b/config-lib/pom.xml index bad146a848a..861681291fe 100644 --- a/config-lib/pom.xml +++ b/config-lib/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-lib</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>${project.artifactId}</name> <dependencies> <dependency> diff --git a/config-lib/src/main/java/com/yahoo/config/InnerNodeVector.java b/config-lib/src/main/java/com/yahoo/config/InnerNodeVector.java index 2b12de8c3fc..ec773b91ac4 100644 --- a/config-lib/src/main/java/com/yahoo/config/InnerNodeVector.java +++ b/config-lib/src/main/java/com/yahoo/config/InnerNodeVector.java @@ -13,9 +13,4 @@ public class InnerNodeVector<NODE extends InnerNode> extends NodeVector<NODE> { vector.addAll(nodes); } - // TODO: Remove for Vespa 7 - public InnerNodeVector(List<NODE> nodes, NODE defaultNode) { - this(nodes); - } - } diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 5c5ed0a5ece..c932958b58b 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -115,20 +115,16 @@ "public abstract java.lang.String getServicesSource()", "public abstract java.util.Optional getDeployment()", "public abstract java.util.Optional getValidationOverrides()", - "public java.util.List getComponentsInfo(com.yahoo.config.provision.Version)", - "public java.util.List getComponentsInfo(com.yahoo.component.Version)", + "public abstract java.util.List getComponentsInfo(com.yahoo.component.Version)", "public abstract java.io.Reader getRankingExpression(java.lang.String)", "public static java.util.Map getBundleSdFiles(java.lang.String, java.util.jar.JarFile)", "public static java.lang.String getFileName(java.util.jar.JarEntry)", "public abstract com.yahoo.config.application.api.ApplicationMetaData getMetaData()", "public abstract java.io.File getFileReference(com.yahoo.path.Path)", "public void validateXML()", - "public void validateXML(java.util.Optional)", "public void validateXMLFor(java.util.Optional)", "public void writeMetaData()", - "public java.util.Map getProvisionInfoMap()", "public java.util.Optional getAllocatedHosts()", - "public java.util.Map getFileRegistryMap()", "public java.util.Map getFileRegistries()", "public abstract java.util.Collection getSearchDefinitions()", "public com.yahoo.config.application.api.ApplicationPackage preprocess(com.yahoo.config.provision.Zone, com.yahoo.config.application.api.DeployLogger)" diff --git a/config-model-api/pom.xml b/config-model-api/pom.xml index 21c3b6f2752..6ca33bed833 100644 --- a/config-model-api/pom.xml +++ b/config-model-api/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-model-api</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index 6b49ea0c172..06f8034453d 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -185,17 +185,7 @@ public interface ApplicationPackage { Optional<Reader> getDeployment(); Optional<Reader> getValidationOverrides(); - /** @deprecated do not override or call. Use the other Version class */ - @Deprecated - default List<ComponentInfo> getComponentsInfo(com.yahoo.config.provision.Version vespaVersion) { - return getComponentsInfo(vespaVersion.toVersion()); - } - - // TODO: Remove the default implementation after December 2018 - @SuppressWarnings("deprecation") - default List<ComponentInfo> getComponentsInfo(Version vespaVersion) { - return getComponentsInfo(com.yahoo.config.provision.Version.from(vespaVersion)); - } + List<ComponentInfo> getComponentsInfo(Version vespaVersion); /** * Reads a ranking expression from file to a string and returns it. @@ -248,12 +238,6 @@ public interface ApplicationPackage { throw new UnsupportedOperationException("This application package cannot validate XML"); } - /** @deprecated do not override or call. Use the other Version class */ - @Deprecated - default void validateXML(Optional<com.yahoo.config.provision.Version> vespaVersion) throws IOException { - validateXMLFor(vespaVersion.map(com.yahoo.config.provision.Version::toVersion)); - } - default void validateXMLFor(Optional<Version> vespaVersion) throws IOException { throw new UnsupportedOperationException("This application package cannot validate XML"); } @@ -262,31 +246,11 @@ public interface ApplicationPackage { throw new UnsupportedOperationException("This application package cannot write its metadata"); } - /** - * Returns the single host allocation info of this, or an empty map if no allocation is available - * - * @deprecated please use #getAllocatedHosts - */ - // TODO: Remove on Vespa 7 - @Deprecated - default Map<com.yahoo.config.provision.Version, AllocatedHosts> getProvisionInfoMap() { - return Collections.emptyMap(); - } - /** Returns the host allocation info of this, or empty if no allocation is available */ default Optional<AllocatedHosts> getAllocatedHosts() { return Optional.empty(); } - /** @deprecated do not override or call. Use getFileRegistries */ - @Deprecated - default Map<com.yahoo.config.provision.Version, FileRegistry> getFileRegistryMap() { - return getFileRegistries().entrySet() - .stream() - .collect(Collectors.toMap(e -> com.yahoo.config.provision.Version.from(e.getKey()), - e -> e.getValue())); - } - default Map<Version, FileRegistry> getFileRegistries() { return Collections.emptyMap(); } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index 084779d2147..e70e8449922 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -219,13 +219,10 @@ public class DeploymentSpecXmlReader { private List<DeploymentSpec.ChangeBlocker> readChangeBlockers(Element root) { List<DeploymentSpec.ChangeBlocker> changeBlockers = new ArrayList<>(); for (Element tag : XML.getChildren(root)) { - // TODO: Remove block-upgrade on Vespa 7 - if ( ! blockChangeTag.equals(tag.getTagName()) && !"block-upgrade".equals(tag.getTagName())) continue; + if ( ! blockChangeTag.equals(tag.getTagName())) continue; - boolean blockVersions = tag.getTagName().equals("block-upgrade") || // TODO: Remove condition on Vespa 7 - trueOrMissing(tag.getAttribute("version")); - boolean blockRevisions = trueOrMissing(tag.getAttribute("revision")) - && !tag.getTagName().equals("block-upgrade"); // TODO: Remove condition on Vespa 7 + boolean blockVersions = trueOrMissing(tag.getAttribute("version")); + boolean blockRevisions = trueOrMissing(tag.getAttribute("revision")); String daySpec = tag.getAttribute("days"); String hourSpec = tag.getAttribute("hours"); diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelFactory.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelFactory.java index 23782162725..0c4e5087d62 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelFactory.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelFactory.java @@ -13,14 +13,7 @@ public interface ModelFactory { * * @return the version of a {@link Model} instance that this factory can create. */ - @SuppressWarnings("deprecation") - default Version version() { // TODO: Remove this default implementationm after December 2018 - return getVersion().toVersion(); - } - - /** @deprecated use and override version(). TODO: Remove this method after December 2018 */ - @Deprecated - default com.yahoo.config.provision.Version getVersion() { return com.yahoo.config.provision.Version.from(version()); } + Version version(); /** * Creates an instance of a {@link Model}. The resulting instance will be used to serve config. No model diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index b40feb3081a..4156439834b 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -296,38 +296,6 @@ public class DeploymentSpecTest { } } - @Test - public void deploymentSpecWithBlockUpgrade() { - StringReader r = new StringReader( - "<deployment>\n" + - " <block-upgrade days='mon,tue' hours='15-16'/>\n" + - // version=false is ignored for block-upgrade - " <block-upgrade version='false' days='sat' hours='10' time-zone='CET'/>\n" + - " <prod>\n" + - " <region active='true'>us-west-1</region>\n" + - " </prod>\n" + - "</deployment>" - ); - DeploymentSpec spec = DeploymentSpec.fromXml(r); - assertEquals(2, spec.changeBlocker().size()); - assertTrue(spec.changeBlocker().get(0).blocksVersions()); - assertFalse(spec.changeBlocker().get(0).blocksRevisions()); - assertEquals(ZoneId.of("UTC"), spec.changeBlocker().get(0).window().zone()); - - assertTrue(spec.changeBlocker().get(1).blocksVersions()); - assertFalse(spec.changeBlocker().get(1).blocksRevisions()); - assertEquals(ZoneId.of("CET"), spec.changeBlocker().get(1).window().zone()); - - assertTrue(spec.canUpgradeAt(Instant.parse("2017-09-18T14:15:30.00Z"))); - assertFalse(spec.canUpgradeAt(Instant.parse("2017-09-18T15:15:30.00Z"))); - assertFalse(spec.canUpgradeAt(Instant.parse("2017-09-18T16:15:30.00Z"))); - assertTrue(spec.canUpgradeAt(Instant.parse("2017-09-18T17:15:30.00Z"))); - - assertTrue(spec.canUpgradeAt(Instant.parse("2017-09-23T09:15:30.00Z"))); - assertFalse(spec.canUpgradeAt(Instant.parse("2017-09-23T08:15:30.00Z"))); // 10 in CET - assertTrue(spec.canUpgradeAt(Instant.parse("2017-09-23T10:15:30.00Z"))); - } - @Test(expected = IllegalArgumentException.class) public void deploymentSpecWithIllegallyOrderedDeploymentSpec1() { StringReader r = new StringReader( diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index ef0a92bb5e9..97898adedf1 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-model-fat</artifactId> <packaging>bundle</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/config-model/pom.xml b/config-model/pom.xml index 57dbadd94ab..965d97bb605 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-model</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/config-model/src/main/.gitignore b/config-model/src/main/.gitignore new file mode 100644 index 00000000000..24600083db4 --- /dev/null +++ b/config-model/src/main/.gitignore @@ -0,0 +1 @@ +!Makefile diff --git a/config-model/src/main/Makefile b/config-model/src/main/Makefile index b1c37f4889f..f63f7dd832f 100644 --- a/config-model/src/main/Makefile +++ b/config-model/src/main/Makefile @@ -1,42 +1,46 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#all: services.xsd hosts.xsd + +srcdir=resources/schema +outputdir=../../target/generated-sources/trang/resources/schema trangjar=../../target/trang.jar -all: resources/schema/services.rng resources/schema/hosts.rng resources/schema/container-include.rng resources/schema/services.xsd resources/schema/hosts.xsd resources/schema/container-include.xsd resources/schema/deployment.xsd +all: ${outputdir} ${outputdir}/services.rng ${outputdir}/hosts.rng ${outputdir}/container-include.rng ${outputdir}/services.xsd ${outputdir}/hosts.xsd ${outputdir}/container-include.xsd ${outputdir}/deployment.xsd + +${outputdir}: + mkdir -p ${outputdir} -resources/schema/services.rng: resources/schema/services.rnc resources/schema/common.rnc resources/schema/admin.rnc resources/schema/clients.rnc resources/schema/docproc.rnc resources/schema/routing.rnc resources/schema/clients-v2.rnc resources/schema/content.rnc resources/schema/genericmodule.rnc resources/schema/legacygenericcluster.rnc resources/schema/genericcluster.rnc resources/schema/legacygenericmodule.rnc resources/schema/containercluster.rnc - java -jar $(trangjar) -I rnc -O rng resources/schema/services.rnc resources/schema/services.rng +${outputdir}/services.rng: ${srcdir}/services.rnc ${srcdir}/common.rnc ${srcdir}/admin.rnc ${srcdir}/clients.rnc ${srcdir}/docproc.rnc ${srcdir}/routing.rnc ${srcdir}/clients-v2.rnc ${srcdir}/content.rnc ${srcdir}/genericmodule.rnc ${srcdir}/legacygenericcluster.rnc ${srcdir}/genericcluster.rnc ${srcdir}/legacygenericmodule.rnc ${srcdir}/containercluster.rnc + java -jar $(trangjar) -I rnc -O rng ${srcdir}/services.rnc ${outputdir}/services.rng -resources/schema/services.xsd: resources/schema/services.rng - java -jar $(trangjar) -I rng -O xsd resources/schema/services.rng resources/schema/services.xsd +${outputdir}/services.xsd: ${outputdir}/services.rng + java -jar $(trangjar) -I rng -O xsd ${outputdir}/services.rng ${outputdir}/services.xsd -resources/schema/container-include.rng: resources/schema/common.rnc resources/schema/containercluster.rnc - java -jar $(trangjar) -I rnc -O rng resources/schema/container-include.rnc resources/schema/container-include.rng +${outputdir}/container-include.rng: ${srcdir}/common.rnc ${srcdir}/containercluster.rnc + java -jar $(trangjar) -I rnc -O rng ${srcdir}/container-include.rnc ${outputdir}/container-include.rng -resources/schema/container-include.xsd: resources/schema/container-include.rng - java -jar $(trangjar) -I rng -O xsd resources/schema/container-include.rng resources/schema/container-include.xsd +${outputdir}/container-include.xsd: ${outputdir}/container-include.rng + java -jar $(trangjar) -I rng -O xsd ${outputdir}/container-include.rng ${outputdir}/container-include.xsd -resources/schema/hosts.rng: resources/schema/hosts.rnc - java -jar $(trangjar) -I rnc -O rng resources/schema/hosts.rnc resources/schema/hosts.rng +${outputdir}/hosts.rng: ${srcdir}/hosts.rnc + java -jar $(trangjar) -I rnc -O rng ${srcdir}/hosts.rnc ${outputdir}/hosts.rng -resources/schema/hosts.xsd: resources/schema/hosts.rng - java -jar $(trangjar) -I rng -O xsd resources/schema/hosts.rng resources/schema/hosts.xsd +${outputdir}/hosts.xsd: ${outputdir}/hosts.rng + java -jar $(trangjar) -I rng -O xsd ${outputdir}/hosts.rng ${outputdir}/hosts.xsd -resources/schema/deployment.rng: resources/schema/deployment.rnc - java -jar $(trangjar) -I rnc -O rng resources/schema/deployment.rnc resources/schema/deployment.rng +${outputdir}/deployment.rng: ${srcdir}/deployment.rnc + java -jar $(trangjar) -I rnc -O rng ${srcdir}/deployment.rnc ${outputdir}/deployment.rng -resources/schema/deployment.xsd: resources/schema/deployment.rng - java -jar $(trangjar) -I rng -O xsd resources/schema/deployment.rng resources/schema/deployment.xsd +${outputdir}/deployment.xsd: ${outputdir}/deployment.rng + java -jar $(trangjar) -I rng -O xsd ${outputdir}/deployment.rng ${outputdir}/deployment.xsd -resources/schema/validation-overrides.rng: resources/schema/validation-overrides.rnc - java -jar $(trangjar) -I rnc -O rng resources/schema/validation-overrides.rnc resources/schema/validation-overrides.rng +${outputdir}/validation-overrides.rng: ${srcdir}/validation-overrides.rnc + java -jar $(trangjar) -I rnc -O rng ${srcdir}/validation-overrides.rnc ${outputdir}/validation-overrides.rng -resources/schema/deployment.xsd: resources/schema/validation-overrides.rng - java -jar $(trangjar) -I rng -O xsd resources/schema/validation-overrides.rng resources/schema/validation-overrides.xsd +${outputdir}/deployment.xsd: ${outputdir}/validation-overrides.rng + java -jar $(trangjar) -I rng -O xsd ${outputdir}/validation-overrides.rng ${outputdir}/validation-overrides.xsd clean: - rm -f resources/schema/*.rng - rm -f resources/schema/*.xsd + rm -rf ${outputdir} diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java index 0b638c14485..93ff88f182b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java @@ -125,8 +125,13 @@ public class ConfigModelRepo implements ConfigModelRepoAdder, Serializable, Iter for (Element servicesElement : children) { String tagName = servicesElement.getTagName(); - if (tagName.equals("config")) continue; // TODO: Remove on Vespa 6 - if (tagName.equals("cluster")) continue; // TODO: Remove on Vespa 6 + if (tagName.equals("config")) { + // TODO: disallow on Vespa 8 + continue; + } + if (tagName.equals("cluster")) { + throw new IllegalArgumentException("<" + tagName + "> on top-level is not allowed anymore"); + } if ((tagName.equals("clients")) && deployState.isHosted()) throw new IllegalArgumentException("<" + tagName + "> is not allowed when running Vespa in a hosted environment"); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java index 59f97a7e535..1620c90acd1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java @@ -51,9 +51,6 @@ public class Index implements Cloneable, Serializable { /** Whether the content of this index is normalized */ private boolean normalized=true; - /** The set of all searchable fields which should be searched with this index. May not null */ - public Set<String> matchGroup=new LinkedHashSet<>(); - private Type type = Type.VESPA; /** The boolean index definition, if set */ @@ -82,22 +79,6 @@ public class Index implements Cloneable, Serializable { public Stemming getStemming() { return stemming; } /** - * Returns the (unmodifiable) set of searchable fields which should be searched - * when this index is searched. This is useful to specify that some attributes should be - * searched as well when an index is searched. - * This set is either empty, or if set contains both the name of this index, and some other - * indexes. - */ - public Set<String> getMatchGroup() { return Collections.unmodifiableSet(matchGroup); } - - /** Adds a searchable field name to be searched when this index is searched */ - public void addToMatchGroup(String name) { - if (name.equals(this.name)) return; - if (matchGroup.size()==0) matchGroup.add(this.name); - matchGroup.add(name); - } - - /** * Whether this field should be stemmed in this search definition, * this is never null */ diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java index ba2421bf5bb..a99300cfd32 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java @@ -64,9 +64,8 @@ public class Search implements Serializable, ImmutableSearch { // True if this doesn't define a search, just some documents. private boolean documentsOnly = false; - // The stemming setting of this search definition. Default is SHORTEST. - // TODO: Change to Stemming.BEST on Vespa 7 - private Stemming stemming = Stemming.SHORTEST; + // The stemming setting of this search definition. Default is BEST. + private Stemming stemming = Stemming.BEST; // Documents contained in this definition. private SDDocumentType docType; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index cf88886029f..bd4daa58253 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -319,9 +319,13 @@ public class SearchBuilder { * @throws ParseException Thrown if there was a problem parsing the string. */ public static SearchBuilder createFromString(String sd) throws ParseException { + return createFromString(sd, new BaseDeployLogger()); + } + + public static SearchBuilder createFromString(String sd, DeployLogger logger) throws ParseException { SearchBuilder builder = new SearchBuilder(MockApplicationPackage.createEmpty()); - builder.importString(sd); - builder.build(); + builder.importString(sd, logger); + builder.build(true, logger); return builder; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java index 1e4a4a2a5d2..a807cc79d95 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java @@ -27,7 +27,6 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { private static final String CMD_HIGHLIGHT = "highlight"; private static final String CMD_INDEX = "index"; private static final String CMD_LOWERCASE = "lowercase"; - private static final String CMD_MATCH_GROUP = "match-group "; private static final String CMD_NORMALIZE = "normalize"; private static final String CMD_STEM = "stem"; private static final String CMD_URLHOST = "urlhost"; @@ -76,24 +75,6 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { } } - @Override - protected void derive(Index index, Search search) { - if (index.getMatchGroup().size() > 0) { - addIndexCommand(index.getName(), CMD_MATCH_GROUP + toSpaceSeparated(index.getMatchGroup())); - } - } - - private String toSpaceSeparated(Collection c) { - StringBuffer b = new StringBuffer(); - for (Iterator i = c.iterator(); i.hasNext();) { - b.append(i.next()); - if (i.hasNext()) { - b.append(" "); - } - } - return b.toString(); - } - private static boolean isPositionArrayField(ImmutableSDField field) { return field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE)); } @@ -442,8 +423,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { if (active != null) { return active; } - // assume default: TODO: Change to Stemming.BEST on Vespa 7 - return Stemming.SHORTEST; + return Stemming.BEST; // assume default } private boolean stemming(ImmutableSDField field) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java index 39d51414607..f8766afbc7b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java @@ -114,9 +114,6 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { .prefix(f.hasPrefix()) .phrases(f.hasPhrases()) .positions(f.hasPositions()); - if (f.getSdType() !=null && !f.getSdType().equals(com.yahoo.searchdefinition.Index.Type.VESPA)) { - ifB.indextype(IndexschemaConfig.Indexfield.Indextype.Enum.valueOf(f.getSdType().toString())); - } if (!f.getCollectionType().equals("SINGLE")) { ifB.collectiontype(IndexschemaConfig.Indexfield.Collectiontype.Enum.valueOf(f.getCollectionType())); } @@ -194,10 +191,7 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { public Index.Type getRawType() { return type; } public String getType() { return type.equals(Index.Type.INT64) - ? "INT64" - : type.equals(Index.Type.BOOLEANTREE) - ? "BOOLEANTREE" - : "STRING"; + ? "INT64" : "STRING"; } public String getCollectionType() { return (sdFieldType == null) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java index c7b698f5835..692fe7666d4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java @@ -260,9 +260,9 @@ public class SDDocumentType implements Cloneable, Serializable { } } } - } + } - public SDField addField(String string, DataType dataType) { + public SDField addField(String string, DataType dataType) { SDField field = new SDField(this, string, dataType); addField(field); return field; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index 049f5392c04..8b523211471 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -83,9 +83,6 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, /** The explicitly index settings on this field */ private Map<String, Index> indices = new java.util.LinkedHashMap<>(); - /** True if body or header is set explicitly for this field */ - private boolean headerOrBodyDefined = false; - private boolean idOverride = false; /** Struct fields defined in this field */ @@ -426,7 +423,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, /** Parse an indexing expression which will use the simple linguistics implementatino suitable for testing */ @SuppressWarnings("deprecation") public void parseIndexingScript(String script) { - parseIndexingScript(script, new SimpleLinguistics(false)); + parseIndexingScript(script, new SimpleLinguistics()); } public void parseIndexingScript(String script, Linguistics linguistics) { @@ -769,15 +766,6 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, return queryCommands; } - - public boolean isHeaderOrBodyDefined() { - return headerOrBodyDefined; - } - - public void setHeaderOrBodyDefined(boolean headerOrBodySetExplicitly) { - this.headerOrBodyDefined = headerOrBodySetExplicitly; - } - /** * The document that this field was declared in, or null * diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java index b5a0ecfbe3d..8a7ee78ea91 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java @@ -17,10 +17,6 @@ public enum Stemming { /** No stemming */ NONE("none"), - /** @deprecated incorrectly don't stem at all */ - @Deprecated - ALL("all"), - /** select shortest possible stem */ SHORTEST("shortest"), @@ -43,12 +39,7 @@ public enum Stemming { @SuppressWarnings("deprecation") public static Stemming get(String stemmingName) { try { - Stemming stemming = Stemming.valueOf(stemmingName.toUpperCase()); - if (stemming.equals(ALL)) { - log.warning("note: stemming ALL is the same as stemming mode SHORTEST"); - stemming = SHORTEST; - } - return stemming; + return Stemming.valueOf(stemmingName.toUpperCase()); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("'" + stemmingName + "' is not a valid stemming setting"); } @@ -71,7 +62,6 @@ public enum Stemming { case MULTIPLE: return StemMode.ALL; case BEST : return StemMode.BEST; case NONE: return StemMode.NONE; - case ALL: return StemMode.SHORTEST; // Intentional; preserve historic behavior default: throw new IllegalStateException("Inconvertible stem mode " + this); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java index 4df3660a967..861a9f530d4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java @@ -17,7 +17,6 @@ public class AttributeOperation implements FieldOperation, FieldOperationContain private Boolean fastSearch; private Boolean fastAccess; private Boolean mutable; - private Boolean prefetch; private Boolean enableBitVectors; private Boolean enableOnlyBitVector; //TODO: Husk sorting!! @@ -73,14 +72,6 @@ public class AttributeOperation implements FieldOperation, FieldOperationContain this.mutable = mutable; } - public Boolean getPrefetch() { - return prefetch; - } - - public void setPrefetch(Boolean prefetch) { - this.prefetch = prefetch; - } - public Boolean getEnableBitVectors() { return enableBitVectors; } @@ -150,9 +141,6 @@ public class AttributeOperation implements FieldOperation, FieldOperationContain if (mutable != null) { attribute.setMutable(mutable); } - if (prefetch != null) { - attribute.setPrefetch(prefetch); - } if (enableBitVectors != null) { attribute.setEnableBitVectors(enableBitVectors); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java index d1dc68373db..dece0064fcc 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java @@ -29,7 +29,7 @@ public class IndexingOperation implements FieldOperation { /** Creates an indexing operation which will use the simple linguistics implementation suitable for testing */ @SuppressWarnings("deprecation") public static IndexingOperation fromStream(SimpleCharStream input, boolean multiLine) throws ParseException { - return fromStream(input, multiLine, new SimpleLinguistics(false)); + return fromStream(input, multiLine, new SimpleLinguistics()); } public static IndexingOperation fromStream(SimpleCharStream input, boolean multiLine, Linguistics linguistics) @@ -51,4 +51,5 @@ public class IndexingOperation implements FieldOperation { } return new IndexingOperation(exp); } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java deleted file mode 100644 index b34db6febd5..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Search; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -public class DeprecateAttributePrefetch extends Processor { - - public DeprecateAttributePrefetch(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(search, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - - for (SDField field : search.allConcreteFields()) { - for (Attribute a : field.getAttributes().values()) { - if (Boolean.TRUE.equals(a.getPrefetchValue())) { - warn(search, field, "Attribute prefetch is deprecated. Use an explicitly defined document summary with all desired fields defined as attribute."); - } - if (Boolean.FALSE.equals(a.getPrefetchValue())) { - warn(search, field, "Attribute prefetch is deprecated. no-prefetch can be removed."); - } - } - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java index 724fb060dc2..2e6187d4545 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java @@ -32,6 +32,7 @@ public class ImplicitSummaries extends Processor { DocumentSummary defaultSummary = search.getSummary("default"); if (defaultSummary == null) { defaultSummary = new DocumentSummary("default"); + defaultSummary.setFromDisk(true); search.addSummary(defaultSummary); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java index 4feb5b6103b..8c60de1241e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java @@ -17,7 +17,7 @@ import java.util.*; /** * This processor modifies all indexing scripts so that they output to the owning field by default. It also prevents - * any output expression from writing to any field except for the owning field. Finally, for <tt>SummaryExpression</tt>, + * any output expression from writing to any field except for the owning field. Finally, for <code>SummaryExpression</code>, * this processor expands to write all appropriate summary fields. * * @author Simon Thoresen Hult diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java index 8396139861d..47433479588 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java @@ -39,7 +39,6 @@ public class Processing { MutableAttributes::new, CreatePositionZCurve::new, WordMatch::new, - DeprecateAttributePrefetch::new, ImportedFieldsResolver::new, ImplicitSummaries::new, ImplicitSummaryFields::new, diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDiskAccessValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDiskAccessValidator.java index d30006a753e..8c0c6a74037 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDiskAccessValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDiskAccessValidator.java @@ -42,11 +42,10 @@ public class SummaryDiskAccessValidator extends Processor { throw new IllegalArgumentException(summaryField + " in " + summary + " references " + source + ", but this field does not exist"); if ( ! isInMemory(field) && ! summary.isFromDisk()) { - // TODO: Set to warning on vespa 7 - deployLogger.log(Level.FINE, summaryField + " in " + summary + " references " + - source + ", which is not an attribute: Using this " + - "summary will cause disk accesses. " + - "Set 'from-disk' on this summary class to silence this warning."); + deployLogger.log(Level.WARNING, summaryField + " in " + summary + " references " + + source + ", which is not an attribute: Using this " + + "summary will cause disk accesses. " + + "Set 'from-disk' on this summary class to silence this warning."); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java index 0f162ee9d51..a9d7f407eb5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java @@ -121,12 +121,6 @@ public class SummaryField extends Field implements Cloneable, TypedKey { public void setImplicit(boolean implicit) { this.implicit=implicit; } - @Override // override to make public - @Deprecated - public void setDataType(DataType type) { - super.setDataType(type); - } - public boolean isImplicit() { return implicit; } public void setTransform(SummaryTransform transform) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index 9e12886143d..6b0c9a1abb2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -44,7 +44,6 @@ public class Admin extends AbstractConfigProducer implements Serializable { private final boolean isHostedVespa; private final Monitoring monitoring; private final Metrics metrics; - private final Map<String, MetricsConsumer> legacyMetricsConsumers; private final List<Configserver> configservers = new ArrayList<>(); private final List<Slobrok> slobroks = new ArrayList<>(); @@ -73,14 +72,16 @@ public class Admin extends AbstractConfigProducer implements Serializable { private FileDistributionConfigProducer fileDistribution; private final boolean multitenant; - public Admin(AbstractConfigProducer parent, Monitoring monitoring, Metrics metrics, - Map<String, MetricsConsumer> legacyMetricsConsumers, boolean multitenant, - FileDistributionConfigProducer fileDistributionConfigProducer,boolean isHostedVespa) { + public Admin(AbstractConfigProducer parent, + Monitoring monitoring, + Metrics metrics, + boolean multitenant, + FileDistributionConfigProducer fileDistributionConfigProducer, + boolean isHostedVespa) { super(parent, "admin"); this.isHostedVespa = isHostedVespa; this.monitoring = monitoring; this.metrics = metrics; - this.legacyMetricsConsumers = legacyMetricsConsumers; this.multitenant = multitenant; this.fileDistribution = fileDistributionConfigProducer; } @@ -94,11 +95,6 @@ public class Admin extends AbstractConfigProducer implements Serializable { public Metrics getUserMetrics() { return metrics; } - /** Returns the configured userMetricConsumers. Empty if not configured */ - public Map<String, MetricsConsumer> getLegacyUserMetricsConsumers(){ - return legacyMetricsConsumers; - } - /** Returns a list of all config servers */ public List<Configserver> getConfigservers() { return configservers; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerClusterVerifier.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerClusterVerifier.java index 5ab16037957..4edfab6d738 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerClusterVerifier.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerClusterVerifier.java @@ -17,7 +17,7 @@ import java.util.Set; * Verifies that all containers added are ClusterControllerContainers and that filters away Linguistics components. */ public class ClusterControllerClusterVerifier implements ContainerClusterVerifier { - static final Set<ComponentSpecification> unwantedComponents = Collections.singleton(new SimpleComponent(ContainerCluster.SIMPLE_LINGUISTICS_PROVIDER).getClassId()); + static final Set<ComponentSpecification> unwantedComponents = Collections.singleton(new SimpleComponent(ContainerCluster.DEFAULT_LINGUISTICS_PROVIDER).getClassId()); @Override public boolean acceptComponent(Component component) { return ! unwantedComponents.contains(component.getClassId()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index ef180119ac0..d64d8d03481 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -69,7 +69,7 @@ public class ClusterControllerContainer extends Container implements addBundle("file:" + getDefaults().underVespaHome("lib/jars/zkfacade-jar-with-dependencies.jar")); log.log(LogLevel.DEBUG, "Adding access log for cluster controller ..."); - addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.queryAccessLog, "controller", isHosted)); + addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, "controller", isHosted)); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index f3f80304b21..4a9f01597c2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -41,7 +41,7 @@ public class VespaMetricSet { metrics.add(new Metric("sentinel.restarts.count")); metrics.add(new Metric("sentinel.totalRestarts.last")); - metrics.add(new Metric("sentinel.uptime.last", "sentinel.uptime")); + metrics.add(new Metric("sentinel.uptime.last")); metrics.add(new Metric("sentinel.running.count")); metrics.add(new Metric("sentinel.running.last")); @@ -51,28 +51,28 @@ public class VespaMetricSet { private static Set<Metric> getOtherMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); - metrics.add(new Metric("slobrok.heartbeats.failed.count", "slobrok.heartbeats.failed")); - metrics.add(new Metric("logd.processed.lines.count", "logd.processed.lines")); + metrics.add(new Metric("slobrok.heartbeats.failed.count")); + metrics.add(new Metric("logd.processed.lines.count")); return metrics; } private static Set<Metric> getConfigServerMetrics() { Set<Metric> metrics =new LinkedHashSet<>(); - metrics.add(new Metric("configserver.requests.count", "configserver.requests")); - metrics.add(new Metric("configserver.failedRequests.count", "configserver.failedRequests")); - metrics.add(new Metric("configserver.latency.average", "configserver.latency")); - metrics.add(new Metric("configserver.cacheConfigElems.last", "configserver.cacheConfigElems")); - metrics.add(new Metric("configserver.cacheChecksumElems.last", "configserver.cacheChecksumElems")); - metrics.add(new Metric("configserver.hosts.last", "configserver.hosts")); - metrics.add(new Metric("configserver.delayedResponses.count", "configserver.delayedResponses")); - metrics.add(new Metric("configserver.sessionChangeErrors.count", "configserver.sessionChangeErrors")); - - metrics.add(new Metric("configserver.zkZNodes.last", "configserver.zkZNodes")); - metrics.add(new Metric("configserver.zkAvgLatency.last", "configserver.zkAvgLatency")); - metrics.add(new Metric("configserver.zkMaxLatency.last", "configserver.zkMaxLatency")); - metrics.add(new Metric("configserver.zkConnections.last", "configserver.zkConnections")); - metrics.add(new Metric("configserver.zkOutstandingRequests.last", "configserver.zkOutstandingRequests")); + metrics.add(new Metric("configserver.requests.count")); + metrics.add(new Metric("configserver.failedRequests.count")); + metrics.add(new Metric("configserver.latency.average")); + metrics.add(new Metric("configserver.cacheConfigElems.last")); + metrics.add(new Metric("configserver.cacheChecksumElems.last")); + metrics.add(new Metric("configserver.hosts.last")); + metrics.add(new Metric("configserver.delayedResponses.count")); + metrics.add(new Metric("configserver.sessionChangeErrors.count")); + + metrics.add(new Metric("configserver.zkZNodes.last")); + metrics.add(new Metric("configserver.zkAvgLatency.last")); + metrics.add(new Metric("configserver.zkMaxLatency.last")); + metrics.add(new Metric("configserver.zkConnections.last")); + metrics.add(new Metric("configserver.zkOutstandingRequests.last")); return metrics; } @@ -80,7 +80,7 @@ public class VespaMetricSet { private static Set<Metric> getContainerMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); - metrics.add(new Metric("handled.requests.count", "handled.requests")); + metrics.add(new Metric("handled.requests.count")); metrics.add(new Metric("handled.latency.average")); metrics.add(new Metric("handled.latency.max")); @@ -126,7 +126,7 @@ public class VespaMetricSet { metrics.add(new Metric("jdisc.deactivated_containers.total.last")); metrics.add(new Metric("jdisc.deactivated_containers.with_retained_refs.last")); - metrics.add(new Metric("athenz-tenant-cert.expiry.seconds.last", "athenz-tenant-cert.expiry.seconds")); + metrics.add(new Metric("athenz-tenant-cert.expiry.seconds.last")); metrics.add(new Metric("jdisc.http.request.prematurely_closed.rate")); @@ -154,7 +154,7 @@ public class VespaMetricSet { metrics.add(new Metric("cluster-controller.retired.count.last")); metrics.add(new Metric("cluster-controller.stopping.count.last")); metrics.add(new Metric("cluster-controller.up.count.last")); - metrics.add(new Metric("cluster-controller.cluster-state-change.count", "content.cluster-controller.cluster-state-change.count")); + metrics.add(new Metric("cluster-controller.cluster-state-change.count")); metrics.add(new Metric("cluster-controller.is-master.last")); // TODO(hakonhall): Update this name once persistent "count" metrics has been implemented. @@ -168,7 +168,7 @@ public class VespaMetricSet { Set<Metric> metrics = new LinkedHashSet<>(); // per chain - metrics.add(new Metric("documents_processed.rate", "documents_processed")); + metrics.add(new Metric("documents_processed.rate")); return metrics; } @@ -176,40 +176,40 @@ public class VespaMetricSet { private static Set<Metric> getQrserverMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); - metrics.add(new Metric("peak_qps.max", "peak_qps")); - metrics.add(new Metric("search_connections.average", "search_connections")); - metrics.add(new Metric("active_queries.average", "active_queries")); + metrics.add(new Metric("peak_qps.max")); + metrics.add(new Metric("search_connections.average")); + metrics.add(new Metric("active_queries.average")); metrics.add(new Metric("feed.latency.average")); - metrics.add(new Metric("queries.rate", "queries")); + metrics.add(new Metric("queries.rate")); metrics.add(new Metric("query_container_latency.average")); - metrics.add(new Metric("query_latency.average", "mean_query_latency")); - metrics.add(new Metric("query_latency.max", "max_query_latency")); - metrics.add(new Metric("query_latency.95percentile", "95p_query_latency")); - metrics.add(new Metric("query_latency.99percentile", "99p_query_latency")); - metrics.add(new Metric("failed_queries.rate", "failed_queries")); - metrics.add(new Metric("degraded_queries.rate", "degraded_queries")); - metrics.add(new Metric("hits_per_query.average", "hits_per_query")); + metrics.add(new Metric("query_latency.average")); + metrics.add(new Metric("query_latency.max")); + metrics.add(new Metric("query_latency.95percentile")); + metrics.add(new Metric("query_latency.99percentile")); + metrics.add(new Metric("failed_queries.rate")); + metrics.add(new Metric("degraded_queries.rate")); + metrics.add(new Metric("hits_per_query.average")); metrics.add(new Metric("documents_covered.count")); metrics.add(new Metric("documents_total.count")); - metrics.add(new Metric("totalhits_per_query.average", "totalhits_per_query")); - metrics.add(new Metric("empty_results.rate", "empty_results")); + metrics.add(new Metric("totalhits_per_query.average")); + metrics.add(new Metric("empty_results.rate")); metrics.add(new Metric("requestsOverQuota.rate")); metrics.add(new Metric("requestsOverQuota.count")); // Errors from qrserver - metrics.add(new Metric("error.timeout.rate","error.timeout")); - metrics.add(new Metric("error.backends_oos.rate","error.backends_oos")); - metrics.add(new Metric("error.plugin_failure.rate","error.plugin_failure")); - metrics.add(new Metric("error.backend_communication_error.rate","error.backend_communication_error")); - metrics.add(new Metric("error.empty_document_summaries.rate","error.empty_document_summaries")); - metrics.add(new Metric("error.invalid_query_parameter.rate","error.invalid_query_parameter")); - metrics.add(new Metric("error.internal_server_error.rate", "error.internal_server_error")); - metrics.add(new Metric("error.misconfigured_server.rate","error.misconfigured_server")); - metrics.add(new Metric("error.invalid_query_transformation.rate","error.invalid_query_transformation")); - metrics.add(new Metric("error.result_with_errors.rate","error.result_with_errors")); - metrics.add(new Metric("error.unspecified.rate","error.unspecified")); - metrics.add(new Metric("error.unhandled_exception.rate","error.unhandled_exception")); + metrics.add(new Metric("error.timeout.rate")); + metrics.add(new Metric("error.backends_oos.rate")); + metrics.add(new Metric("error.plugin_failure.rate")); + metrics.add(new Metric("error.backend_communication_error.rate")); + metrics.add(new Metric("error.empty_document_summaries.rate")); + metrics.add(new Metric("error.invalid_query_parameter.rate")); + metrics.add(new Metric("error.internal_server_error.rate")); + metrics.add(new Metric("error.misconfigured_server.rate")); + metrics.add(new Metric("error.invalid_query_transformation.rate")); + metrics.add(new Metric("error.result_with_errors.rate")); + metrics.add(new Metric("error.unspecified.rate")); + metrics.add(new Metric("error.unhandled_exception.rate")); return metrics; } @@ -217,18 +217,18 @@ public class VespaMetricSet { private static Set<Metric> getSearchNodeMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); - metrics.add(new Metric("proton.numstoreddocs.last", "documents_total")); - metrics.add(new Metric("proton.numindexeddocs.last", "documents_ready")); - metrics.add(new Metric("proton.numactivedocs.last", "documents_active")); - metrics.add(new Metric("proton.numremoveddocs.last", "documents_removed")); + metrics.add(new Metric("content.proton.documentdb.documents.total.last")); + metrics.add(new Metric("content.proton.documentdb.documents.ready.last")); + metrics.add(new Metric("content.proton.documentdb.documents.active.last")); + metrics.add(new Metric("content.proton.documentdb.documents.removed.last")); - metrics.add(new Metric("proton.docsinmemory.last", "documents_inmemory")); - metrics.add(new Metric("proton.diskusage.last", "diskusage")); - metrics.add(new Metric("proton.memoryusage.max", "content.proton.memoryusage.max")); - metrics.add(new Metric("proton.transport.query.count.rate", "query_requests")); - metrics.add(new Metric("proton.docsum.docs.rate", "document_requests")); - metrics.add(new Metric("proton.docsum.latency.average", "content.proton.transport.docsum.latency.average")); - metrics.add(new Metric("proton.transport.query.latency.average", "query_latency")); + metrics.add(new Metric("content.proton.documentdb.index.docs_in_memory.last")); + metrics.add(new Metric("content.proton.documentdb.disk_usage.last")); + metrics.add(new Metric("content.proton.documentdb.memory_usage.allocated_bytes.max")); + metrics.add(new Metric("content.proton.transport.query.count.rate")); + metrics.add(new Metric("content.proton.docsum.docs.rate")); + metrics.add(new Metric("content.proton.docsum.latency.average")); + metrics.add(new Metric("content.proton.transport.query.latency.average")); // jobs metrics.add(new Metric("content.proton.documentdb.job.total.average")); @@ -343,10 +343,10 @@ public class VespaMetricSet { private static Set<Metric> getStorageMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); - metrics.add(new Metric("vds.datastored.alldisks.docs.average","docs")); - metrics.add(new Metric("vds.datastored.alldisks.bytes.average","bytes")); - metrics.add(new Metric("vds.visitor.allthreads.averagevisitorlifetime.sum.average","visitorlifetime")); - metrics.add(new Metric("vds.visitor.allthreads.averagequeuewait.sum.average","visitorqueuewait")); + metrics.add(new Metric("vds.datastored.alldisks.docs.average")); + metrics.add(new Metric("vds.datastored.alldisks.bytes.average")); + metrics.add(new Metric("vds.visitor.allthreads.averagevisitorlifetime.sum.average")); + metrics.add(new Metric("vds.visitor.allthreads.averagequeuewait.sum.average")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.count.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.count.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.count.rate")); @@ -354,12 +354,12 @@ public class VespaMetricSet { metrics.add(new Metric("vds.filestor.alldisks.allthreads.createiterator.count.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.visit.sum.count.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove_location.sum.count.rate")); - metrics.add(new Metric("vds.filestor.alldisks.queuesize.average","diskqueuesize")); - metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.average","diskqueuewait")); + metrics.add(new Metric("vds.filestor.alldisks.queuesize.average")); + metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.average")); metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.average")); metrics.add(new Metric("vds.visitor.allthreads.completed.sum.average")); - metrics.add(new Metric("vds.visitor.allthreads.created.sum.rate","visit")); + metrics.add(new Metric("vds.visitor.allthreads.created.sum.rate")); metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.average")); metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.average")); @@ -397,18 +397,18 @@ public class VespaMetricSet { metrics.add(new Metric("vds.idealstate.buckets_toomanycopies.average")); metrics.add(new Metric("vds.idealstate.buckets.average")); metrics.add(new Metric("vds.idealstate.buckets_notrusted.average")); - metrics.add(new Metric("vds.idealstate.delete_bucket.done_ok.rate","deleteok")); - metrics.add(new Metric("vds.idealstate.delete_bucket.done_failed.rate","deletefailed")); - metrics.add(new Metric("vds.idealstate.delete_bucket.pending.average","deletepending")); - metrics.add(new Metric("vds.idealstate.merge_bucket.done_ok.rate","mergeok")); - metrics.add(new Metric("vds.idealstate.merge_bucket.done_failed.rate","mergefailed")); - metrics.add(new Metric("vds.idealstate.merge_bucket.pending.average","mergepending")); - metrics.add(new Metric("vds.idealstate.split_bucket.done_ok.rate","splitok")); - metrics.add(new Metric("vds.idealstate.split_bucket.done_failed.rate","splitfailed")); - metrics.add(new Metric("vds.idealstate.split_bucket.pending.average","splitpending")); - metrics.add(new Metric("vds.idealstate.join_bucket.done_ok.rate","joinok")); - metrics.add(new Metric("vds.idealstate.join_bucket.done_failed.rate","joinfailed")); - metrics.add(new Metric("vds.idealstate.join_bucket.pending.average","joinpending")); + metrics.add(new Metric("vds.idealstate.delete_bucket.done_ok.rate")); + metrics.add(new Metric("vds.idealstate.delete_bucket.done_failed.rate")); + metrics.add(new Metric("vds.idealstate.delete_bucket.pending.average")); + metrics.add(new Metric("vds.idealstate.merge_bucket.done_ok.rate")); + metrics.add(new Metric("vds.idealstate.merge_bucket.done_failed.rate")); + metrics.add(new Metric("vds.idealstate.merge_bucket.pending.average")); + metrics.add(new Metric("vds.idealstate.split_bucket.done_ok.rate")); + metrics.add(new Metric("vds.idealstate.split_bucket.done_failed.rate")); + metrics.add(new Metric("vds.idealstate.split_bucket.pending.average")); + metrics.add(new Metric("vds.idealstate.join_bucket.done_ok.rate")); + metrics.add(new Metric("vds.idealstate.join_bucket.done_failed.rate")); + metrics.add(new Metric("vds.idealstate.join_bucket.pending.average")); metrics.add(new Metric("vds.idealstate.garbage_collection.done_ok.rate")); metrics.add(new Metric("vds.idealstate.garbage_collection.done_failed.rate")); metrics.add(new Metric("vds.idealstate.garbage_collection.pending.average")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java index fa28802b3ac..3a9a5d59b1a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -72,18 +72,12 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu @Override protected Admin doBuild(DeployState deployState, AbstractConfigProducer parent, Element adminElement) { - Monitoring monitoring = getMonitoring(getChildWithFallback(adminElement, "monitoring", "yamas")); + Monitoring monitoring = getMonitoring(XML.getChild(adminElement,"monitoring")); Metrics metrics = new MetricsBuilder(applicationType, predefinedMetricSets) .buildMetrics(XML.getChild(adminElement, "metrics")); - Map<String, MetricsConsumer> legacyMetricsConsumers = DomMetricBuilderHelper - .buildMetricsConsumers(XML.getChild(adminElement, "metric-consumers")); - if (! legacyMetricsConsumers.isEmpty()) { - deployState.getDeployLogger().log(WARNING, "Element 'metric-consumers' is deprecated and will be removed in Vespa 7. Use 'metrics' instead!"); - } FileDistributionConfigProducer fileDistributionConfigProducer = getFileDistributionConfigProducer(parent); - Admin admin = new Admin(parent, monitoring, metrics, legacyMetricsConsumers, multitenant, - fileDistributionConfigProducer, deployState.isHosted()); + Admin admin = new Admin(parent, monitoring, metrics, multitenant, fileDistributionConfigProducer, deployState.isHosted()); admin.setApplicationType(applicationType); doBuildAdmin(deployState, admin, adminElement); new ModelConfigProvider(admin); @@ -95,12 +89,6 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu return new FileDistributionConfigProducer(parent, fileRegistry, configServerSpecs); } - private Element getChildWithFallback(Element parent, String childName, String alternativeChildName) { - Element child = XML.getChild(parent, childName); - if (child != null) return child; - return XML.getChild(parent, alternativeChildName); - } - protected abstract void doBuildAdmin(DeployState deployState, Admin admin, Element adminE); private Monitoring getMonitoring(Element monitoringElement) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientsBuilder.java index 24da87200b8..c107a85bcb5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientsBuilder.java @@ -30,7 +30,7 @@ public class DomClientsBuilder extends LegacyConfigModelBuilder<Clients> { String version = clientsE.getAttribute("version"); if (version.startsWith("2.")) { DomV20ClientsBuilder parser = new DomV20ClientsBuilder(clients, version); - parser.build(modelContext.getDeployState(), clientsE); + parser.build(clientsE); } else { throw new IllegalArgumentException("Version '" + version + "' of 'clients' not supported."); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomMetricBuilderHelper.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomMetricBuilderHelper.java deleted file mode 100644 index 4c1fdd3f92c..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomMetricBuilderHelper.java +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.builder.xml.dom; - -import com.yahoo.text.XML; -import com.yahoo.vespa.model.admin.monitoring.Metric; -import com.yahoo.vespa.model.admin.monitoring.MetricSet; -import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; -import org.w3c.dom.Element; - -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static com.yahoo.vespa.model.admin.monitoring.DefaultMetricsConsumer.VESPA_CONSUMER_ID; - -/** - * Helper class for parsing metric config. - * - * TODO: Remove when 'metric-consumers' under 'admin' is disallowed - * - * @author Trygve Berdal - */ -public class DomMetricBuilderHelper { - - private static final String LEGACY_DEFAULT_CONSUMER_ID = "yamas"; - - /** - * Build metricConsumer config - * - * @param spec xml element - * @return a map from metric name to a {@link MetricsConsumer} - */ - protected static Map<String, MetricsConsumer> buildMetricsConsumers(Element spec) { - Map<String, MetricsConsumer> metricsConsumers = new LinkedHashMap<>(); - List<Element> consumersElem = XML.getChildren(spec, "consumer"); - for (Element consumer : consumersElem) { - String consumerName = getConsumerName(consumer); - Set<Metric> metrics = new LinkedHashSet<>(); - List<Element> metricsEl = XML.getChildren(consumer, "metric"); - if (metricsEl != null) { - for (Element metric : metricsEl) { - String metricName = metric.getAttribute("name"); - String outputName = metric.getAttribute("output-name"); - metrics.add(new Metric(metricName, outputName)); - } - } - MetricsConsumer metricsConsumer = new MetricsConsumer(consumerName, - new MetricSet(metricSetId(consumerName), metrics)); - metricsConsumers.put(consumerName, metricsConsumer); - } - return metricsConsumers; - } - - // Converts the old default consumer id to the new default id. - private static String getConsumerName(Element consumerElement) { - String givenName = consumerElement.getAttribute("name"); - if (givenName.equals(LEGACY_DEFAULT_CONSUMER_ID)) - return VESPA_CONSUMER_ID; - else - return givenName; - } - - private static String metricSetId(String consumerName) { - return "legacy-user-metrics-" + consumerName; - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java index 81f901fc849..067d653ea40 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java @@ -128,9 +128,7 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui } } else if (equals("transactionlog", e)) { for (Element subElem : XML.getChildren(e)) { - if (equals("maxentries", subElem)) { - deployLogger.log(Level.WARNING, "Element 'transactionlog.maxentries is deprecated and ignored in 'native' flush strategy. Use 'transactionlog.maxsize' to limit by size."); - } else if (equals("maxsize", subElem)) { + if (equals("maxsize", subElem)) { fs.transactionLogMaxSize = asLong(subElem); } } @@ -224,8 +222,6 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui c.maxSize = asLong(e); } else if (equals("maxsize-percent", e)) { c.maxSizePercent = asDouble(e); - } else if (equals("maxentries", e)) { - deployLogger.log(Level.WARNING, "Element 'maxentries is deprecated and ignored. Will only limit by size."); } else if (equals("initialentries", e)) { c.initialEntries = asLong(e); } else if (equals("compression", e)) { @@ -250,15 +246,8 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui for (Element e : XML.getChildren(spec)) { if (equals("maxfilesize", e)) { s.logStore.maxFileSize = asLong(e); - } else if (equals("maxdiskbloatfactor", e)) { - deployLogger.log(Level.WARNING, - "Element 'maxdiskbloatfactor is deprecated and ignored." + - " The min value from flush.memory.xxx.diskbloatfactor is used instead"); } else if (equals("minfilesizefactor", e)) { s.logStore.minFileSizeFactor = asDouble(e); - } else if (equals("numthreads", e)) { - deployLogger.log(Level.WARNING, "Element 'numthreads is deprecated. Use feeding.concurrency instead."); - s.logStore.numThreads = asInt(e); } else if (equals("chunk", e)) { s.logStore.chunk = new Tuning.SearchNode.Summary.Store.Component(true); handleSummaryStoreComponent(deployLogger, e, s.logStore.chunk); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java index 6a02619bdb4..74e3ac581c4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java @@ -2,26 +2,11 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.config.content.spooler.SpoolerConfig; -import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.text.XML; -import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.model.SimpleConfigProducer; -import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder.DomConfigProducerBuilder; import com.yahoo.vespa.model.clients.Clients; -import com.yahoo.vespa.model.clients.VespaSpoolMaster; -import com.yahoo.vespa.model.clients.VespaSpooler; -import com.yahoo.vespa.model.clients.VespaSpoolerProducer; -import com.yahoo.vespa.model.clients.VespaSpoolerService; -import com.yahoo.vespaclient.config.FeederConfig; import org.w3c.dom.Element; -import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - /** * Builds the Clients plugin * @@ -38,13 +23,8 @@ public class DomV20ClientsBuilder { this.clients = clients; } - public void build(DeployState deployState, Element spec) { - NodeList children = spec.getElementsByTagName("spoolers"); - for (int i = 0; i < children.getLength(); i++) { - createSpoolers(deployState, clients.getConfigProducer(), (Element) children.item(i), clients); - } - - children = spec.getElementsByTagName("load-types"); + public void build(Element spec) { + NodeList children = spec.getElementsByTagName("load-types"); for (int i = 0; i < children.getLength(); i++) { createLoadTypes((Element) children.item(i), clients); } @@ -57,283 +37,4 @@ public class DomV20ClientsBuilder { } } - /** - * Creates VespaSpooler objects using the given xml Element. - */ - private void createSpoolers(DeployState deployState, AbstractConfigProducer pcp, Element element, Clients clients) { - String jvmArgs = null; - if (element.hasAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME)) jvmArgs=element.getAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME); - SimpleConfigProducer spoolerCfg = new VespaDomBuilder.DomSimpleConfigProducerBuilder(element.getNodeName()). - build(deployState, pcp, element); - Element spoolersFeederOptions = findFeederOptions(element); - createSpoolMasters(deployState, spoolerCfg, element); - for (Element e : XML.getChildren(element, "spooler")) { - String configId = e.getAttribute("id").trim(); - FeederConfig.Builder feederConfig = getFeederConfig(spoolersFeederOptions, e); - SpoolerConfig.Builder spoolConfig = getSpoolConfig(e); - if (configId.length() == 0) { - int index = clients.getVespaSpoolers().size(); - VespaSpoolerService spoolerService = new VespaSpoolerServiceBuilder(index, new VespaSpooler(feederConfig, spoolConfig)). - build(deployState, spoolerCfg, e); - if ("".equals(spoolerService.getJvmOptions()) && jvmArgs!=null) spoolerService.setJvmOptions(jvmArgs); - spoolerService.setProp("index", String.valueOf(index)); - clients.getVespaSpoolers().add(spoolerService); - } else { - new VespaSpoolerProducerBuilder(configId, new VespaSpooler(feederConfig, spoolConfig)). - build(deployState, spoolerCfg, e); - } - } - } - - private void createSpoolMasters(DeployState deployState, SimpleConfigProducer producer, Element element) { - int i=0; - for (Element e : XML.getChildren(element, "spoolmaster")) - new VespaSpoolMasterBuilder(i++).build(deployState, producer, e); - } - - private SpoolerConfig.Builder getSpoolConfig(Element conf) { - SpoolerConfig.Builder builder = new SpoolerConfig.Builder(); - if (conf.getAttributes().getNamedItem("directory") != null) { - builder.directory(Defaults.getDefaults().underVespaHome(conf.getAttributes().getNamedItem("directory").getNodeValue())); - } - if (conf.getAttributes().getNamedItem("keepsuccess") != null) { - builder.keepsuccess(getBooleanFromAttribute(conf, "keepsuccess")); - } - if (conf.getAttributes().getNamedItem("maxfailuresize") != null) { - builder.maxfailuresize(getIntegerFromAttribute(conf, "maxfailuresize")); - } - if (conf.getAttributes().getNamedItem("maxfatalfailuresize") != null) { - builder.maxfatalfailuresize(getIntegerFromAttribute(conf, "maxfatalfailuresize")); - } - if (conf.getAttributes().getNamedItem("threads") != null) { - builder.threads(getIntegerFromAttribute(conf, "threads")); - } - if (conf.getAttributes().getNamedItem("maxretries") != null) { - builder.maxretries(getIntegerFromAttribute(conf, "maxretries")); - } - - NodeList children = conf.getElementsByTagName("parsers"); - if (children.getLength() == 1) { - children = ((Element)children.item(0)).getElementsByTagName("parser"); - } - - for (int i=0; i < children.getLength(); i++) { - Element e = (Element)children.item(i); - - String type = e.getAttributes().getNamedItem("type").getNodeValue(); - NodeList params = e.getElementsByTagName("parameter"); - - SpoolerConfig.Parsers.Builder parserBuilder = new SpoolerConfig.Parsers.Builder(); - parserBuilder.classname(type); - if (params.getLength() > 0) { - List<SpoolerConfig.Parsers.Parameters.Builder> parametersBuilders = new ArrayList<>(); - for (int j = 0; j < params.getLength(); j++) { - SpoolerConfig.Parsers.Parameters.Builder parametersBuilder = new SpoolerConfig.Parsers.Parameters.Builder(); - Element p = (Element) params.item(j); - parametersBuilder.key(getStringFromAttribute(p, "key")); - parametersBuilder.value(getStringFromAttribute(p, "value")); - parametersBuilders.add(parametersBuilder); - } - parserBuilder.parameters(parametersBuilders); - } - - builder.parsers.add(parserBuilder); - } - return builder; - } - - Boolean getBooleanFromAttribute(Element e, String attributeName) { - return Boolean.parseBoolean(e.getAttributes().getNamedItem(attributeName).getNodeValue()); - } - - Integer getIntegerFromAttribute(Element e, String attributeName) { - return Integer.parseInt(e.getAttributes().getNamedItem(attributeName).getNodeValue()); - } - - String getStringFromAttribute(Element e, String attributeName) { - return e.getAttributes().getNamedItem(attributeName).getNodeValue(); - } - - private FeederConfig.Builder getFeederConfig(Element gatewaysFeederOptions, Element e) { - FeederOptionsParser foParser = new FeederOptionsParser(); - if (gatewaysFeederOptions!=null) { - foParser.parseFeederOptions(gatewaysFeederOptions).getFeederConfig(); - } - foParser.parseFeederOptions(e); - return foParser.getFeederConfig(); - } - - /** - * Finds the feederoptions subelement in the given xml Element. - * - * @param element The xml Element - * @return The feederoptions xml Element - */ - private Element findFeederOptions(Element element) { - for (Element child : XML.getChildren(element)) { - if (child.getNodeName().equals("feederoptions")) { - return child; - } - } - return null; - } - - private static class VespaSpoolerServiceBuilder extends DomConfigProducerBuilder<VespaSpoolerService> { - private int index; - private VespaSpooler spoolerConfig; - - public VespaSpoolerServiceBuilder(int index, VespaSpooler spoolerConfig) { - this.index = index; - this.spoolerConfig = spoolerConfig; - } - - @Override - protected VespaSpoolerService doBuild(DeployState deployState, AbstractConfigProducer parent, Element spec) { - return new VespaSpoolerService(parent, index, spoolerConfig); - } - } - - private static class VespaSpoolerProducerBuilder extends DomConfigProducerBuilder<VespaSpoolerProducer> { - private String name=null; - private VespaSpooler spooler; - - public VespaSpoolerProducerBuilder(String name, VespaSpooler spooler) { - this.name = name; - this.spooler = spooler; - } - - @Override - protected VespaSpoolerProducer doBuild(DeployState deployState, AbstractConfigProducer parent, Element producerSpec) { - return new VespaSpoolerProducer(parent, name, spooler); - } - } - - private static class VespaSpoolMasterBuilder extends DomConfigProducerBuilder<VespaSpoolMaster> { - int index; - - public VespaSpoolMasterBuilder(int index) { - super(); - this.index = index; - } - - @Override - protected VespaSpoolMaster doBuild(DeployState deployState, AbstractConfigProducer parent, Element spec) { - return new VespaSpoolMaster(parent, index); - } - } - - /** - * This class parses the feederoptions xml tag and produces Vespa config output. - * - * @author Gunnar Gauslaa Bergem - */ - private class FeederOptionsParser implements Serializable { - private static final long serialVersionUID = 1L; - // All member variables are objects so that we can switch on null values. - private Boolean abortondocumenterror = null; - private String route = null; - private Integer maxpendingdocs = null; - private Integer maxpendingbytes = null; - private Boolean retryenabled = null; - private Double retrydelay = null; - private Double timeout = null; - private Integer tracelevel = null; - private Integer mbusport = null; - private String docprocChain = null; - - /** - * Constructs an empty feeder options object with all members set to null. - */ - public FeederOptionsParser() { - // empty - } - - /** - * Parses the content of the given XML element as feeder options. - * - * @param conf The XML element to parse. - */ - public FeederOptionsParser parseFeederOptions(Element conf) { - for (Node node : XML.getChildren(conf)) { - String nodename = node.getNodeName(); - Node firstchild = node.getFirstChild(); - String childval = (firstchild != null) ? firstchild.getNodeValue() : null; - - switch (nodename) { - case "abortondocumenterror": - abortondocumenterror = Boolean.valueOf(childval); - break; - case "maxpendingdocs": - maxpendingdocs = Integer.valueOf(childval); - break; - case "maxpendingbytes": - maxpendingbytes = Integer.valueOf(childval); - break; - case "retryenabled": - retryenabled = Boolean.valueOf(childval); - break; - case "retrydelay": - retrydelay = Double.valueOf(childval); - break; - case "timeout": - timeout = Double.valueOf(childval); - break; - case "route": - route = childval; - break; - case "tracelevel": - tracelevel = Integer.valueOf(childval); - break; - case "mbusport": - mbusport = Integer.valueOf(childval); - break; - case "docprocchain": - docprocChain = childval; - break; - } - } - return this; - } - - /** - * Returns a feeder options config string of the content of this. - * - * @return A config string. - */ - public FeederConfig.Builder getFeederConfig() { - FeederConfig.Builder builder = new FeederConfig.Builder(); - if (abortondocumenterror != null) { - builder.abortondocumenterror(abortondocumenterror); - } - if (route != null && route.length() > 0) { - builder.route(route); - } - if (maxpendingdocs != null) { - builder.maxpendingdocs(maxpendingdocs); - } - if (maxpendingbytes != null) { - builder.maxpendingbytes(maxpendingbytes); - } - if (retryenabled != null) { - builder.retryenabled(retryenabled); - } - if (retrydelay != null) { - builder.retrydelay(retrydelay); - } - if (timeout != null) { - builder.timeout(timeout); - } - if (tracelevel != null) { - builder.tracelevel(tracelevel); - } - if (mbusport != null) { - builder.mbusport(mbusport); - } - if (docprocChain != null && docprocChain.length() > 0) { - builder.docprocchain(docprocChain); - } - return builder; - } - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilder.java index 66b425be8f5..534b07339ec 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilder.java @@ -2,8 +2,6 @@ package com.yahoo.vespa.model.builder.xml.dom.chains.search; import com.yahoo.binaryprefix.BinaryPrefix; -import com.yahoo.component.chain.dependencies.Dependencies; -import com.yahoo.component.chain.model.ChainedComponentModel; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.model.ChainSpecification; import com.yahoo.config.model.deploy.DeployState; @@ -13,7 +11,7 @@ import com.yahoo.text.XML; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.builder.xml.dom.BinaryScaledAmountParser; import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder; -import com.yahoo.vespa.model.container.search.searchchain.HttpProvider; +import com.yahoo.vespa.model.container.search.searchchain.GenericProvider; import com.yahoo.vespa.model.container.search.searchchain.LocalProvider; import com.yahoo.vespa.model.container.search.searchchain.Provider; import com.yahoo.vespa.model.container.search.searchchain.Source; @@ -23,7 +21,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; /** * Builds a provider from xml. @@ -211,9 +208,7 @@ public class DomProviderBuilder extends DomGenericTargetBuilder<Provider> { ProviderReader providerReader, FederationOptions federationOptions) { if (providerReader.type == null) { - return buildEmptyHttpProvider(specWithoutInnerSearchers, providerReader, federationOptions); - } else if (com.yahoo.search.searchchain.model.federation.HttpProviderSpec.includesType(providerReader.type)) { // TODO: Remove on Vespa 7 - return buildHttpProvider(specWithoutInnerSearchers, providerReader, federationOptions); + return new GenericProvider(specWithoutInnerSearchers, federationOptions); } else if (LocalProviderSpec.includesType(providerReader.type)) { return buildLocalProvider(specWithoutInnerSearchers, providerReader, federationOptions); } else { @@ -236,63 +231,6 @@ public class DomProviderBuilder extends DomGenericTargetBuilder<Provider> { } } - // TODO: Remove on Vespa 7 - @SuppressWarnings("deprecation") - private Provider buildHttpProvider(ChainSpecification specWithoutInnerSearchers, ProviderReader providerReader, FederationOptions federationOptions) { - ensureEmpty(specWithoutInnerSearchers.componentId, providerReader.clusterName); - - Provider httpProvider = buildEmptyHttpProvider(specWithoutInnerSearchers, providerReader, federationOptions); - - httpProvider.addInnerComponent(new com.yahoo.vespa.model.container.search.searchchain.HttpProviderSearcher( - new ChainedComponentModel( - com.yahoo.search.searchchain.model.federation.HttpProviderSpec.toBundleInstantiationSpecification(com.yahoo.search.searchchain.model.federation.HttpProviderSpec.Type.valueOf(providerReader.type)), - Dependencies.emptyDependencies()))); - - return httpProvider; - } - - // TODO: Remove on Vespa 7 - @SuppressWarnings("deprecation") - private Provider buildEmptyHttpProvider(ChainSpecification specWithoutInnerSearchers, ProviderReader providerReader, FederationOptions federationOptions) { - ensureEmpty(specWithoutInnerSearchers.componentId, providerReader.clusterName); - - return new HttpProvider(specWithoutInnerSearchers, - federationOptions, - new com.yahoo.search.searchchain.model.federation.HttpProviderSpec( - providerReader.cacheWeight, - providerReader.path, - toHttpProviderNodes(providerReader.nodes), - providerReader.certificateApplicationId, - providerReader.certificateTtl, - providerReader.certificateRetryWait, - toHttpProviderNode(providerReader.certificateProxy), - providerReader.cacheSizeMB, - connectionParameters(providerReader))); - } - - // TODO: Remove on Vespa 7 - @SuppressWarnings("deprecation") - private com.yahoo.search.searchchain.model.federation.HttpProviderSpec.Node toHttpProviderNode(Node node) { - if (node == null) return null; - return new com.yahoo.search.searchchain.model.federation.HttpProviderSpec.Node(node.host, node.port); - } - - @SuppressWarnings("deprecation") - private List<com.yahoo.search.searchchain.model.federation.HttpProviderSpec.Node> toHttpProviderNodes(List<Node> nodes) { - if (nodes == null) return null; - return nodes.stream().map(node -> toHttpProviderNode(node)).collect(Collectors.toList()); - } - - // TODO: Remove on Vespa 7 - @SuppressWarnings("deprecation") - private com.yahoo.search.searchchain.model.federation.HttpProviderSpec.ConnectionParameters connectionParameters(ProviderReader providerReader) { - return new com.yahoo.search.searchchain.model.federation.HttpProviderSpec.ConnectionParameters( - providerReader.readTimeout, - providerReader.connectionTimeout, - providerReader.connectionPoolTimeout, - providerReader.retries); - } - private void ensureEmpty(ComponentId componentId, Object... objects) { for (Object object : objects) { if (object != null) { @@ -313,10 +251,7 @@ public class DomProviderBuilder extends DomGenericTargetBuilder<Provider> { @Override public String toString() { - return "Node{" + - "host='" + host + '\'' + - ", port=" + port + - '}'; + return "Node{" + "host='" + host + '\'' + ", port=" + port + '}'; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java index 75c10ea2667..9b33568b61e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java @@ -22,23 +22,10 @@ import java.util.List; public class Clients extends ConfigModel { private static final long serialVersionUID = 1L; - private List<VespaSpoolerService> vespaSpoolers = new LinkedList<>(); private LoadTypeSet loadTypes = new LoadTypeSet(); - private final AbstractConfigProducer parent; - public Clients(ConfigModelContext modelContext) { super(modelContext); - this.parent = modelContext.getParentProducer(); - } - - public AbstractConfigProducer getConfigProducer() { - return parent; - } - - /** Returns the mutable list of spoolers which is then used to add spoolers */ - public List<VespaSpoolerService> getVespaSpoolers() { - return vespaSpoolers; } public LoadTypeSet getLoadTypes() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java index 9801eab9f2b..c2394e7cac3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java @@ -1,29 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.clients; -import com.yahoo.component.ComponentId; -import com.yahoo.component.ComponentSpecification; -import com.yahoo.component.chain.Phase; -import com.yahoo.component.chain.dependencies.Dependencies; -import com.yahoo.component.chain.model.ChainSpecification; -import com.yahoo.component.chain.model.ChainedComponentModel; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Handler; -import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; -import com.yahoo.vespa.model.container.search.ContainerSearch; -import com.yahoo.vespa.model.container.search.searchchain.SearchChain; -import com.yahoo.vespa.model.container.search.searchchain.SearchChains; -import com.yahoo.vespa.model.container.search.searchchain.Searcher; import com.yahoo.vespaclient.config.FeederConfig; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.Set; -import java.util.TreeSet; /** * @author Einar M R Rosenvinge @@ -35,53 +21,14 @@ public class ContainerDocumentApi implements FeederConfig.Producer { public ContainerDocumentApi(ContainerCluster cluster, Options options) { this.options = options; - setupLegacySearchers(cluster); setupHandlers(cluster); } private void setupHandlers(ContainerCluster cluster) { - cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandler", "feed")); - cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerRemove", "remove")); - cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation", "removelocation")); - cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerGet", "get")); - cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerVisit", "visit")); cluster.addComponent(newVespaClientHandler("com.yahoo.document.restapi.resource.RestApi", "document/v1/*")); - cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerCompatibility", "document")); - cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerStatus", "feedstatus")); cluster.addComponent(newVespaClientHandler("com.yahoo.vespa.http.server.FeedHandler", ContainerCluster.RESERVED_URI_PREFIX + "/feedapi")); } - private void setupLegacySearchers(ContainerCluster cluster) { - Set<ComponentSpecification> inherited = new TreeSet<>(); - - SearchChain vespaGetChain = new SearchChain(new ChainSpecification(new ComponentId("vespaget"), - new ChainSpecification.Inheritance(inherited, null), new ArrayList<>(), new TreeSet<>())); - vespaGetChain.addInnerComponent(newVespaClientSearcher("com.yahoo.storage.searcher.GetSearcher")); - - SearchChain vespaVisitChain = new SearchChain(new ChainSpecification(new ComponentId("vespavisit"), - new ChainSpecification.Inheritance(inherited, null), new ArrayList<>(), new TreeSet<>())); - vespaVisitChain.addInnerComponent(newVespaClientSearcher("com.yahoo.storage.searcher.VisitSearcher")); - - SearchChains chains; - if (cluster.getSearch() != null) { - chains = cluster.getSearchChains(); - } else { - chains = new SearchChains(cluster, "searchchain"); - } - chains.add(vespaGetChain); - chains.add(vespaVisitChain); - - if (cluster.getSearch() == null) { - ContainerSearch containerSearch = new ContainerSearch(cluster, chains, new ContainerSearch.Options()); - cluster.setSearch(containerSearch); - - ProcessingHandler<SearchChains> searchHandler = new ProcessingHandler<>(chains, - "com.yahoo.search.handler.SearchHandler"); - searchHandler.addServerBindings("http://*/search/*", "https://*/search/*"); - cluster.addComponent(searchHandler); - } - } - private Handler newVespaClientHandler(String componentId, String bindingSuffix) { Handler<AbstractConfigProducer<?>> handler = new Handler<>(new ComponentModel( BundleInstantiationSpecification.getFromStrings(componentId, null, vespaClientBundleSpecification), "")); @@ -93,12 +40,6 @@ public class ContainerDocumentApi implements FeederConfig.Producer { return handler; } - private Searcher newVespaClientSearcher(String componentSpec) { - return new Searcher<>(new ChainedComponentModel( - BundleInstantiationSpecification.getFromStrings(componentSpec, null, vespaClientBundleSpecification), - new Dependencies(null, null, null))); - } - @Override public void getConfig(FeederConfig.Builder builder) { if (options.abortondocumenterror != null) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolMaster.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolMaster.java deleted file mode 100644 index 7013a09009a..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolMaster.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.clients; - -import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.vespa.model.AbstractService; - -/** - * The spoolmaster program, which is used when multiple spooler instances are used to provide - * multi colo HTTP feeding. - * @author vegardh - * - */ -public class VespaSpoolMaster extends AbstractService { - - public VespaSpoolMaster(AbstractConfigProducer parent, int index) { - super(parent, "spoolmaster."+index); - } - - @Override - public int getPortCount() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public String getStartupCommand() { - return "exec vespa-spoolmaster"; - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpooler.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpooler.java deleted file mode 100644 index c750a5d2a89..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpooler.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.clients; - -import com.yahoo.vespa.config.content.spooler.SpoolerConfig; -import com.yahoo.config.subscription.ConfigInstanceUtil; -import com.yahoo.vespaclient.config.FeederConfig; - -/** - * Holds configuration for VespaSpoolers. Actual services use VespaSpoolerService, - * while virtual services can be generated for external spoolers (VespaSpoolerProducer). - * - * @author <a href="mailto:thomasg@yahoo-inc.com">Gunnar Gauslaa Bergem</a> - * @author Vidar Larsen - */ -public class VespaSpooler { - private final SpoolerConfig.Builder spoolConfig; - private final FeederConfig.Builder feederConfig; - - public VespaSpooler(FeederConfig.Builder feederConfig, SpoolerConfig.Builder spoolConfig) { - this.feederConfig = feederConfig; - this.spoolConfig = spoolConfig; - } - - public void getConfig(SpoolerConfig.Builder builder) { - ConfigInstanceUtil.setValues(builder, spoolConfig); - } - - public void getConfig(FeederConfig.Builder builder) { - ConfigInstanceUtil.setValues(builder, feederConfig); - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerProducer.java deleted file mode 100644 index 3fae5389a94..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerProducer.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.clients; - -import com.yahoo.vespa.config.content.spooler.SpoolerConfig; -import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.vespaclient.config.FeederConfig; - -/** - * This model represents a config producer for spooler used for feeding documents to Vespa. - * - * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a> - * @author Vidar Larsen - */ -public class VespaSpoolerProducer extends AbstractConfigProducer implements SpoolerConfig.Producer, FeederConfig.Producer { - private static final long serialVersionUID = 1L; - private VespaSpooler spoolerConfig; - - public VespaSpoolerProducer(AbstractConfigProducer parent, String configId, VespaSpooler spooler) { - super(parent, configId); - spoolerConfig = spooler; - } - - @Override - public void getConfig(SpoolerConfig.Builder builder) { - spoolerConfig.getConfig(builder); - } - - @Override - public void getConfig(FeederConfig.Builder builder) { - spoolerConfig.getConfig(builder); - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerService.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerService.java deleted file mode 100644 index 378c85dc325..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerService.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.clients; - -import com.yahoo.vespa.config.content.spooler.SpoolerConfig; -import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.vespa.model.AbstractService; -import com.yahoo.vespaclient.config.FeederConfig; - -/** - * This model represents a spooler used for feeding documents to Vespa. - * - * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a> - * @author Vidar Larsen - */ -public class VespaSpoolerService extends AbstractService implements SpoolerConfig.Producer, FeederConfig.Producer { - private static final long serialVersionUID = 1L; - private VespaSpooler spooler; - - public VespaSpoolerService(AbstractConfigProducer parent, int index, VespaSpooler spooler) { - super(parent, "spooler." + index); - this.spooler = spooler; - } - - public int getPortCount() { - return 0; - } - - public String getStartupCommand() { - return "exec vespaspooler "+ getJvmOptions(); - } - - @Override - public void getConfig(SpoolerConfig.Builder builder) { - spooler.getConfig(builder); - } - - @Override - public void getConfig(FeederConfig.Builder builder) { - spooler.getConfig(builder); - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 8d46eb3718f..b90f18d4f24 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -26,7 +26,6 @@ import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.container.jdisc.ContainerMbusConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; import com.yahoo.container.jdisc.config.HealthMonitorConfig; -import com.yahoo.container.jdisc.config.MetricDefaultsConfig; import com.yahoo.container.jdisc.jrt.DefaultJrtFactory; import com.yahoo.container.jdisc.messagebus.MbusServerProvider; import com.yahoo.container.jdisc.state.StateHandler; @@ -117,7 +116,6 @@ public final class ContainerCluster PageTemplatesConfig.Producer, SemanticRulesConfig.Producer, DocprocConfig.Producer, - MetricDefaultsConfig.Producer, ClusterInfoConfig.Producer, ServletPathsConfig.Producer, RoutingProviderConfig.Producer, @@ -140,7 +138,7 @@ public final class ContainerCluster public static final String BINDINGS_OVERVIEW_HANDLER_CLASS = BindingsOverviewHandler.class.getName(); public static final String STATE_HANDLER_CLASS = "com.yahoo.container.jdisc.state.StateHandler"; public static final String STATISTICS_HANDLER_CLASS = "com.yahoo.container.config.StatisticsRequestHandler"; - public static final String SIMPLE_LINGUISTICS_PROVIDER = "com.yahoo.language.provider.SimpleLinguisticsProvider"; + public static final String DEFAULT_LINGUISTICS_PROVIDER = "com.yahoo.language.provider.DefaultLinguisticsProvider"; public static final String CMS = "-XX:+UseConcMarkSweepGC -XX:MaxTenuringThreshold=15 -XX:NewRatio=1"; public static final String G1GC = "-XX:+UseG1GC -XX:MaxTenuringThreshold=15"; @@ -176,7 +174,6 @@ public final class ContainerCluster private final boolean isHostedVespa; private Map<String, String> concreteDocumentTypes = new LinkedHashMap<>(); - private MetricDefaultsConfig.Factory.Enum defaultMetricConsumerFactory; private ApplicationMetaData applicationMetaData = null; @@ -221,7 +218,7 @@ public final class ContainerCluster addSimpleComponent(ThreadPoolProvider.class); addSimpleComponent(com.yahoo.concurrent.classlock.ClassLocking.class); addSimpleComponent("com.yahoo.jdisc.http.filter.SecurityFilterInvoker"); - addSimpleComponent(SIMPLE_LINGUISTICS_PROVIDER); + addSimpleComponent(DEFAULT_LINGUISTICS_PROVIDER); addSimpleComponent("com.yahoo.container.jdisc.SecretStoreProvider"); addSimpleComponent("com.yahoo.container.jdisc.CertificateStoreProvider"); addSimpleComponent("com.yahoo.container.jdisc.metric.MetricConsumerProviderProvider"); @@ -232,7 +229,6 @@ public final class ContainerCluster addSimpleComponent(com.yahoo.metrics.simple.jdisc.JdiscMetricsFactory.class.getName(), null, MetricProperties.BUNDLE_SYMBOLIC_NAME); addSimpleComponent("com.yahoo.container.jdisc.state.StateMonitor"); addSimpleComponent("com.yahoo.container.jdisc.ContainerThreadFactory"); - addSimpleComponent("com.yahoo.container.protect.FreezeDetector"); addSimpleComponent("com.yahoo.container.handler.VipStatus"); addSimpleComponent(com.yahoo.container.handler.ClustersStatus.class.getName()); addSimpleComponent(DefaultJrtFactory.class); @@ -294,7 +290,6 @@ public final class ContainerCluster private void addJaxProviders() { addSimpleComponent(com.yahoo.container.xml.providers.DatatypeFactoryProvider.class); addSimpleComponent(com.yahoo.container.xml.providers.DocumentBuilderFactoryProvider.class); - addSimpleComponent(com.yahoo.container.xml.providers.JAXBContextFactoryProvider.class); addSimpleComponent(com.yahoo.container.xml.providers.SAXParserFactoryProvider.class); addSimpleComponent(com.yahoo.container.xml.providers.SchemaFactoryProvider.class); addSimpleComponent(com.yahoo.container.xml.providers.TransformerFactoryProvider.class); @@ -631,9 +626,6 @@ public final class ContainerCluster } else if (isHostedVespa()) { jvmBuilder.heapSizeAsPercentageOfPhysicalMemory(getHostClusterId().isPresent() ? 17 : 60); } - if (containerSearch!=null) { - jvmBuilder.directMemorySizeCache(containerSearch.totalCacheSizeMb()); - } if (jvmGCOptions != null) { jvmBuilder.gcopts(jvmGCOptions); } else { @@ -710,7 +702,7 @@ public final class ContainerCluster } public void addDefaultSearchAccessLog() { - addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.queryAccessLog, getName(), isHostedVespa)); + addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, getName(), isHostedVespa)); } @Override @@ -723,11 +715,6 @@ public final class ContainerCluster } @Override - public void getConfig(MetricDefaultsConfig.Builder builder) { - if (defaultMetricConsumerFactory != null) builder.factory(defaultMetricConsumerFactory); - } - - @Override public void getConfig(ClusterInfoConfig.Builder builder) { builder.clusterId(name); builder.nodeCount(containers.size()); @@ -764,11 +751,6 @@ public final class ContainerCluster return builders; } - public void setDefaultMetricConsumerFactory(MetricDefaultsConfig.Factory.Enum defaultMetricConsumerFactory) { - Objects.requireNonNull(defaultMetricConsumerFactory, "defaultMetricConsumerFactory"); - this.defaultMetricConsumerFactory = defaultMetricConsumerFactory; - } - public boolean isHostedVespa() { return isHostedVespa; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java index d9cb43c1850..009aa454ab3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java @@ -8,8 +8,6 @@ import com.yahoo.container.logging.JSONAccessLog; import com.yahoo.osgi.provider.model.ComponentModel; import edu.umd.cs.findbugs.annotations.Nullable; -import static com.yahoo.container.core.AccessLogConfig.FileHandler.RotateScheme; - /** * @author Tony Vaagenes * @author gjoranv @@ -20,7 +18,6 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL private final String fileNamePattern; private final String rotationInterval; - private final RotateScheme.Enum rotationScheme; private final Boolean compression; private final boolean isHostedVespa; private final String symlinkName; @@ -29,7 +26,7 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL { this(logType, String.format("logs/vespa/qrs/%s.%s.%s", capitalize(logType.name()), clusterName, "%Y%m%d%H%M%S"), - null, null, null, isHostedVespa, + null, null, isHostedVespa, capitalize(logType.name()) + "." + clusterName); } @@ -40,7 +37,6 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL public AccessLogComponent(AccessLogType logType, String fileNamePattern, String rotationInterval, - RotateScheme.Enum rotationScheme, Boolean compressOnRotation, boolean isHostedVespa, String symlinkName) @@ -48,7 +44,6 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL super(new ComponentModel(accessLogClass(logType), null, "container-core", null)); this.fileNamePattern = fileNamePattern; this.rotationInterval = rotationInterval; - this.rotationScheme = rotationScheme; this.compression = compressOnRotation; this.isHostedVespa = isHostedVespa; this.symlinkName = symlinkName; @@ -81,8 +76,6 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL builder.pattern(fileNamePattern); if (rotationInterval != null) builder.rotation(rotationInterval); - if (rotationScheme != null) - builder.rotateScheme(rotationScheme); if (symlinkName != null) builder.symlink(symlinkName); if (compression != null) { @@ -97,18 +90,4 @@ public final class AccessLogComponent extends SimpleComponent implements AccessL public String getFileNamePattern() { return fileNamePattern; } - - public static final RotateScheme.Enum rotateScheme(@Nullable String name) { - if (name == null) - return null; - - switch (name) { - case "date": - return RotateScheme.Enum.DATE; - case "sequence": - return RotateScheme.Enum.SEQUENCE; - default: - throw new IllegalArgumentException("Invalid rotation scheme " + name); - } - } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java index a9d3ec0e5a2..73fc9c0cf41 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java @@ -5,13 +5,10 @@ import com.yahoo.component.ComponentId; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.osgi.provider.model.ComponentModel; -import com.yahoo.text.XML; import com.yahoo.vespa.model.container.component.SimpleComponent; -import com.yahoo.vespa.model.container.http.ssl.LegacySslProvider; -import org.w3c.dom.Element; +import com.yahoo.vespa.model.container.http.ssl.DummySslProvider; import static com.yahoo.component.ComponentSpecification.fromString; -import static com.yahoo.jdisc.http.ConnectorConfig.Ssl.KeyStoreType; /** * @author Einar M R Rosenvinge @@ -22,16 +19,14 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig private final String name; private final int listenPort; - private final Element legacyConfig; private final SimpleComponent sslProviderComponent; public ConnectorFactory(String name, int listenPort) { - this(name, listenPort, null, new LegacySslProvider(name)); + this(name, listenPort, new DummySslProvider(name)); } public ConnectorFactory(String name, int listenPort, - Element legacyConfig, SimpleComponent sslProviderComponent) { super(new ComponentModel( new BundleInstantiationSpecification(new ComponentId(name), @@ -39,7 +34,6 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig fromString("jdisc_http_service")))); this.name = name; this.listenPort = listenPort; - this.legacyConfig = legacyConfig; this.sslProviderComponent = sslProviderComponent; addChild(sslProviderComponent); inject(sslProviderComponent); @@ -47,7 +41,6 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig @Override public void getConfig(ConnectorConfig.Builder connectorBuilder) { - configureWithLegacyHttpConfig(legacyConfig, connectorBuilder); connectorBuilder.listenPort(listenPort); connectorBuilder.name(name); ((ConnectorConfig.Producer)sslProviderComponent).getConfig(connectorBuilder); @@ -61,97 +54,4 @@ public class ConnectorFactory extends SimpleComponent implements ConnectorConfig return listenPort; } - // TODO Remove support for legacy config in Vespa 7 - @Deprecated - private static void configureWithLegacyHttpConfig(Element legacyConfig, ConnectorConfig.Builder connectorBuilder) { - if (legacyConfig != null) { - { - Element tcpKeepAliveEnabled = XML.getChild(legacyConfig, "tcpKeepAliveEnabled"); - if (tcpKeepAliveEnabled != null) { - connectorBuilder.tcpKeepAliveEnabled(Boolean.valueOf(XML.getValue(tcpKeepAliveEnabled).trim())); - } - } - { - Element tcpNoDelayEnabled = XML.getChild(legacyConfig, "tcpNoDelayEnabled"); - if (tcpNoDelayEnabled != null) { - connectorBuilder.tcpNoDelay(Boolean.valueOf(XML.getValue(tcpNoDelayEnabled).trim())); - } - } - { - Element tcpListenBacklogLength = XML.getChild(legacyConfig, "tcpListenBacklogLength"); - if (tcpListenBacklogLength != null) { - connectorBuilder.acceptQueueSize(Integer.parseInt(XML.getValue(tcpListenBacklogLength).trim())); - } - } - { - Element idleConnectionTimeout = XML.getChild(legacyConfig, "idleConnectionTimeout"); - if (idleConnectionTimeout != null) { - connectorBuilder.idleTimeout(Double.parseDouble(XML.getValue(idleConnectionTimeout).trim())); - } - } - { - Element soLinger = XML.getChild(legacyConfig, "soLinger"); - if (soLinger != null) { - - connectorBuilder.soLingerTime(Double.parseDouble(XML.getValue(soLinger).trim())); - } - } - { - Element sendBufferSize = XML.getChild(legacyConfig, "sendBufferSize"); - if (sendBufferSize != null) { - connectorBuilder.outputBufferSize(Integer.parseInt(XML.getValue(sendBufferSize).trim())); - } - } - { - Element maxHeaderSize = XML.getChild(legacyConfig, "maxHeaderSize"); - if (maxHeaderSize != null) { - connectorBuilder.headerCacheSize(Integer.parseInt(XML.getValue(maxHeaderSize).trim())); - } - } - - Element ssl = XML.getChild(legacyConfig, "ssl"); - Element sslEnabled = XML.getChild(ssl, "enabled"); - if (ssl != null && sslEnabled != null && Boolean.parseBoolean(XML.getValue(sslEnabled).trim())) { - ConnectorConfig.Ssl.Builder sslBuilder = new ConnectorConfig.Ssl.Builder(); - sslBuilder.enabled(true); - { - Element keyStoreType = XML.getChild(ssl, "keyStoreType"); - if (keyStoreType != null) { - sslBuilder.keyStoreType(KeyStoreType.Enum.valueOf(XML.getValue(keyStoreType).trim())); - } - } - { - Element keyStorePath = XML.getChild(ssl, "keyStorePath"); - if (keyStorePath != null) { - sslBuilder.keyStorePath(XML.getValue(keyStorePath).trim()); - } - } - { - Element trustStorePath = XML.getChild(ssl, "trustStorePath"); - if (trustStorePath != null) { - sslBuilder.trustStorePath(XML.getValue(trustStorePath).trim()); - } - } - { - Element keyDBKey = XML.getChild(ssl, "keyDBKey"); - if (keyDBKey != null) { - sslBuilder.keyDbKey(XML.getValue(keyDBKey).trim()); - } - } - { - Element algorithm = XML.getChild(ssl, "algorithm"); - if (algorithm != null) { - sslBuilder.sslKeyManagerFactoryAlgorithm(XML.getValue(algorithm).trim()); - } - } - { - Element protocol = XML.getChild(ssl, "protocol"); - if (protocol != null) { - sslBuilder.protocol(XML.getValue(protocol).trim()); - } - } - connectorBuilder.ssl(sslBuilder); - } - } - } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java index 4eed3628bfd..9e85a889075 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.container.http; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.container.jdisc.config.HttpServerConfig; import com.yahoo.jdisc.http.ServerConfig; import com.yahoo.vespa.model.container.component.chain.Chain; import com.yahoo.vespa.model.container.component.chain.ChainedComponent; @@ -19,8 +18,7 @@ import java.util.Optional; * * @author Tony Vaagenes */ -public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> - implements HttpServerConfig.Producer, ServerConfig.Producer { +public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> implements ServerConfig.Producer { public static class Binding { public final ComponentSpecification filterId; @@ -89,12 +87,6 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> } @Override - public void getConfig(HttpServerConfig.Builder builder) { - for (Binding binding: bindings) - builder.filter(filterBindings(binding)); - } - - @Override public void getConfig(ServerConfig.Builder builder) { for (final Binding binding : bindings) { builder.filter( @@ -104,14 +96,6 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> } } - static HttpServerConfig.Filter.Builder filterBindings(Binding binding) { - HttpServerConfig.Filter.Builder builder = new HttpServerConfig.Filter.Builder(); - builder.id(binding.filterId.stringValue()). - binding(binding.binding); - return builder; - } - - @Override public void validate() throws Exception { validate(bindings); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/LegacySslProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/DummySslProvider.java index 7ab553c45b9..4e2ee61f33f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/LegacySslProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/DummySslProvider.java @@ -4,25 +4,22 @@ package com.yahoo.vespa.model.container.http.ssl; import com.yahoo.component.ComponentId; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.jdisc.http.ConnectorConfig; -import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; -import com.yahoo.jdisc.http.ssl.impl.LegacySslContextFactoryProvider; +import com.yahoo.jdisc.http.ssl.ThrowingSslContextFactoryProvider; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.component.SimpleComponent; import static com.yahoo.component.ComponentSpecification.fromString; /** - * Provides a legacy implementation of {@link SslContextFactoryProvider} to be injected into non-ssl connectors and connectors using legacy ssl config override - * * @author bjorncs */ -public class LegacySslProvider extends SimpleComponent implements ConnectorConfig.Producer { +public class DummySslProvider extends SimpleComponent implements ConnectorConfig.Producer { - public static final String COMPONENT_ID_PREFIX = "legacy-ssl-provider@"; - public static final String COMPONENT_CLASS = LegacySslContextFactoryProvider.class.getName(); + public static final String COMPONENT_ID_PREFIX = "dummy-ssl-provider@"; + public static final String COMPONENT_CLASS = ThrowingSslContextFactoryProvider.class.getName(); public static final String COMPONENT_BUNDLE = "jdisc_http_service"; - public LegacySslProvider(String serverName) { + public DummySslProvider(String serverName) { super(new ComponentModel( new BundleInstantiationSpecification(new ComponentId(COMPONENT_ID_PREFIX + serverName), fromString(COMPONENT_CLASS), @@ -30,7 +27,5 @@ public class LegacySslProvider extends SimpleComponent implements ConnectorConfi } @Override - public void getConfig(ConnectorConfig.Builder builder) { - - } -} + public void getConfig(ConnectorConfig.Builder builder) {} +}
\ No newline at end of file diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java index 02840ff380b..4c41aaa504c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java @@ -10,37 +10,24 @@ import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.http.ConnectorFactory; import com.yahoo.vespa.model.container.http.ssl.CustomSslProvider; import com.yahoo.vespa.model.container.http.ssl.DefaultSslProvider; -import com.yahoo.vespa.model.container.http.ssl.LegacySslProvider; +import com.yahoo.vespa.model.container.http.ssl.DummySslProvider; import org.w3c.dom.Element; import java.util.Optional; -import java.util.logging.Level; -import java.util.logging.Logger; /** * @author Einar M R Rosenvinge * @author mortent */ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuilder<ConnectorFactory> { - private static final Logger log = Logger.getLogger(JettyConnectorBuilder.class.getName()); @Override protected ConnectorFactory doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element serverSpec) { String name = XmlHelper.getIdString(serverSpec); int port = HttpBuilder.readPort(serverSpec, deployState.isHosted(), deployState.getDeployLogger()); - Element legacyServerConfig = XML.getChild(serverSpec, "config"); - if (legacyServerConfig != null) { - String configName = legacyServerConfig.getAttribute("name"); - if (configName.equals("container.jdisc.config.http-server")) { - deployState.getDeployLogger().log(Level.WARNING, "The config 'container.jdisc.config.http-server' is deprecated and will be removed in a later version of Vespa." - + " Please use 'jdisc.http.connector' instead, see http://docs.vespa.ai/documentation/jdisc/http-server-and-filters.html#configuring-jetty-server"); - } else { - legacyServerConfig = null; - } - } SimpleComponent sslProviderComponent = getSslConfigComponents(name, serverSpec); - return new ConnectorFactory(name, port, legacyServerConfig, sslProviderComponent); + return new ConnectorFactory(name, port, sslProviderComponent); } SimpleComponent getSslConfigComponents(String serverName, Element serverSpec) { @@ -63,7 +50,7 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil String bundle = sslProviderConfigurator.getAttribute("bundle"); return new CustomSslProvider(serverName, className, bundle); } else { - return new LegacySslProvider(serverName); + return new DummySslProvider(serverName); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 3a7b7864554..6c537e359d1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -8,7 +8,6 @@ import com.yahoo.prelude.semantics.SemanticRulesConfig; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.ContainerSubsystem; -import com.yahoo.vespa.model.container.search.searchchain.HttpProvider; import com.yahoo.vespa.model.container.search.searchchain.LocalProvider; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; import com.yahoo.search.config.IndexInfoConfig; @@ -110,18 +109,6 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> if (pageTemplates!=null) pageTemplates.getConfig(builder); } - public int totalCacheSizeMb() { - return totalHttpProviderCacheSize(); - } - - private int totalHttpProviderCacheSize() { - int totalCacheSizeMb = 0; - for (HttpProvider provider: getChains().httpProviders()) - totalCacheSizeMb += provider.cacheSizeMB(); - - return totalCacheSizeMb; - } - @Override public void getConfig(IndexInfoConfig.Builder builder) { for (AbstractSearchCluster sc : systems) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/GenericProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/GenericProvider.java new file mode 100644 index 00000000000..a82d2da2a6b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/GenericProvider.java @@ -0,0 +1,27 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.search.federation.ProviderConfig; +import com.yahoo.search.searchchain.model.federation.FederationOptions; + +/** + * A generic provider, used when no type is given. + * + * @author Tony Vaagenes + */ +public class GenericProvider extends Provider implements ProviderConfig.Producer { + + /* + * Config producer for the contained http searcher.. + */ + @SuppressWarnings("deprecation") + public GenericProvider(ChainSpecification specWithoutInnerSearchers, FederationOptions federationOptions) { + super(specWithoutInnerSearchers, federationOptions); + } + + @Override + public void getConfig(ProviderConfig.Builder builder) { + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProvider.java deleted file mode 100644 index 62b2f2ccbc6..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProvider.java +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.search.searchchain; - -import com.yahoo.binaryprefix.BinaryPrefix; -import com.yahoo.binaryprefix.BinaryScaledAmount; -import com.yahoo.component.chain.model.ChainSpecification; -import com.yahoo.search.cache.QrBinaryCacheConfig; -import com.yahoo.search.cache.QrBinaryCacheRegionConfig; -import com.yahoo.search.federation.ProviderConfig; -import com.yahoo.search.searchchain.model.federation.FederationOptions; - -import java.util.ArrayList; -import java.util.List; - -import static com.yahoo.search.federation.ProviderConfig.Node; -import static com.yahoo.search.federation.ProviderConfig.Yca; - - -/** - * A provider containing a http searcher. - * - * @author Tony Vaagenes - */ -public class HttpProvider extends Provider implements ProviderConfig.Producer, - QrBinaryCacheConfig.Producer, - QrBinaryCacheRegionConfig.Producer { - - @SuppressWarnings("deprecation") - private final com.yahoo.search.searchchain.model.federation.HttpProviderSpec providerSpec; - - /* - * Config producer for the contained http searcher.. - */ - @SuppressWarnings("deprecation") - public HttpProvider(ChainSpecification specWithoutInnerSearchers, FederationOptions federationOptions, com.yahoo.search.searchchain.model.federation.HttpProviderSpec providerSpec) { - super(specWithoutInnerSearchers, federationOptions); - this.providerSpec = providerSpec; - } - - @Override - public void getConfig(ProviderConfig.Builder builder) { - if (providerSpec.path != null) - builder.path(providerSpec.path); - if (providerSpec.connectionParameters.readTimeout != null) - builder.readTimeout(providerSpec.connectionParameters.readTimeout ); - if (providerSpec.connectionParameters.connectionTimeout != null) - builder.connectionTimeout(providerSpec.connectionParameters.connectionTimeout); - if (providerSpec.connectionParameters.connectionPoolTimeout != null) - builder.connectionPoolTimeout(providerSpec.connectionParameters.connectionPoolTimeout); - if (providerSpec.connectionParameters.retries != null) - builder.retries(providerSpec.connectionParameters.retries); - - builder.node(getNodes(providerSpec.nodes)); - - if (providerSpec.ycaApplicationId != null) { - builder.yca(getCertificate(providerSpec)); - } - } - - @SuppressWarnings("deprecation") - private static Yca.Builder getCertificate(com.yahoo.search.searchchain.model.federation.HttpProviderSpec providerSpec) { - Yca.Builder certificate = new Yca.Builder() - .applicationId(providerSpec.ycaApplicationId); - - if (providerSpec.ycaProxy != null) { - certificate.useProxy(true); - if (providerSpec.ycaProxy.host != null) { - certificate.host(providerSpec.ycaProxy.host) - .port(providerSpec.ycaProxy.port); - } - } - if (providerSpec.ycaCertificateTtl != null) certificate.ttl(providerSpec.ycaCertificateTtl); - if (providerSpec.ycaRetryWait != null) certificate.ttl(providerSpec.ycaRetryWait); - return certificate; - } - - @SuppressWarnings("deprecation") - private static List<Node.Builder> getNodes(List<com.yahoo.search.searchchain.model.federation.HttpProviderSpec.Node> nodeSpecs) { - ArrayList<Node.Builder> nodes = new ArrayList<>(); - for (com.yahoo.search.searchchain.model.federation.HttpProviderSpec.Node node : nodeSpecs) { - nodes.add( - new Node.Builder() - .host(node.host) - .port(node.port)); - } - return nodes; - } - - public int cacheSizeMB() { - return providerSpec.cacheSizeMB != null ? providerSpec.cacheSizeMB : 0; - } - - @Override - public void getConfig(QrBinaryCacheConfig.Builder builder) { - builder.cache_size(cacheSizeMB()); - } - - @Override - public void getConfig(QrBinaryCacheRegionConfig.Builder builder) { - builder.region_size(cacheSizeMB()); - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProviderSearcher.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProviderSearcher.java deleted file mode 100644 index 8c74496c895..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProviderSearcher.java +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.search.searchchain; - -import com.yahoo.component.chain.model.ChainedComponentModel; - -/** - * @author Tony Vaagenes - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated -public class HttpProviderSearcher extends Searcher<ChainedComponentModel> { - - public HttpProviderSearcher(ChainedComponentModel model) { - super(model); - } - - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java index 42d81e19496..24ff885f131 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java @@ -74,11 +74,6 @@ public class SearchChains extends Chains<SearchChain> { return CollectionUtil.filter(allChains().allComponents(), LocalProvider.class); } - - public Collection<HttpProvider> httpProviders() { - return CollectionUtil.filter(allChains().allComponents(), HttpProvider.class); - } - /* * If searchChain is a provider, its sources must already have been attached. */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java index ddea1a189bc..0eb34ead7bf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java @@ -59,7 +59,6 @@ public class AccessLogBuilder { accessLogType, fileNamePattern(spec), rotationInterval(spec), - rotationScheme(spec), compressOnRotation(spec), isHostedVespa, symlinkName(spec)); @@ -74,10 +73,6 @@ public class AccessLogBuilder { return (compress.isEmpty() ? null : Boolean.parseBoolean(compress)); } - private AccessLogConfig.FileHandler.RotateScheme.Enum rotationScheme(Element spec) { - return AccessLogComponent.rotateScheme(nullIfEmpty(spec.getAttribute("rotationScheme"))); - } - private String rotationInterval(Element spec) { return nullIfEmpty(spec.getAttribute("rotationInterval")); } @@ -106,7 +101,7 @@ public class AccessLogBuilder { AccessLogTypeLiteral typeLiteral = getOptionalAttribute(accessLogSpec, "type"). map(AccessLogTypeLiteral::fromAttributeValue). - orElse(AccessLogTypeLiteral.VESPA); + orElse(AccessLogTypeLiteral.JSON); AccessLogType logType = logTypeFor(typeLiteral); if (logType == null) { return Optional.empty(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java index 13dfc3ae745..11d467a61d5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java @@ -57,7 +57,6 @@ public class BundleMapper { bundleFromClass.put("com.yahoo.prelude.fastsearch.VespaBackEndSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.querytransform.CJKSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.querytransform.CollapsePhraseSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.querytransform.IndexCombinatorSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.querytransform.LiteralBoostSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.querytransform.NoRankingSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.querytransform.NonPhrasingSearcher", searchAndDocprocBundle); @@ -66,15 +65,12 @@ public class BundleMapper { bundleFromClass.put("com.yahoo.prelude.querytransform.RecallSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.querytransform.StemmingSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.searcher.BlendingSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.DocumentSourceSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.searcher.FieldCollapsingSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.searcher.FillSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.searcher.JSONDebugSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.searcher.JuniperSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.searcher.MultipleResultsSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.searcher.PosSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.QuerySnapshotSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.QueryValidatingSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.searcher.QuotingSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.searcher.ValidateSortingSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.semantics.SemanticSearcher", searchAndDocprocBundle); @@ -110,10 +106,8 @@ public class BundleMapper { bundleFromClass.put("com.yahoo.search.query.rewrite.rewriters.NameRewriter", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.search.querytransform.AllLowercasingSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.search.querytransform.DefaultPositionSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.querytransform.LegacyCombinator", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.search.querytransform.LowercasingSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.search.querytransform.NGramSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.search.querytransform.QueryCombinator", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.search.querytransform.VespaLowercasingSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.search.rendering.Renderer", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.search.rendering.SectionedRenderer", searchAndDocprocBundle); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index b5c1b1e1496..4401bfa08dd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -25,7 +25,6 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Rotation; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; -import com.yahoo.container.jdisc.config.MetricDefaultsConfig; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.searchdefinition.derived.RankProfileList; import com.yahoo.text.XML; @@ -174,7 +173,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addDefaultHandlers(cluster); addStatusHandlers(cluster, context); - setDefaultMetricConsumerFactory(cluster); addHttp(deployState, spec, cluster); @@ -184,7 +182,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addClientProviders(deployState, spec, cluster); addServerProviders(deployState, spec, cluster); - addLegacyFilters(deployState, spec, cluster); // TODO: Remove for Vespa 7 addAthensCopperArgos(cluster, context); // Must be added after nodes. } @@ -256,10 +253,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addConfiguredComponents(deployState, cluster, spec, "component"); } - private void setDefaultMetricConsumerFactory(ContainerCluster cluster) { - cluster.setDefaultMetricConsumerFactory(MetricDefaultsConfig.Factory.Enum.STATE_MONITOR); - } - private void addDefaultHandlers(ContainerCluster cluster) { addDefaultHandlersExceptStatus(cluster); } @@ -301,21 +294,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addConfiguredComponents(deployState, cluster, spec, "server"); } - private void addLegacyFilters(DeployState deployState, Element spec, ContainerCluster cluster) { - for (Component component : buildLegacyFilters(deployState, cluster, spec)) { - cluster.addComponent(component); - } - } - - private List<Component> buildLegacyFilters(DeployState deployState, AbstractConfigProducer ancestor, Element spec) { - List<Component> components = new ArrayList<>(); - - for (Element node : XML.getChildren(spec, "filter")) { - components.add(new DomFilterBuilder().build(deployState, ancestor, node)); - } - return components; - } - private void addAccessLogs(DeployState deployState, ContainerCluster cluster, Element spec) { List<Element> accessLogElements = getAccessLogElements(spec); @@ -464,7 +442,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } else if ((zone.system() == SystemName.dev) || isHostedVespa) { return null; } else { - return ContainerCluster.CMS; + return ContainerCluster.G1GC; } } private static String getJvmOptions(ContainerCluster cluster, Element nodesElement, DeployLogger deployLogger) { @@ -480,7 +458,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { jvmOptions = nodesElement.getAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME); if (incompatibleGCOptions(jvmOptions)) { deployLogger.log(Level.WARNING, "You need to move out your GC related options from 'jvmargs' to 'jvm-gc-options'"); - cluster.setJvmGCOptions(ContainerCluster.CMS); + cluster.setJvmGCOptions(ContainerCluster.G1GC); } } return jvmOptions; @@ -849,7 +827,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { /** - * Disallow renderers named "DefaultRenderer" or "JsonRenderer" + * Disallow renderers named "XmlRenderer" or "JsonRenderer" */ private static void validateRendererElement(Element element) { String idAttr = element.getAttribute("id"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index 8d0824f89e5..79619ac0c1f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -238,10 +238,8 @@ public class Content extends ConfigModel { IndexedSearchCluster indexedSearchCluster = content.getCluster().getSearch().getIndexed(); if (indexedSearchCluster.hasExplicitIndexingCluster()) { setExistingIndexingCluster(indexedSearchCluster, content.containers); - } else if (content.isHosted) { - setContainerAsIndexingCluster(indexedSearchCluster, content, modelContext, root); } else { - createImplicitIndexingCluster(indexedSearchCluster, content, modelContext, root); + setContainerAsIndexingCluster(indexedSearchCluster, content, modelContext, root); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index fb3af7f3652..67e33d7b87a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -716,17 +716,6 @@ public class ContentCluster extends AbstractConfigProducer implements addedmetrics("*"). removedtags("thread"). tags("disk")); - - Map<String, MetricsConsumer> consumers = getRoot().getAdmin().getLegacyUserMetricsConsumers(); - if (consumers != null) { - for (Map.Entry<String, MetricsConsumer> e : consumers.entrySet()) { - MetricsmanagerConfig.Consumer.Builder b = getMetricBuilder(e.getKey(), builder); - for (Metric m : e.getValue().getMetrics().values()) { - b.addedmetrics(m.name); - } - } - } - } private static final String DEFAULT_BUCKET_SPACE = "default"; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/package-info.java index e5f76dd398e..f88eda4e7a2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/package-info.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/package-info.java @@ -99,7 +99,7 @@ com.yahoo.config.model.producer.AbstractConfigProducer </p> - <h3><a name="plugin_loading">Plugin Loading</a></h3> + <h3 id="plugin_loading">Plugin Loading</h3> <p>Each highest-level node in the setup file from the user's application specification corresponds to a {@link @@ -138,7 +138,7 @@ com.yahoo.config.model.producer.AbstractConfigProducer </ul> - <h3><a name="port_allocation">Port Allocation</a></h3> + <h3 id="port_allocation">Port Allocation</h3> <p>Each {@link com.yahoo.vespa.model.Host Host} has an available dynamic port range running from {@link diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java index 2e3c0681f75..8e8192b74fa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java @@ -282,14 +282,12 @@ public class Tuning extends AbstractConfigProducer implements PartitionsConfig.P public static class LogStore { public Long maxFileSize = null; - public Integer numThreads = null; public Component chunk = null; public Double minFileSizeFactor = null; public void getConfig(ProtonConfig.Summary.Log.Builder log) { if (maxFileSize!=null) log.maxfilesize(maxFileSize); if (minFileSizeFactor!=null) log.minfilesizefactor(minFileSizeFactor); - if (numThreads != null) log.numthreads(numThreads); if (chunk != null) { chunk.getConfig(log.chunk); } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index e50cbabeb9f..883dfc7ac72 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -114,7 +114,7 @@ public class SDParser { */ @SuppressWarnings("deprecation") private IndexingOperation newIndexingOperation(boolean multiline) throws ParseException { - return newIndexingOperation(multiline, new SimpleLinguistics(false)); + return newIndexingOperation(multiline, new SimpleLinguistics()); } /** @@ -221,7 +221,6 @@ TOKEN : | < WEIGHT: "weight" > | < TYPE: "type" > | < INDEX: "index" > -| < RISE: "rise" > | < MTOKEN: "token" > | < TEXT: "text" > | < WORD: "word" > @@ -298,8 +297,6 @@ TOKEN : | < MUTABLE: "mutable" > | < FASTSEARCH: "fast-search" > | < HUGE: "huge" > -| < PREFETCH: "prefetch" > -| < NOPREFETCH: "no-prefetch" > | < TENSOR_TYPE: "tensor(" (~["(",")"])+ ")" > | < TENSOR_VALUE_SL: "value" (" ")* ":" (" ")* ("{"<BRACE_SL_LEVEL_1>) ("\n")? > | < TENSOR_VALUE_ML: "value" (<SEARCHLIB_SKIP>)? "{" (["\n"," "])* ("{"<BRACE_ML_LEVEL_1>) (["\n"," "])* "}" ("\n")? > @@ -1209,15 +1206,6 @@ Object attributeSetting(FieldOperationContainer field, AttributeOperation attrib | <MUTABLE> { attribute.setMutable(true); } | <ENABLEBITVECTORS> { attribute.setEnableBitVectors(true); } | <ENABLEONLYBITVECTOR> { attribute.setEnableOnlyBitVector(true); } - - | <NOPREFETCH> { - deployLogger.log(Level.WARNING, field + ": 'attribute : no-prefetch' is deprecated and has no effect."); - attribute.setPrefetch(false); - } - | <PREFETCH> { - deployLogger.log(Level.WARNING, field + ": 'attribute : prefetch' is deprecated and has no effect."); - attribute.setPrefetch(true); - } | sorting(field, attributeName) | <ALIAS> { String alias; String aliasedName=attributeName; } [aliasedName = identifier()] <COLON> alias = identifier() { attribute.setDoAlias(true); @@ -1517,8 +1505,6 @@ void body(SDField field) : { } <BODY> { deployLogger.log(Level.WARNING, field + ": 'header/body' is deprecated and has no effect."); - field.setHeader(false); - field.setHeaderOrBodyDefined(true); } } @@ -1532,8 +1518,6 @@ void header(SDField field) : { } <HEADER> { deployLogger.log(Level.WARNING, field + ": 'header/body' is deprecated and has no effect."); - field.setHeader(true); - field.setHeaderOrBodyDefined(true); } } @@ -1715,8 +1699,7 @@ Object documentSummary(Search search) : DocumentSummary summary; } { - ( ( <DOCUMENTSUMMARY> | - <SUMMARY> { deployLogger.log(Level.WARNING, "Directive 'summary' is deprecated, use 'document-summary' instead."); } ) // TODO: Remove on Vespa 7 + ( <DOCUMENTSUMMARY> name = identifier() { search.addSummary(summary = new DocumentSummary(name)); } lbrace() ( @@ -1793,11 +1776,6 @@ Object indexBody(IndexOperation index) : ( <PREFIX> { index.setPrefix(true); } | <ALIAS> <COLON> str = identifier() { index.addAlias(str); } | <STEMMING> <COLON> str = identifier() { index.setStemming(str); } - | <RISE> { - if (true) throw new ParseException("'index:rise' is no longer an option. Use 'indexing:attribute' instead. " + - "If it is a weighted set field you should also add 'attribute:fast-search'." + - "This change will require refeeding."); - } | <ARITY> <COLON> arity = integer() { index.setArity(arity); } | <LOWERBOUND> <COLON> num = consumeLong() { index.setLowerBound(num); } | <UPPERBOUND> <COLON> num = consumeLong() { index.setUpperBound(num); } @@ -2551,14 +2529,12 @@ String identifier() : { } | <MUTABLE> | <NEVER> | <NONE> - | <NOPREFETCH> | <NORMAL> | <NORMALIZING> | <OFF> | <ON> | <ONDEMAND> | <ORDER> - | <PREFETCH> | <PREFIX> | <PRIMARY> | <PROPERTIES> @@ -2573,7 +2549,6 @@ String identifier() : { } | <REFERENCE> | <REMOVEIFZERO> | <RERANKCOUNT> - | <RISE> | <SEARCH> | <SECONDARY> | <SECONDPHASE> diff --git a/config-model/src/main/resources/schema/.gitignore b/config-model/src/main/resources/schema/.gitignore index b60c856f91a..e8bf39e2289 100644 --- a/config-model/src/main/resources/schema/.gitignore +++ b/config-model/src/main/resources/schema/.gitignore @@ -1,2 +1,2 @@ -*.rng *.xsd +*.rng diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc index 37cd6476399..3729f3ab57c 100644 --- a/config-model/src/main/resources/schema/admin.rnc +++ b/config-model/src/main/resources/schema/admin.rnc @@ -10,8 +10,8 @@ AdminV2 = (ConfigServer | ConfigServers)? & FileDistribution? & AdminSlobroks? & - (LegacyAdminMonitoring | AdminMonitoring)? & - (LegacyMetricConsumers | Metrics)? & + AdminMonitoring? & + Metrics? & ClusterControllers? & LogForwarding? } @@ -29,8 +29,8 @@ AdminV4 = AdminV4Slobroks? & AdminV4LogServers? & GenericConfig* & - (LegacyAdminMonitoring | AdminMonitoring)? & - (LegacyMetricConsumers | Metrics)? & + AdminMonitoring? & + Metrics? & LogForwarding? } @@ -58,14 +58,6 @@ AdminMonitoring = attribute systemname { xsd:string }? } -# TODO: Alternative name for the 'monitoring' tag. Remove on Vespa 7 -LegacyAdminMonitoring = - element yamas { - attribute interval { xsd:int }?, - attribute systemname { xsd:string }? - } - - ConfigServer = element configserver { service.attlist } @@ -94,17 +86,6 @@ Metrics = element metrics { }+ } -# TODO: Remove on Vespa 7 -LegacyMetricConsumers = element metric-consumers { - element consumer { - attribute name { xsd:string { pattern = "[a-zA-Z][_a-zA-Z0-9]*" } }& - element metric { - attribute name{xsd:Name} & - attribute output-name{xsd:Name}? - }+ - }+ -} - ClusterControllers = element cluster-controllers { attribute standalone-zookeeper { xsd:string }? & element cluster-controller { diff --git a/config-model/src/main/resources/schema/clients-v2.rnc b/config-model/src/main/resources/schema/clients-v2.rnc index 1076d332dd5..5e50135edee 100644 --- a/config-model/src/main/resources/schema/clients-v2.rnc +++ b/config-model/src/main/resources/schema/clients-v2.rnc @@ -1,8 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. Clients20 = attribute version { "2.0" } & - LoadTypes? & - Spoolers20? + LoadTypes? LoadTypes = element load-types { element type { @@ -11,38 +10,6 @@ LoadTypes = element load-types { }* } -Spoolers20 = element spoolers { - attribute jvmargs { text }? & - SpoolMaster20* & - FeederOptions20? & - Spooler20+ -} - -Spooler20 = element spooler { - service.attlist? & - attribute id { xsd:string }? & - attribute directory { xsd:string }? & - attribute keepsuccess { xsd:boolean }? & - attribute maxfailuresize { xsd:integer }? & - attribute maxfatalfailuresize { xsd:integer }? & - attribute threads { xsd:integer }? & - attribute maxretries { xsd:integer }? & - FeederOptionsOpts20? & - element parsers { - element parser { - attribute type { text } & - element parameter { - attribute key { xsd:string { minLength = "1" } } & - attribute value { xsd:string { minLength = "1" } } - }* - }+ - }? -} - -SpoolMaster20 = element spoolmaster { - service.attlist? -} - FeederOptions20 = element feederoptions { FeederOptionsOpts20 & DocProcChain? diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 5dbcffce736..a30f61809ad 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -24,7 +24,6 @@ ContainerServices = Client* & Server* & Http? & - HttpFilter? & AccessLog* & SecretStore? & GenericConfig* @@ -73,8 +72,7 @@ AccessLog = element accesslog { attribute compressOnRotation { xsd:boolean }? & attribute symlinkName { string }? & - attribute rotationInterval { string }? & - attribute rotationScheme { string "date" | string "sequence" }? + attribute rotationInterval { string }? } SecretStore = element secret-store { diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc index c0313cd50ef..ae2af6618d7 100644 --- a/config-model/src/main/resources/schema/content.rnc +++ b/config-model/src/main/resources/schema/content.rnc @@ -302,7 +302,6 @@ Tuning = element tuning { element maxage { xsd:nonNegativeInteger }? }? & element transactionlog { - element maxentries { xsd:nonNegativeInteger }? & element maxsize { xsd:nonNegativeInteger }? }? & element conservative { @@ -336,16 +335,13 @@ Tuning = element tuning { element maxsize { xsd:nonNegativeInteger }? & element maxsize-percent { xsd:double { minInclusive = "0.0" maxInclusive = "50.0" } }? & element initialentries { xsd:nonNegativeInteger }? & - element maxentries { xsd:nonNegativeInteger }? & TuningCompression? }? & element logstore { element maxfilesize { xsd:nonNegativeInteger }? & - element maxdiskbloatfactor { xsd:double { minInclusive = "0.0" } }? & element minfilesizefactor { xsd:double { minInclusive = "0.10" maxInclusive = "1.0" } }? & element chunk { element maxsize { xsd:nonNegativeInteger }? & - element maxentries { xsd:nonNegativeInteger }? & TuningCompression? }? }? diff --git a/config-model/src/main/resources/schema/deployment.rnc b/config-model/src/main/resources/schema/deployment.rnc index 4fc34862365..b2c3dcf7b5a 100644 --- a/config-model/src/main/resources/schema/deployment.rnc +++ b/config-model/src/main/resources/schema/deployment.rnc @@ -12,7 +12,6 @@ start = element deployment { Notifications? & Test? & Staging? & - BlockUpgrade* & Prod* } @@ -28,12 +27,6 @@ BlockChange = element block-change { attribute time-zone { xsd:string }? } -BlockUpgrade = element block-upgrade { # Legacy name - remove on Vespa 7 - attribute days { xsd:string } & - attribute hours { xsd:string } & - attribute time-zone { xsd:string }? -} - Notifications = element notifications { attribute when { xsd:string }? & Email* diff --git a/config-model/src/main/resources/schema/services.rnc b/config-model/src/main/resources/schema/services.rnc index c4eb7435ca6..fe728506124 100644 --- a/config-model/src/main/resources/schema/services.rnc +++ b/config-model/src/main/resources/schema/services.rnc @@ -11,7 +11,6 @@ include "legacygenericcluster.rnc" start = element services { attribute version { "1.0" }? & - attribute major-version { text }? & # TODO: Remove on Vespa 7 attribute application-type { "hosted-infrastructure" }? & LegacyGenericCluster* & GenericCluster* & diff --git a/config-model/src/test/cfg/admin/metricconfig/services.xml b/config-model/src/test/cfg/admin/metricconfig/services.xml index a2a03ca0b08..b4858b1b1bf 100644 --- a/config-model/src/test/cfg/admin/metricconfig/services.xml +++ b/config-model/src/test/cfg/admin/metricconfig/services.xml @@ -5,20 +5,20 @@ <adminserver hostalias="node1"/> <logserver hostalias="node1"/> <monitoring interval="60"/> - <metric-consumers> - <consumer name="fooConsumer"> - <metric name="some.foo.metric" output-name="someFooMetric"/> - <metric name="some.foo.metric2" output-name="someFooMetric2"/> - <metric name="proton.numdocs.average" output-name="tull"/> + <metrics> + <consumer id="fooConsumer"> + <metric id="some.foo.metric" display-name="someFooMetric"/> + <metric id="some.foo.metric2" display-name="someFooMetric2"/> + <metric id="proton.numdocs.average" display-name="tull"/> </consumer> - <consumer name="fooConsumer2"> - <metric name="some.foo.metric3" output-name="someFooMetric3"/> + <consumer id="fooConsumer2"> + <metric id="some.foo.metric3" display-name="someFooMetric3"/> </consumer> - <consumer name="yamas"> - <metric name="vds.distributor.bytesstored.average" output-name="tullball"/> - <metric name="proton.numdocs.average" output-name="overridden"/> + <consumer id="yamas"> + <metric id="vds.distributor.bytesstored.average" display-name="tullball"/> + <metric id="proton.numdocs.average" display-name="overridden"/> </consumer> - </metric-consumers> + </metrics> </admin> <container version="1.0"> diff --git a/config-model/src/test/cfg/application/metricsconfig/services.xml b/config-model/src/test/cfg/application/metricsconfig/services.xml index 868173bc00e..fc3c6e8ce06 100644 --- a/config-model/src/test/cfg/application/metricsconfig/services.xml +++ b/config-model/src/test/cfg/application/metricsconfig/services.xml @@ -3,20 +3,7 @@ <services> <simple version="1.0"> - <simpleservice hostalias="host2"> - <metric-consumers> - <consumer name="yamas"> - <metric name="onlyMine" output-name="user"/> - </consumer> - </metric-consumers> - </simpleservice> - - <simpleservice hostalias="host2"> - <metric-consumers> - <consumer name="yamas"> - <metric name="test" output-name="user"/> - </consumer> - </metric-consumers> - </simpleservice> + <simpleservice hostalias="host2" /> + <simpleservice hostalias="host2" /> </simple> </services> diff --git a/config-model/src/test/cfg/clients/advancedconfig.v2/hosts.xml b/config-model/src/test/cfg/clients/advancedconfig.v2/hosts.xml deleted file mode 100644 index 9eafb2106cb..00000000000 --- a/config-model/src/test/cfg/clients/advancedconfig.v2/hosts.xml +++ /dev/null @@ -1,7 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<hosts> - <host name="localhost"> - <alias>node1</alias> - </host> -</hosts> diff --git a/config-model/src/test/cfg/clients/advancedconfig.v2/searchdefinitions/music.sd b/config-model/src/test/cfg/clients/advancedconfig.v2/searchdefinitions/music.sd deleted file mode 100644 index da9e2d14c55..00000000000 --- a/config-model/src/test/cfg/clients/advancedconfig.v2/searchdefinitions/music.sd +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search music { - document music { - field f1 type string { - indexing: summary | index - # index-to: f1, all - } - field f2 type string { - indexing: summary | index - # index-to: f2, all - } - } -} diff --git a/config-model/src/test/cfg/clients/advancedconfig.v2/services.xml b/config-model/src/test/cfg/clients/advancedconfig.v2/services.xml deleted file mode 100644 index e6102b6f8fd..00000000000 --- a/config-model/src/test/cfg/clients/advancedconfig.v2/services.xml +++ /dev/null @@ -1,70 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<services xmlns=""> - <admin version="2.0"> - <adminserver hostalias="node1" /> - <logserver hostalias="node1" /> - <slobroks> - <slobrok hostalias="node1" /> - </slobroks> - </admin> - - <clients version="2.0"> - <spoolers> - <feederoptions> - <timeout>90</timeout> - </feederoptions> - - <spooler hostalias="node1" keepsuccess="true"> - <abortondocumenterror>false</abortondocumenterror> - <maxpendingbytes>8000</maxpendingbytes> - <parsers> - <parser type="com.yahoo.vespaspooler.XMLFileParser"/> - <parser type="com.yahoo.vespaspooler.MusicFileParser"/> - <parser type="com.yahoo.vespaspooler.MusicParser"> - <parameter key="route" value="default"/> - </parser> - </parsers> - </spooler> - - <spooler hostalias="node1"> - <abortondocumenterror>false</abortondocumenterror> - <maxpendingbytes>4000</maxpendingbytes> - <timeout>50</timeout> - <parsers> - <parser type="com.yahoo.vespaspooler.MusicParser"> - <parameter key="route" value="othercluster"/> - </parser> - </parsers> - </spooler> - - <spooler id="plan9"> - <route>myroute</route> - <mbusport>14064</mbusport> - - <parsers> - <parser type="com.yahoo.vespaspooler.MusicFileParser" /> - </parsers> - </spooler> - - </spoolers> - </clients> - - <container version="1.0"> - <search/> - <nodes> - <node hostalias="node1"/> - </nodes> - </container> - - <content version="1.0" id="music"> - <redundancy>2</redundancy> - <documents> - <document type="music" mode="index"/> - </documents> - <nodes> - <node hostalias="node1" distribution-key="0" /> - </nodes> - </content> - -</services> diff --git a/config-model/src/test/cfg/clients/simpleconfig.v2.docprocv3/hosts.xml b/config-model/src/test/cfg/clients/simpleconfig.v2.docprocv3/hosts.xml deleted file mode 100644 index 9eafb2106cb..00000000000 --- a/config-model/src/test/cfg/clients/simpleconfig.v2.docprocv3/hosts.xml +++ /dev/null @@ -1,7 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<hosts> - <host name="localhost"> - <alias>node1</alias> - </host> -</hosts> diff --git a/config-model/src/test/cfg/clients/simpleconfig.v2.docprocv3/searchdefinitions/music.sd b/config-model/src/test/cfg/clients/simpleconfig.v2.docprocv3/searchdefinitions/music.sd deleted file mode 100644 index da9e2d14c55..00000000000 --- a/config-model/src/test/cfg/clients/simpleconfig.v2.docprocv3/searchdefinitions/music.sd +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search music { - document music { - field f1 type string { - indexing: summary | index - # index-to: f1, all - } - field f2 type string { - indexing: summary | index - # index-to: f2, all - } - } -} diff --git a/config-model/src/test/cfg/clients/simpleconfig.v2.docprocv3/services.xml b/config-model/src/test/cfg/clients/simpleconfig.v2.docprocv3/services.xml deleted file mode 100644 index ded66571026..00000000000 --- a/config-model/src/test/cfg/clients/simpleconfig.v2.docprocv3/services.xml +++ /dev/null @@ -1,63 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<services> - - <admin version="2.0"> - <adminserver hostalias="node1" /> - <logserver hostalias="node1" /> - <slobroks> - <slobrok hostalias="node1" /> - </slobroks> - </admin> - - <clients version="2.0"> - <spoolers> - <spooler hostalias="node1" maxfailuresize="100000" maxfatalfailuresize="1000000" threads="5"> - <abortondocumenterror>false</abortondocumenterror> - <maxpendingbytes>8000</maxpendingbytes> - <tracelevel>7</tracelevel> - <parsers> - <parser type="com.yahoo.vespaspooler.XMLFileParser"/> - <parser type="com.yahoo.vespaspooler.MusicFileParser"/> - <parser type="com.yahoo.vespaspooler.MusicParser"> - <parameter key="route" value="default"/> - <parameter key="foo" value="bar"/> - </parser> - </parsers> - </spooler> - </spoolers> - </clients> - - <container version="1.0"> - - <nodes> - <node hostalias="node1"/> - </nodes> - - <document-api/> - - <document-processing> - <chain id="main"> - <documentprocessor id="com.yahoo.docprocs.FoobarDocumentProcessor" /> - </chain> - </document-processing> - - <search/> - - </container> - - <content version="1.0" id="music"> - - <redundancy>2</redundancy> - - <documents> - <document type="music" mode="index"/> - </documents> - - <nodes> - <node hostalias="node1" distribution-key="0" /> - </nodes> - - </content> - -</services> diff --git a/config-model/src/test/cfg/clients/simpleconfig.v2/searchdefinitions/.gitignore b/config-model/src/test/cfg/clients/simpleconfig.v2/searchdefinitions/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/config-model/src/test/cfg/clients/simpleconfig.v2/searchdefinitions/.gitignore +++ /dev/null diff --git a/config-model/src/test/configmodel/types/documentmanager.cfg b/config-model/src/test/configmodel/types/documentmanager.cfg index aaaebccecfd..a6a56bf65c1 100644 --- a/config-model/src/test/configmodel/types/documentmanager.cfg +++ b/config-model/src/test/configmodel/types/documentmanager.cfg @@ -110,113 +110,116 @@ datatype[19].id -794985308 datatype[19].arraytype[0].datatype 1707615575 datatype[20].id 69621385 datatype[20].arraytype[0].datatype 339965458 -datatype[21].id 1901258752 +datatype[21].id -372512406 datatype[21].maptype[0].keytype 0 -datatype[21].maptype[0].valtype -2092985853 -datatype[22].id 759956026 -datatype[22].arraytype[0].datatype -2092985853 -datatype[23].id -389833101 +datatype[21].maptype[0].valtype 1707615575 +datatype[22].id 1416345047 +datatype[22].arraytype[0].datatype -372512406 +datatype[23].id 1901258752 datatype[23].maptype[0].keytype 0 -datatype[23].maptype[0].valtype 294108848 -datatype[24].id 1328581348 -datatype[24].structtype[0].name "types.header" -datatype[24].structtype[0].version 0 -datatype[24].structtype[0].compresstype NONE -datatype[24].structtype[0].compresslevel 0 -datatype[24].structtype[0].compressthreshold 95 -datatype[24].structtype[0].compressminsize 800 -datatype[24].structtype[0].field[0].name "abyte" -datatype[24].structtype[0].field[0].datatype 16 -datatype[24].structtype[0].field[0].detailedtype "" -datatype[24].structtype[0].field[1].name "along" -datatype[24].structtype[0].field[1].datatype 4 -datatype[24].structtype[0].field[1].detailedtype "" -datatype[24].structtype[0].field[2].name "arrayfield" -datatype[24].structtype[0].field[2].datatype -1245117006 -datatype[24].structtype[0].field[2].detailedtype "" -datatype[24].structtype[0].field[3].name "setfield" -datatype[24].structtype[0].field[3].datatype 1328286588 -datatype[24].structtype[0].field[3].detailedtype "" -datatype[24].structtype[0].field[4].name "pos" -datatype[24].structtype[0].field[4].datatype 1381038251 -datatype[24].structtype[0].field[4].detailedtype "" -datatype[24].structtype[0].field[5].name "setfield2" -datatype[24].structtype[0].field[5].datatype 18 -datatype[24].structtype[0].field[5].detailedtype "" -datatype[24].structtype[0].field[6].name "setfield3" -datatype[24].structtype[0].field[6].datatype 2125328771 -datatype[24].structtype[0].field[6].detailedtype "" -datatype[24].structtype[0].field[7].name "setfield4" -datatype[24].structtype[0].field[7].datatype 2065577986 -datatype[24].structtype[0].field[7].detailedtype "" -datatype[24].structtype[0].field[8].name "tagfield" -datatype[24].structtype[0].field[8].datatype 18 -datatype[24].structtype[0].field[8].detailedtype "" -datatype[24].structtype[0].field[9].name "structfield" -datatype[24].structtype[0].field[9].datatype 109267174 -datatype[24].structtype[0].field[9].detailedtype "" -datatype[24].structtype[0].field[10].name "structarrayfield" -datatype[24].structtype[0].field[10].datatype -1244829667 -datatype[24].structtype[0].field[10].detailedtype "" -datatype[24].structtype[0].field[11].name "stringmapfield" -datatype[24].structtype[0].field[11].datatype 339965458 -datatype[24].structtype[0].field[11].detailedtype "" -datatype[24].structtype[0].field[12].name "intmapfield" -datatype[24].structtype[0].field[12].datatype -1584287606 -datatype[24].structtype[0].field[12].detailedtype "" -datatype[24].structtype[0].field[13].name "floatmapfield" -datatype[24].structtype[0].field[13].datatype 2125154557 -datatype[24].structtype[0].field[13].detailedtype "" -datatype[24].structtype[0].field[14].name "longmapfield" -datatype[24].structtype[0].field[14].datatype -1715531035 -datatype[24].structtype[0].field[14].detailedtype "" -datatype[24].structtype[0].field[15].name "doublemapfield" -datatype[24].structtype[0].field[15].datatype 2138385264 -datatype[24].structtype[0].field[15].detailedtype "" -datatype[24].structtype[0].field[16].name "arraymapfield" -datatype[24].structtype[0].field[16].datatype 435886609 -datatype[24].structtype[0].field[16].detailedtype "" -datatype[24].structtype[0].field[17].name "arrarr" -datatype[24].structtype[0].field[17].datatype -794985308 -datatype[24].structtype[0].field[17].detailedtype "" -datatype[24].structtype[0].field[18].name "maparr" -datatype[24].structtype[0].field[18].datatype 69621385 -datatype[24].structtype[0].field[18].detailedtype "" -datatype[24].structtype[0].field[19].name "mystructfield" -datatype[24].structtype[0].field[19].datatype -2092985853 -datatype[24].structtype[0].field[19].detailedtype "" -datatype[24].structtype[0].field[20].name "mystructmap" -datatype[24].structtype[0].field[20].datatype 1901258752 -datatype[24].structtype[0].field[20].detailedtype "" -datatype[24].structtype[0].field[21].name "mystructarr" -datatype[24].structtype[0].field[21].datatype 759956026 -datatype[24].structtype[0].field[21].detailedtype "" -datatype[24].structtype[0].field[22].name "Folders" -datatype[24].structtype[0].field[22].datatype -389833101 -datatype[24].structtype[0].field[22].detailedtype "" -datatype[24].structtype[0].field[23].name "juletre" -datatype[24].structtype[0].field[23].datatype 4 -datatype[24].structtype[0].field[23].detailedtype "" -datatype[24].structtype[0].field[24].name "album0" -datatype[24].structtype[0].field[24].datatype 18 -datatype[24].structtype[0].field[24].detailedtype "" -datatype[24].structtype[0].field[25].name "album1" -datatype[24].structtype[0].field[25].datatype 18 -datatype[24].structtype[0].field[25].detailedtype "" -datatype[24].structtype[0].field[26].name "other" -datatype[24].structtype[0].field[26].datatype 4 -datatype[24].structtype[0].field[26].detailedtype "" -datatype[24].structtype[0].field[27].name "rankfeatures" -datatype[24].structtype[0].field[27].datatype 2 -datatype[24].structtype[0].field[27].detailedtype "" -datatype[24].structtype[0].field[28].name "summaryfeatures" -datatype[24].structtype[0].field[28].datatype 2 -datatype[24].structtype[0].field[28].detailedtype "" -datatype[25].id -372512406 +datatype[23].maptype[0].valtype -2092985853 +datatype[24].id 759956026 +datatype[24].arraytype[0].datatype -2092985853 +datatype[25].id -389833101 datatype[25].maptype[0].keytype 0 -datatype[25].maptype[0].valtype 1707615575 -datatype[26].id 1416345047 -datatype[26].arraytype[0].datatype -372512406 +datatype[25].maptype[0].valtype 294108848 +datatype[26].id 1328581348 +datatype[26].structtype[0].name "types.header" +datatype[26].structtype[0].version 0 +datatype[26].structtype[0].compresstype NONE +datatype[26].structtype[0].compresslevel 0 +datatype[26].structtype[0].compressthreshold 95 +datatype[26].structtype[0].compressminsize 800 +datatype[26].structtype[0].field[0].name "abyte" +datatype[26].structtype[0].field[0].datatype 16 +datatype[26].structtype[0].field[0].detailedtype "" +datatype[26].structtype[0].field[1].name "along" +datatype[26].structtype[0].field[1].datatype 4 +datatype[26].structtype[0].field[1].detailedtype "" +datatype[26].structtype[0].field[2].name "arrayfield" +datatype[26].structtype[0].field[2].datatype -1245117006 +datatype[26].structtype[0].field[2].detailedtype "" +datatype[26].structtype[0].field[3].name "setfield" +datatype[26].structtype[0].field[3].datatype 1328286588 +datatype[26].structtype[0].field[3].detailedtype "" +datatype[26].structtype[0].field[4].name "pos" +datatype[26].structtype[0].field[4].datatype 1381038251 +datatype[26].structtype[0].field[4].detailedtype "" +datatype[26].structtype[0].field[5].name "setfield2" +datatype[26].structtype[0].field[5].datatype 18 +datatype[26].structtype[0].field[5].detailedtype "" +datatype[26].structtype[0].field[6].name "setfield3" +datatype[26].structtype[0].field[6].datatype 2125328771 +datatype[26].structtype[0].field[6].detailedtype "" +datatype[26].structtype[0].field[7].name "setfield4" +datatype[26].structtype[0].field[7].datatype 2065577986 +datatype[26].structtype[0].field[7].detailedtype "" +datatype[26].structtype[0].field[8].name "tagfield" +datatype[26].structtype[0].field[8].datatype 18 +datatype[26].structtype[0].field[8].detailedtype "" +datatype[26].structtype[0].field[9].name "structfield" +datatype[26].structtype[0].field[9].datatype 109267174 +datatype[26].structtype[0].field[9].detailedtype "" +datatype[26].structtype[0].field[10].name "structarrayfield" +datatype[26].structtype[0].field[10].datatype -1244829667 +datatype[26].structtype[0].field[10].detailedtype "" +datatype[26].structtype[0].field[11].name "stringmapfield" +datatype[26].structtype[0].field[11].datatype 339965458 +datatype[26].structtype[0].field[11].detailedtype "" +datatype[26].structtype[0].field[12].name "intmapfield" +datatype[26].structtype[0].field[12].datatype -1584287606 +datatype[26].structtype[0].field[12].detailedtype "" +datatype[26].structtype[0].field[13].name "floatmapfield" +datatype[26].structtype[0].field[13].datatype 2125154557 +datatype[26].structtype[0].field[13].detailedtype "" +datatype[26].structtype[0].field[14].name "longmapfield" +datatype[26].structtype[0].field[14].datatype -1715531035 +datatype[26].structtype[0].field[14].detailedtype "" +datatype[26].structtype[0].field[15].name "doublemapfield" +datatype[26].structtype[0].field[15].datatype 2138385264 +datatype[26].structtype[0].field[15].detailedtype "" +datatype[26].structtype[0].field[16].name "arraymapfield" +datatype[26].structtype[0].field[16].datatype 435886609 +datatype[26].structtype[0].field[16].detailedtype "" +datatype[26].structtype[0].field[17].name "arrarr" +datatype[26].structtype[0].field[17].datatype -794985308 +datatype[26].structtype[0].field[17].detailedtype "" +datatype[26].structtype[0].field[18].name "maparr" +datatype[26].structtype[0].field[18].datatype 69621385 +datatype[26].structtype[0].field[18].detailedtype "" +datatype[26].structtype[0].field[19].name "complexarray" +datatype[26].structtype[0].field[19].datatype 1416345047 +datatype[26].structtype[0].field[19].detailedtype "" +datatype[26].structtype[0].field[20].name "mystructfield" +datatype[26].structtype[0].field[20].datatype -2092985853 +datatype[26].structtype[0].field[20].detailedtype "" +datatype[26].structtype[0].field[21].name "mystructmap" +datatype[26].structtype[0].field[21].datatype 1901258752 +datatype[26].structtype[0].field[21].detailedtype "" +datatype[26].structtype[0].field[22].name "mystructarr" +datatype[26].structtype[0].field[22].datatype 759956026 +datatype[26].structtype[0].field[22].detailedtype "" +datatype[26].structtype[0].field[23].name "Folders" +datatype[26].structtype[0].field[23].datatype -389833101 +datatype[26].structtype[0].field[23].detailedtype "" +datatype[26].structtype[0].field[24].name "juletre" +datatype[26].structtype[0].field[24].datatype 4 +datatype[26].structtype[0].field[24].detailedtype "" +datatype[26].structtype[0].field[25].name "album0" +datatype[26].structtype[0].field[25].datatype 18 +datatype[26].structtype[0].field[25].detailedtype "" +datatype[26].structtype[0].field[26].name "album1" +datatype[26].structtype[0].field[26].datatype 18 +datatype[26].structtype[0].field[26].detailedtype "" +datatype[26].structtype[0].field[27].name "other" +datatype[26].structtype[0].field[27].datatype 4 +datatype[26].structtype[0].field[27].detailedtype "" +datatype[26].structtype[0].field[28].name "rankfeatures" +datatype[26].structtype[0].field[28].datatype 2 +datatype[26].structtype[0].field[28].detailedtype "" +datatype[26].structtype[0].field[29].name "summaryfeatures" +datatype[26].structtype[0].field[29].datatype 2 +datatype[26].structtype[0].field[29].detailedtype "" datatype[27].id 348447225 datatype[27].structtype[0].name "types.body" datatype[27].structtype[0].version 0 @@ -224,9 +227,6 @@ datatype[27].structtype[0].compresstype NONE datatype[27].structtype[0].compresslevel 0 datatype[27].structtype[0].compressthreshold 95 datatype[27].structtype[0].compressminsize 800 -datatype[27].structtype[0].field[0].name "complexarray" -datatype[27].structtype[0].field[0].datatype 1416345047 -datatype[27].structtype[0].field[0].detailedtype "" datatype[28].id -853072901 datatype[28].documenttype[0].name "types" datatype[28].documenttype[0].version 0 @@ -260,4 +260,4 @@ datatype[28].documenttype[0].fieldsets{[document]}.fields[22] "setfield4" datatype[28].documenttype[0].fieldsets{[document]}.fields[23] "stringmapfield" datatype[28].documenttype[0].fieldsets{[document]}.fields[24] "structarrayfield" datatype[28].documenttype[0].fieldsets{[document]}.fields[25] "structfield" -datatype[28].documenttype[0].fieldsets{[document]}.fields[26] "tagfield"
\ No newline at end of file +datatype[28].documenttype[0].fieldsets{[document]}.fields[26] "tagfield" diff --git a/config-model/src/test/configmodel/types/documenttypes.cfg b/config-model/src/test/configmodel/types/documenttypes.cfg index 97785618e8e..7ff795bc33c 100644 --- a/config-model/src/test/configmodel/types/documenttypes.cfg +++ b/config-model/src/test/configmodel/types/documenttypes.cfg @@ -355,11 +355,11 @@ documenttype[0].datatype[19].sstruct.compression.type NONE documenttype[0].datatype[19].sstruct.compression.level 0 documenttype[0].datatype[19].sstruct.compression.threshold 95 documenttype[0].datatype[19].sstruct.compression.minsize 200 -documenttype[0].datatype[20].id 1901258752 +documenttype[0].datatype[20].id -372512406 documenttype[0].datatype[20].type MAP documenttype[0].datatype[20].array.element.id 0 documenttype[0].datatype[20].map.key.id 0 -documenttype[0].datatype[20].map.value.id -2092985853 +documenttype[0].datatype[20].map.value.id 1707615575 documenttype[0].datatype[20].wset.key.id 0 documenttype[0].datatype[20].wset.createifnonexistent false documenttype[0].datatype[20].wset.removeifzero false @@ -370,9 +370,9 @@ documenttype[0].datatype[20].sstruct.compression.type NONE documenttype[0].datatype[20].sstruct.compression.level 0 documenttype[0].datatype[20].sstruct.compression.threshold 95 documenttype[0].datatype[20].sstruct.compression.minsize 200 -documenttype[0].datatype[21].id 759956026 +documenttype[0].datatype[21].id 1416345047 documenttype[0].datatype[21].type ARRAY -documenttype[0].datatype[21].array.element.id -2092985853 +documenttype[0].datatype[21].array.element.id -372512406 documenttype[0].datatype[21].map.key.id 0 documenttype[0].datatype[21].map.value.id 0 documenttype[0].datatype[21].wset.key.id 0 @@ -385,11 +385,11 @@ documenttype[0].datatype[21].sstruct.compression.type NONE documenttype[0].datatype[21].sstruct.compression.level 0 documenttype[0].datatype[21].sstruct.compression.threshold 95 documenttype[0].datatype[21].sstruct.compression.minsize 200 -documenttype[0].datatype[22].id -389833101 +documenttype[0].datatype[22].id 1901258752 documenttype[0].datatype[22].type MAP documenttype[0].datatype[22].array.element.id 0 documenttype[0].datatype[22].map.key.id 0 -documenttype[0].datatype[22].map.value.id 294108848 +documenttype[0].datatype[22].map.value.id -2092985853 documenttype[0].datatype[22].wset.key.id 0 documenttype[0].datatype[22].wset.createifnonexistent false documenttype[0].datatype[22].wset.removeifzero false @@ -400,171 +400,26 @@ documenttype[0].datatype[22].sstruct.compression.type NONE documenttype[0].datatype[22].sstruct.compression.level 0 documenttype[0].datatype[22].sstruct.compression.threshold 95 documenttype[0].datatype[22].sstruct.compression.minsize 200 -documenttype[0].datatype[23].id 1328581348 -documenttype[0].datatype[23].type STRUCT -documenttype[0].datatype[23].array.element.id 0 +documenttype[0].datatype[23].id 759956026 +documenttype[0].datatype[23].type ARRAY +documenttype[0].datatype[23].array.element.id -2092985853 documenttype[0].datatype[23].map.key.id 0 documenttype[0].datatype[23].map.value.id 0 documenttype[0].datatype[23].wset.key.id 0 documenttype[0].datatype[23].wset.createifnonexistent false documenttype[0].datatype[23].wset.removeifzero false documenttype[0].datatype[23].annotationref.annotation.id 0 -documenttype[0].datatype[23].sstruct.name "types.header" +documenttype[0].datatype[23].sstruct.name "" documenttype[0].datatype[23].sstruct.version 0 documenttype[0].datatype[23].sstruct.compression.type NONE documenttype[0].datatype[23].sstruct.compression.level 0 documenttype[0].datatype[23].sstruct.compression.threshold 95 documenttype[0].datatype[23].sstruct.compression.minsize 200 -documenttype[0].datatype[23].sstruct.field[0].name "abyte" -documenttype[0].datatype[23].sstruct.field[0].id 110138156 -documenttype[0].datatype[23].sstruct.field[0].id_v6 1369099343 -documenttype[0].datatype[23].sstruct.field[0].datatype 16 -documenttype[0].datatype[23].sstruct.field[0].detailedtype "" -documenttype[0].datatype[23].sstruct.field[1].name "along" -documenttype[0].datatype[23].sstruct.field[1].id 1206464520 -documenttype[0].datatype[23].sstruct.field[1].id_v6 871280609 -documenttype[0].datatype[23].sstruct.field[1].datatype 4 -documenttype[0].datatype[23].sstruct.field[1].detailedtype "" -documenttype[0].datatype[23].sstruct.field[2].name "arrayfield" -documenttype[0].datatype[23].sstruct.field[2].id 965790107 -documenttype[0].datatype[23].sstruct.field[2].id_v6 1010955705 -documenttype[0].datatype[23].sstruct.field[2].datatype -1245117006 -documenttype[0].datatype[23].sstruct.field[2].detailedtype "" -documenttype[0].datatype[23].sstruct.field[3].name "setfield" -documenttype[0].datatype[23].sstruct.field[3].id 761581914 -documenttype[0].datatype[23].sstruct.field[3].id_v6 1762943268 -documenttype[0].datatype[23].sstruct.field[3].datatype 1328286588 -documenttype[0].datatype[23].sstruct.field[3].detailedtype "" -documenttype[0].datatype[23].sstruct.field[4].name "pos" -documenttype[0].datatype[23].sstruct.field[4].id 1041567475 -documenttype[0].datatype[23].sstruct.field[4].id_v6 26353693 -documenttype[0].datatype[23].sstruct.field[4].datatype 1381038251 -documenttype[0].datatype[23].sstruct.field[4].detailedtype "" -documenttype[0].datatype[23].sstruct.field[5].name "setfield2" -documenttype[0].datatype[23].sstruct.field[5].id 1066659198 -documenttype[0].datatype[23].sstruct.field[5].id_v6 813038565 -documenttype[0].datatype[23].sstruct.field[5].datatype 18 -documenttype[0].datatype[23].sstruct.field[5].detailedtype "" -documenttype[0].datatype[23].sstruct.field[6].name "setfield3" -documenttype[0].datatype[23].sstruct.field[6].id 1180155772 -documenttype[0].datatype[23].sstruct.field[6].id_v6 1697232199 -documenttype[0].datatype[23].sstruct.field[6].datatype 2125328771 -documenttype[0].datatype[23].sstruct.field[6].detailedtype "" -documenttype[0].datatype[23].sstruct.field[7].name "setfield4" -documenttype[0].datatype[23].sstruct.field[7].id 1254131631 -documenttype[0].datatype[23].sstruct.field[7].id_v6 119755202 -documenttype[0].datatype[23].sstruct.field[7].datatype 2065577986 -documenttype[0].datatype[23].sstruct.field[7].detailedtype "" -documenttype[0].datatype[23].sstruct.field[8].name "tagfield" -documenttype[0].datatype[23].sstruct.field[8].id 1653562069 -documenttype[0].datatype[23].sstruct.field[8].id_v6 938523246 -documenttype[0].datatype[23].sstruct.field[8].datatype 18 -documenttype[0].datatype[23].sstruct.field[8].detailedtype "" -documenttype[0].datatype[23].sstruct.field[9].name "structfield" -documenttype[0].datatype[23].sstruct.field[9].id 486207386 -documenttype[0].datatype[23].sstruct.field[9].id_v6 418303145 -documenttype[0].datatype[23].sstruct.field[9].datatype 109267174 -documenttype[0].datatype[23].sstruct.field[9].detailedtype "" -documenttype[0].datatype[23].sstruct.field[10].name "structarrayfield" -documenttype[0].datatype[23].sstruct.field[10].id 335048518 -documenttype[0].datatype[23].sstruct.field[10].id_v6 607034174 -documenttype[0].datatype[23].sstruct.field[10].datatype -1244829667 -documenttype[0].datatype[23].sstruct.field[10].detailedtype "" -documenttype[0].datatype[23].sstruct.field[11].name "stringmapfield" -documenttype[0].datatype[23].sstruct.field[11].id 117465687 -documenttype[0].datatype[23].sstruct.field[11].id_v6 1492788095 -documenttype[0].datatype[23].sstruct.field[11].datatype 339965458 -documenttype[0].datatype[23].sstruct.field[11].detailedtype "" -documenttype[0].datatype[23].sstruct.field[12].name "intmapfield" -documenttype[0].datatype[23].sstruct.field[12].id 121004462 -documenttype[0].datatype[23].sstruct.field[12].id_v6 1642487905 -documenttype[0].datatype[23].sstruct.field[12].datatype -1584287606 -documenttype[0].datatype[23].sstruct.field[12].detailedtype "" -documenttype[0].datatype[23].sstruct.field[13].name "floatmapfield" -documenttype[0].datatype[23].sstruct.field[13].id 1239120925 -documenttype[0].datatype[23].sstruct.field[13].id_v6 1609437589 -documenttype[0].datatype[23].sstruct.field[13].datatype 2125154557 -documenttype[0].datatype[23].sstruct.field[13].detailedtype "" -documenttype[0].datatype[23].sstruct.field[14].name "longmapfield" -documenttype[0].datatype[23].sstruct.field[14].id 477718745 -documenttype[0].datatype[23].sstruct.field[14].id_v6 920341968 -documenttype[0].datatype[23].sstruct.field[14].datatype -1715531035 -documenttype[0].datatype[23].sstruct.field[14].detailedtype "" -documenttype[0].datatype[23].sstruct.field[15].name "doublemapfield" -documenttype[0].datatype[23].sstruct.field[15].id 877047192 -documenttype[0].datatype[23].sstruct.field[15].id_v6 957317090 -documenttype[0].datatype[23].sstruct.field[15].datatype 2138385264 -documenttype[0].datatype[23].sstruct.field[15].detailedtype "" -documenttype[0].datatype[23].sstruct.field[16].name "arraymapfield" -documenttype[0].datatype[23].sstruct.field[16].id 1670805928 -documenttype[0].datatype[23].sstruct.field[16].id_v6 1940354311 -documenttype[0].datatype[23].sstruct.field[16].datatype 435886609 -documenttype[0].datatype[23].sstruct.field[16].detailedtype "" -documenttype[0].datatype[23].sstruct.field[17].name "arrarr" -documenttype[0].datatype[23].sstruct.field[17].id 1962567166 -documenttype[0].datatype[23].sstruct.field[17].id_v6 885141301 -documenttype[0].datatype[23].sstruct.field[17].datatype -794985308 -documenttype[0].datatype[23].sstruct.field[17].detailedtype "" -documenttype[0].datatype[23].sstruct.field[18].name "maparr" -documenttype[0].datatype[23].sstruct.field[18].id 904375219 -documenttype[0].datatype[23].sstruct.field[18].id_v6 63700074 -documenttype[0].datatype[23].sstruct.field[18].datatype 69621385 -documenttype[0].datatype[23].sstruct.field[18].detailedtype "" -documenttype[0].datatype[23].sstruct.field[19].name "mystructfield" -documenttype[0].datatype[23].sstruct.field[19].id 1348513378 -documenttype[0].datatype[23].sstruct.field[19].id_v6 2033170300 -documenttype[0].datatype[23].sstruct.field[19].datatype -2092985853 -documenttype[0].datatype[23].sstruct.field[19].detailedtype "" -documenttype[0].datatype[23].sstruct.field[20].name "mystructmap" -documenttype[0].datatype[23].sstruct.field[20].id 1511423250 -documenttype[0].datatype[23].sstruct.field[20].id_v6 449602635 -documenttype[0].datatype[23].sstruct.field[20].datatype 1901258752 -documenttype[0].datatype[23].sstruct.field[20].detailedtype "" -documenttype[0].datatype[23].sstruct.field[21].name "mystructarr" -documenttype[0].datatype[23].sstruct.field[21].id 595856991 -documenttype[0].datatype[23].sstruct.field[21].id_v6 764861972 -documenttype[0].datatype[23].sstruct.field[21].datatype 759956026 -documenttype[0].datatype[23].sstruct.field[21].detailedtype "" -documenttype[0].datatype[23].sstruct.field[22].name "Folders" -documenttype[0].datatype[23].sstruct.field[22].id 34575524 -documenttype[0].datatype[23].sstruct.field[22].id_v6 280569744 -documenttype[0].datatype[23].sstruct.field[22].datatype -389833101 -documenttype[0].datatype[23].sstruct.field[22].detailedtype "" -documenttype[0].datatype[23].sstruct.field[23].name "juletre" -documenttype[0].datatype[23].sstruct.field[23].id 1039981530 -documenttype[0].datatype[23].sstruct.field[23].id_v6 2073084146 -documenttype[0].datatype[23].sstruct.field[23].datatype 4 -documenttype[0].datatype[23].sstruct.field[23].detailedtype "" -documenttype[0].datatype[23].sstruct.field[24].name "album0" -documenttype[0].datatype[23].sstruct.field[24].id 764312262 -documenttype[0].datatype[23].sstruct.field[24].id_v6 1409364160 -documenttype[0].datatype[23].sstruct.field[24].datatype 18 -documenttype[0].datatype[23].sstruct.field[24].detailedtype "" -documenttype[0].datatype[23].sstruct.field[25].name "album1" -documenttype[0].datatype[23].sstruct.field[25].id 1967160809 -documenttype[0].datatype[23].sstruct.field[25].id_v6 1833811264 -documenttype[0].datatype[23].sstruct.field[25].datatype 18 -documenttype[0].datatype[23].sstruct.field[25].detailedtype "" -documenttype[0].datatype[23].sstruct.field[26].name "other" -documenttype[0].datatype[23].sstruct.field[26].id 2443357 -documenttype[0].datatype[23].sstruct.field[26].id_v6 903806222 -documenttype[0].datatype[23].sstruct.field[26].datatype 4 -documenttype[0].datatype[23].sstruct.field[26].detailedtype "" -documenttype[0].datatype[23].sstruct.field[27].name "rankfeatures" -documenttype[0].datatype[23].sstruct.field[27].id 1883197392 -documenttype[0].datatype[23].sstruct.field[27].id_v6 699950698 -documenttype[0].datatype[23].sstruct.field[27].datatype 2 -documenttype[0].datatype[23].sstruct.field[27].detailedtype "" -documenttype[0].datatype[23].sstruct.field[28].name "summaryfeatures" -documenttype[0].datatype[23].sstruct.field[28].id 1840337115 -documenttype[0].datatype[23].sstruct.field[28].id_v6 1981648971 -documenttype[0].datatype[23].sstruct.field[28].datatype 2 -documenttype[0].datatype[23].sstruct.field[28].detailedtype "" -documenttype[0].datatype[24].id -372512406 +documenttype[0].datatype[24].id -389833101 documenttype[0].datatype[24].type MAP documenttype[0].datatype[24].array.element.id 0 documenttype[0].datatype[24].map.key.id 0 -documenttype[0].datatype[24].map.value.id 1707615575 +documenttype[0].datatype[24].map.value.id 294108848 documenttype[0].datatype[24].wset.key.id 0 documenttype[0].datatype[24].wset.createifnonexistent false documenttype[0].datatype[24].wset.removeifzero false @@ -575,21 +430,171 @@ documenttype[0].datatype[24].sstruct.compression.type NONE documenttype[0].datatype[24].sstruct.compression.level 0 documenttype[0].datatype[24].sstruct.compression.threshold 95 documenttype[0].datatype[24].sstruct.compression.minsize 200 -documenttype[0].datatype[25].id 1416345047 -documenttype[0].datatype[25].type ARRAY -documenttype[0].datatype[25].array.element.id -372512406 +documenttype[0].datatype[25].id 1328581348 +documenttype[0].datatype[25].type STRUCT +documenttype[0].datatype[25].array.element.id 0 documenttype[0].datatype[25].map.key.id 0 documenttype[0].datatype[25].map.value.id 0 documenttype[0].datatype[25].wset.key.id 0 documenttype[0].datatype[25].wset.createifnonexistent false documenttype[0].datatype[25].wset.removeifzero false documenttype[0].datatype[25].annotationref.annotation.id 0 -documenttype[0].datatype[25].sstruct.name "" +documenttype[0].datatype[25].sstruct.name "types.header" documenttype[0].datatype[25].sstruct.version 0 documenttype[0].datatype[25].sstruct.compression.type NONE documenttype[0].datatype[25].sstruct.compression.level 0 documenttype[0].datatype[25].sstruct.compression.threshold 95 documenttype[0].datatype[25].sstruct.compression.minsize 200 +documenttype[0].datatype[25].sstruct.field[0].name "abyte" +documenttype[0].datatype[25].sstruct.field[0].id 110138156 +documenttype[0].datatype[25].sstruct.field[0].id_v6 1369099343 +documenttype[0].datatype[25].sstruct.field[0].datatype 16 +documenttype[0].datatype[25].sstruct.field[0].detailedtype "" +documenttype[0].datatype[25].sstruct.field[1].name "along" +documenttype[0].datatype[25].sstruct.field[1].id 1206464520 +documenttype[0].datatype[25].sstruct.field[1].id_v6 871280609 +documenttype[0].datatype[25].sstruct.field[1].datatype 4 +documenttype[0].datatype[25].sstruct.field[1].detailedtype "" +documenttype[0].datatype[25].sstruct.field[2].name "arrayfield" +documenttype[0].datatype[25].sstruct.field[2].id 965790107 +documenttype[0].datatype[25].sstruct.field[2].id_v6 1010955705 +documenttype[0].datatype[25].sstruct.field[2].datatype -1245117006 +documenttype[0].datatype[25].sstruct.field[2].detailedtype "" +documenttype[0].datatype[25].sstruct.field[3].name "setfield" +documenttype[0].datatype[25].sstruct.field[3].id 761581914 +documenttype[0].datatype[25].sstruct.field[3].id_v6 1762943268 +documenttype[0].datatype[25].sstruct.field[3].datatype 1328286588 +documenttype[0].datatype[25].sstruct.field[3].detailedtype "" +documenttype[0].datatype[25].sstruct.field[4].name "pos" +documenttype[0].datatype[25].sstruct.field[4].id 1041567475 +documenttype[0].datatype[25].sstruct.field[4].id_v6 26353693 +documenttype[0].datatype[25].sstruct.field[4].datatype 1381038251 +documenttype[0].datatype[25].sstruct.field[4].detailedtype "" +documenttype[0].datatype[25].sstruct.field[5].name "setfield2" +documenttype[0].datatype[25].sstruct.field[5].id 1066659198 +documenttype[0].datatype[25].sstruct.field[5].id_v6 813038565 +documenttype[0].datatype[25].sstruct.field[5].datatype 18 +documenttype[0].datatype[25].sstruct.field[5].detailedtype "" +documenttype[0].datatype[25].sstruct.field[6].name "setfield3" +documenttype[0].datatype[25].sstruct.field[6].id 1180155772 +documenttype[0].datatype[25].sstruct.field[6].id_v6 1697232199 +documenttype[0].datatype[25].sstruct.field[6].datatype 2125328771 +documenttype[0].datatype[25].sstruct.field[6].detailedtype "" +documenttype[0].datatype[25].sstruct.field[7].name "setfield4" +documenttype[0].datatype[25].sstruct.field[7].id 1254131631 +documenttype[0].datatype[25].sstruct.field[7].id_v6 119755202 +documenttype[0].datatype[25].sstruct.field[7].datatype 2065577986 +documenttype[0].datatype[25].sstruct.field[7].detailedtype "" +documenttype[0].datatype[25].sstruct.field[8].name "tagfield" +documenttype[0].datatype[25].sstruct.field[8].id 1653562069 +documenttype[0].datatype[25].sstruct.field[8].id_v6 938523246 +documenttype[0].datatype[25].sstruct.field[8].datatype 18 +documenttype[0].datatype[25].sstruct.field[8].detailedtype "" +documenttype[0].datatype[25].sstruct.field[9].name "structfield" +documenttype[0].datatype[25].sstruct.field[9].id 486207386 +documenttype[0].datatype[25].sstruct.field[9].id_v6 418303145 +documenttype[0].datatype[25].sstruct.field[9].datatype 109267174 +documenttype[0].datatype[25].sstruct.field[9].detailedtype "" +documenttype[0].datatype[25].sstruct.field[10].name "structarrayfield" +documenttype[0].datatype[25].sstruct.field[10].id 335048518 +documenttype[0].datatype[25].sstruct.field[10].id_v6 607034174 +documenttype[0].datatype[25].sstruct.field[10].datatype -1244829667 +documenttype[0].datatype[25].sstruct.field[10].detailedtype "" +documenttype[0].datatype[25].sstruct.field[11].name "stringmapfield" +documenttype[0].datatype[25].sstruct.field[11].id 117465687 +documenttype[0].datatype[25].sstruct.field[11].id_v6 1492788095 +documenttype[0].datatype[25].sstruct.field[11].datatype 339965458 +documenttype[0].datatype[25].sstruct.field[11].detailedtype "" +documenttype[0].datatype[25].sstruct.field[12].name "intmapfield" +documenttype[0].datatype[25].sstruct.field[12].id 121004462 +documenttype[0].datatype[25].sstruct.field[12].id_v6 1642487905 +documenttype[0].datatype[25].sstruct.field[12].datatype -1584287606 +documenttype[0].datatype[25].sstruct.field[12].detailedtype "" +documenttype[0].datatype[25].sstruct.field[13].name "floatmapfield" +documenttype[0].datatype[25].sstruct.field[13].id 1239120925 +documenttype[0].datatype[25].sstruct.field[13].id_v6 1609437589 +documenttype[0].datatype[25].sstruct.field[13].datatype 2125154557 +documenttype[0].datatype[25].sstruct.field[13].detailedtype "" +documenttype[0].datatype[25].sstruct.field[14].name "longmapfield" +documenttype[0].datatype[25].sstruct.field[14].id 477718745 +documenttype[0].datatype[25].sstruct.field[14].id_v6 920341968 +documenttype[0].datatype[25].sstruct.field[14].datatype -1715531035 +documenttype[0].datatype[25].sstruct.field[14].detailedtype "" +documenttype[0].datatype[25].sstruct.field[15].name "doublemapfield" +documenttype[0].datatype[25].sstruct.field[15].id 877047192 +documenttype[0].datatype[25].sstruct.field[15].id_v6 957317090 +documenttype[0].datatype[25].sstruct.field[15].datatype 2138385264 +documenttype[0].datatype[25].sstruct.field[15].detailedtype "" +documenttype[0].datatype[25].sstruct.field[16].name "arraymapfield" +documenttype[0].datatype[25].sstruct.field[16].id 1670805928 +documenttype[0].datatype[25].sstruct.field[16].id_v6 1940354311 +documenttype[0].datatype[25].sstruct.field[16].datatype 435886609 +documenttype[0].datatype[25].sstruct.field[16].detailedtype "" +documenttype[0].datatype[25].sstruct.field[17].name "arrarr" +documenttype[0].datatype[25].sstruct.field[17].id 1962567166 +documenttype[0].datatype[25].sstruct.field[17].id_v6 885141301 +documenttype[0].datatype[25].sstruct.field[17].datatype -794985308 +documenttype[0].datatype[25].sstruct.field[17].detailedtype "" +documenttype[0].datatype[25].sstruct.field[18].name "maparr" +documenttype[0].datatype[25].sstruct.field[18].id 904375219 +documenttype[0].datatype[25].sstruct.field[18].id_v6 63700074 +documenttype[0].datatype[25].sstruct.field[18].datatype 69621385 +documenttype[0].datatype[25].sstruct.field[18].detailedtype "" +documenttype[0].datatype[25].sstruct.field[19].name "complexarray" +documenttype[0].datatype[25].sstruct.field[19].id 795629533 +documenttype[0].datatype[25].sstruct.field[19].id_v6 658530305 +documenttype[0].datatype[25].sstruct.field[19].datatype 1416345047 +documenttype[0].datatype[25].sstruct.field[19].detailedtype "" +documenttype[0].datatype[25].sstruct.field[20].name "mystructfield" +documenttype[0].datatype[25].sstruct.field[20].id 1348513378 +documenttype[0].datatype[25].sstruct.field[20].id_v6 2033170300 +documenttype[0].datatype[25].sstruct.field[20].datatype -2092985853 +documenttype[0].datatype[25].sstruct.field[20].detailedtype "" +documenttype[0].datatype[25].sstruct.field[21].name "mystructmap" +documenttype[0].datatype[25].sstruct.field[21].id 1511423250 +documenttype[0].datatype[25].sstruct.field[21].id_v6 449602635 +documenttype[0].datatype[25].sstruct.field[21].datatype 1901258752 +documenttype[0].datatype[25].sstruct.field[21].detailedtype "" +documenttype[0].datatype[25].sstruct.field[22].name "mystructarr" +documenttype[0].datatype[25].sstruct.field[22].id 595856991 +documenttype[0].datatype[25].sstruct.field[22].id_v6 764861972 +documenttype[0].datatype[25].sstruct.field[22].datatype 759956026 +documenttype[0].datatype[25].sstruct.field[22].detailedtype "" +documenttype[0].datatype[25].sstruct.field[23].name "Folders" +documenttype[0].datatype[25].sstruct.field[23].id 34575524 +documenttype[0].datatype[25].sstruct.field[23].id_v6 280569744 +documenttype[0].datatype[25].sstruct.field[23].datatype -389833101 +documenttype[0].datatype[25].sstruct.field[23].detailedtype "" +documenttype[0].datatype[25].sstruct.field[24].name "juletre" +documenttype[0].datatype[25].sstruct.field[24].id 1039981530 +documenttype[0].datatype[25].sstruct.field[24].id_v6 2073084146 +documenttype[0].datatype[25].sstruct.field[24].datatype 4 +documenttype[0].datatype[25].sstruct.field[24].detailedtype "" +documenttype[0].datatype[25].sstruct.field[25].name "album0" +documenttype[0].datatype[25].sstruct.field[25].id 764312262 +documenttype[0].datatype[25].sstruct.field[25].id_v6 1409364160 +documenttype[0].datatype[25].sstruct.field[25].datatype 18 +documenttype[0].datatype[25].sstruct.field[25].detailedtype "" +documenttype[0].datatype[25].sstruct.field[26].name "album1" +documenttype[0].datatype[25].sstruct.field[26].id 1967160809 +documenttype[0].datatype[25].sstruct.field[26].id_v6 1833811264 +documenttype[0].datatype[25].sstruct.field[26].datatype 18 +documenttype[0].datatype[25].sstruct.field[26].detailedtype "" +documenttype[0].datatype[25].sstruct.field[27].name "other" +documenttype[0].datatype[25].sstruct.field[27].id 2443357 +documenttype[0].datatype[25].sstruct.field[27].id_v6 903806222 +documenttype[0].datatype[25].sstruct.field[27].datatype 4 +documenttype[0].datatype[25].sstruct.field[27].detailedtype "" +documenttype[0].datatype[25].sstruct.field[28].name "rankfeatures" +documenttype[0].datatype[25].sstruct.field[28].id 1883197392 +documenttype[0].datatype[25].sstruct.field[28].id_v6 699950698 +documenttype[0].datatype[25].sstruct.field[28].datatype 2 +documenttype[0].datatype[25].sstruct.field[28].detailedtype "" +documenttype[0].datatype[25].sstruct.field[29].name "summaryfeatures" +documenttype[0].datatype[25].sstruct.field[29].id 1840337115 +documenttype[0].datatype[25].sstruct.field[29].id_v6 1981648971 +documenttype[0].datatype[25].sstruct.field[29].datatype 2 +documenttype[0].datatype[25].sstruct.field[29].detailedtype "" documenttype[0].datatype[26].id 348447225 documenttype[0].datatype[26].type STRUCT documenttype[0].datatype[26].array.element.id 0 @@ -605,11 +610,6 @@ documenttype[0].datatype[26].sstruct.compression.type NONE documenttype[0].datatype[26].sstruct.compression.level 0 documenttype[0].datatype[26].sstruct.compression.threshold 95 documenttype[0].datatype[26].sstruct.compression.minsize 200 -documenttype[0].datatype[26].sstruct.field[0].name "complexarray" -documenttype[0].datatype[26].sstruct.field[0].id 795629533 -documenttype[0].datatype[26].sstruct.field[0].id_v6 658530305 -documenttype[0].datatype[26].sstruct.field[0].datatype 1416345047 -documenttype[0].datatype[26].sstruct.field[0].detailedtype "" documenttype[0].fieldsets{[document]}.fields[0] "Folders" documenttype[0].fieldsets{[document]}.fields[1] "abyte" documenttype[0].fieldsets{[document]}.fields[2] "album0" @@ -636,4 +636,4 @@ documenttype[0].fieldsets{[document]}.fields[22] "setfield4" documenttype[0].fieldsets{[document]}.fields[23] "stringmapfield" documenttype[0].fieldsets{[document]}.fields[24] "structarrayfield" documenttype[0].fieldsets{[document]}.fields[25] "structfield" -documenttype[0].fieldsets{[document]}.fields[26] "tagfield"
\ No newline at end of file +documenttype[0].fieldsets{[document]}.fields[26] "tagfield" diff --git a/config-model/src/test/derived/advanced/ilscripts.cfg b/config-model/src/test/derived/advanced/ilscripts.cfg index faf39ddc4d9..b78cb892501 100644 --- a/config-model/src/test/derived/advanced/ilscripts.cfg +++ b/config-model/src/test/derived/advanced/ilscripts.cfg @@ -12,9 +12,9 @@ ilscript[].content[] "clear_state | guard { 1 | set_var superdupermod; 2 | set_v ilscript[].content[] "clear_state | guard { 0 | set_var superduperus | set_var superdupereu | set_var superduperasia; input debug_src | lowercase | tokenize normalize | summary debug | index debug | split \";\" | for_each { switch { case \"superduperus\": 10 | set_var superduperus; case \"superdupereu\": 10 | set_var superdupereu; case \"superduperasia\": 10 | set_var superduperasia; } }; }" ilscript[].content[] "clear_state | guard { input location_str | to_pos | zcurve | attribute location_zcurve; }" ilscript[].content[] "clear_state | guard { input title_src | summary mysummary; }" -ilscript[].content[] "clear_state | guard { input product_src | switch { case \"\": \"myweb\" | tokenize stem:\"SHORTEST\" | index product | summary product; default: input product_src | tokenize stem:\"SHORTEST\" | index product | summary product; }; }" -ilscript[].content[] "clear_state | guard { if (input product2_src == \"foo\") { \"bar\" | tokenize stem:\"SHORTEST\" | index product2; } else { \"baz\" | tokenize stem:\"SHORTEST\" | index product2; }; }" -ilscript[].content[] "clear_state | guard { input product3_src | switch { case \"\": \"myweb\" | tokenize stem:\"SHORTEST\" | index product3 | summary product3; default: input product3_src | tokenize stem:\"SHORTEST\" | index product3 | summary product3; }; }" +ilscript[].content[] "clear_state | guard { input product_src | switch { case \"\": \"myweb\" | tokenize stem:\"BEST\" | index product | summary product; default: input product_src | tokenize stem:\"BEST\" | index product | summary product; }; }" +ilscript[].content[] "clear_state | guard { if (input product2_src == \"foo\") { \"bar\" | tokenize stem:\"BEST\" | index product2; } else { \"baz\" | tokenize stem:\"BEST\" | index product2; }; }" +ilscript[].content[] "clear_state | guard { input product3_src | switch { case \"\": \"myweb\" | tokenize stem:\"BEST\" | index product3 | summary product3; default: input product3_src | tokenize stem:\"BEST\" | index product3 | summary product3; }; }" ilscript[].content[] "clear_state | guard { input title_src | tokenize normalize | index title; }" ilscript[].content[] "clear_state | guard { input title_src | summary title_s; }" ilscript[].content[] "input attributes_src | passthrough attributes_src" diff --git a/config-model/src/test/derived/advanced/index-info.cfg b/config-model/src/test/derived/advanced/index-info.cfg index a927e9635a3..07b58814ea3 100644 --- a/config-model/src/test/derived/advanced/index-info.cfg +++ b/config-model/src/test/derived/advanced/index-info.cfg @@ -60,7 +60,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "product" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "product" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "product" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "product2" @@ -68,7 +68,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "product2" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "product2" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "product2" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "product3" @@ -76,7 +76,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "product3" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "product3" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "product3" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "rankfeatures" diff --git a/config-model/src/test/derived/arrays/ilscripts.cfg b/config-model/src/test/derived/arrays/ilscripts.cfg index c48f4a22ce7..0490835db33 100644 --- a/config-model/src/test/derived/arrays/ilscripts.cfg +++ b/config-model/src/test/derived/arrays/ilscripts.cfg @@ -6,8 +6,8 @@ ilscript[].docfield[] "ratings" ilscript[].docfield[] "a" ilscript[].docfield[] "b" ilscript[].docfield[] "c" -ilscript[].content[] "clear_state | guard { input tags | for_each { tokenize normalize stem:\"SHORTEST\" } | summary tags | index tags; }" +ilscript[].content[] "clear_state | guard { input tags | for_each { tokenize normalize stem:\"BEST\" } | summary tags | index tags; }" ilscript[].content[] "clear_state | guard { input ratings | summary ratings | attribute ratings; }" -ilscript[].content[] "clear_state | guard { input a | tokenize normalize stem:\"SHORTEST\" | index a; }" -ilscript[].content[] "clear_state | guard { input b | for_each { tokenize normalize stem:\"SHORTEST\" } | index b | attribute b; }" -ilscript[].content[] "clear_state | guard { input c | for_each { tokenize normalize stem:\"SHORTEST\" } | summary c | index c; }" +ilscript[].content[] "clear_state | guard { input a | tokenize normalize stem:\"BEST\" | index a; }" +ilscript[].content[] "clear_state | guard { input b | for_each { tokenize normalize stem:\"BEST\" } | index b | attribute b; }" +ilscript[].content[] "clear_state | guard { input c | for_each { tokenize normalize stem:\"BEST\" } | summary c | index c; }" diff --git a/config-model/src/test/derived/arrays/index-info.cfg b/config-model/src/test/derived/arrays/index-info.cfg index abd5fef2ba9..b18f1eaf55f 100644 --- a/config-model/src/test/derived/arrays/index-info.cfg +++ b/config-model/src/test/derived/arrays/index-info.cfg @@ -10,7 +10,7 @@ indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "tags" indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "tags" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "tags" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "tags" @@ -26,7 +26,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "a" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "a" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "a" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "a" @@ -38,7 +38,7 @@ indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "b" indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "b" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "b" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "b" @@ -50,7 +50,7 @@ indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "c" indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "c" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "c" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "c" @@ -68,6 +68,6 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "default" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "default" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "default" indexinfo[].command[].command "normalize" diff --git a/config-model/src/test/derived/attributeprefetch/attributeprefetch.sd b/config-model/src/test/derived/attributeprefetch/attributeprefetch.sd index 0bf6b7a037d..ef3755e90d7 100644 --- a/config-model/src/test/derived/attributeprefetch/attributeprefetch.sd +++ b/config-model/src/test/derived/attributeprefetch/attributeprefetch.sd @@ -3,81 +3,63 @@ search prefetch { document prefetch { field singlebyte type byte { indexing: attribute - attribute: prefetch } field multibyte type array<byte> { indexing: attribute - attribute: prefetch } field wsbyte type weightedset<byte> { indexing: attribute - attribute: prefetch } field singleint type int { indexing: attribute - attribute: prefetch } field multiint type array<int> { indexing: attribute - attribute: prefetch } field wsint type weightedset<int> { indexing: attribute - attribute: prefetch } field singlelong type long { indexing: attribute - attribute: prefetch } field multilong type array<long> { indexing: attribute - attribute: prefetch } field wslong type weightedset<long> { indexing: attribute - attribute: prefetch } field singlefloat type float { indexing: attribute - attribute: prefetch } field multifloat type array<float> { indexing: attribute - attribute: prefetch } field wsfloat type weightedset<float> { indexing: attribute - attribute: prefetch } field singledouble type double { indexing: attribute - attribute: prefetch } field multidouble type array<double> { indexing: attribute - attribute: prefetch } field wsdouble type weightedset<double> { indexing: attribute - attribute: prefetch } field singlestring type string { indexing: attribute - attribute: prefetch match { token } } field multistring type array<string> { indexing: attribute - attribute: prefetch match { token } } field wsstring type weightedset<string> { indexing: attribute - attribute: prefetch match { token } diff --git a/config-model/src/test/derived/attributeprefetch/summary.cfg b/config-model/src/test/derived/attributeprefetch/summary.cfg index a0d7bc11662..13b17464946 100644 --- a/config-model/src/test/derived/attributeprefetch/summary.cfg +++ b/config-model/src/test/derived/attributeprefetch/summary.cfg @@ -1,51 +1,27 @@ defaultsummaryid 1151071433 -classes[].id 1151071433 -classes[].name "default" -classes[].fields[].name "rankfeatures" -classes[].fields[].type "featuredata" -classes[].fields[].name "summaryfeatures" -classes[].fields[].type "featuredata" -classes[].fields[].name "documentid" -classes[].fields[].type "longstring" -classes[].id 1279606967 -classes[].name "attributeprefetch" -classes[].fields[].name "singlebyte" -classes[].fields[].type "byte" -classes[].fields[].name "multibyte" -classes[].fields[].type "jsonstring" -classes[].fields[].name "wsbyte" -classes[].fields[].type "jsonstring" -classes[].fields[].name "singleint" -classes[].fields[].type "integer" -classes[].fields[].name "multiint" -classes[].fields[].type "jsonstring" -classes[].fields[].name "wsint" -classes[].fields[].type "jsonstring" -classes[].fields[].name "singlelong" -classes[].fields[].type "int64" -classes[].fields[].name "multilong" -classes[].fields[].type "jsonstring" -classes[].fields[].name "wslong" -classes[].fields[].type "jsonstring" -classes[].fields[].name "singlefloat" -classes[].fields[].type "float" -classes[].fields[].name "multifloat" -classes[].fields[].type "jsonstring" -classes[].fields[].name "wsfloat" -classes[].fields[].type "jsonstring" -classes[].fields[].name "singledouble" -classes[].fields[].type "double" -classes[].fields[].name "multidouble" -classes[].fields[].type "jsonstring" -classes[].fields[].name "wsdouble" -classes[].fields[].type "jsonstring" -classes[].fields[].name "singlestring" -classes[].fields[].type "longstring" -classes[].fields[].name "multistring" -classes[].fields[].type "jsonstring" -classes[].fields[].name "wsstring" -classes[].fields[].type "jsonstring" -classes[].fields[].name "rankfeatures" -classes[].fields[].type "featuredata" -classes[].fields[].name "summaryfeatures" -classes[].fields[].type "featuredata" +classes[0].id 1151071433 +classes[0].name "default" +classes[0].fields[0].name "rankfeatures" +classes[0].fields[0].type "featuredata" +classes[0].fields[1].name "summaryfeatures" +classes[0].fields[1].type "featuredata" +classes[0].fields[2].name "documentid" +classes[0].fields[2].type "longstring" +classes[1].id 1980470965 +classes[1].name "attributeprefetch" +classes[1].fields[0].name "singlebyte" +classes[1].fields[0].type "byte" +classes[1].fields[1].name "singleint" +classes[1].fields[1].type "integer" +classes[1].fields[2].name "singlelong" +classes[1].fields[2].type "int64" +classes[1].fields[3].name "singlefloat" +classes[1].fields[3].type "float" +classes[1].fields[4].name "singledouble" +classes[1].fields[4].type "double" +classes[1].fields[5].name "singlestring" +classes[1].fields[5].type "longstring" +classes[1].fields[6].name "rankfeatures" +classes[1].fields[6].type "featuredata" +classes[1].fields[7].name "summaryfeatures" +classes[1].fields[7].type "featuredata"
\ No newline at end of file diff --git a/config-model/src/test/derived/attributeprefetch/summarymap.cfg b/config-model/src/test/derived/attributeprefetch/summarymap.cfg index ca1cff1b640..28bc686a266 100644 --- a/config-model/src/test/derived/attributeprefetch/summarymap.cfg +++ b/config-model/src/test/derived/attributeprefetch/summarymap.cfg @@ -1,61 +1,25 @@ defaultoutputclass -1 -override[].field "rankfeatures" -override[].command "rankfeatures" -override[].arguments "" -override[].field "summaryfeatures" -override[].command "summaryfeatures" -override[].arguments "" -override[].field "singlebyte" -override[].command "attribute" -override[].arguments "singlebyte" -override[].field "multibyte" -override[].command "attribute" -override[].arguments "multibyte" -override[].field "wsbyte" -override[].command "attribute" -override[].arguments "wsbyte" -override[].field "singleint" -override[].command "attribute" -override[].arguments "singleint" -override[].field "multiint" -override[].command "attribute" -override[].arguments "multiint" -override[].field "wsint" -override[].command "attribute" -override[].arguments "wsint" -override[].field "singlelong" -override[].command "attribute" -override[].arguments "singlelong" -override[].field "multilong" -override[].command "attribute" -override[].arguments "multilong" -override[].field "wslong" -override[].command "attribute" -override[].arguments "wslong" -override[].field "singlefloat" -override[].command "attribute" -override[].arguments "singlefloat" -override[].field "multifloat" -override[].command "attribute" -override[].arguments "multifloat" -override[].field "wsfloat" -override[].command "attribute" -override[].arguments "wsfloat" -override[].field "singledouble" -override[].command "attribute" -override[].arguments "singledouble" -override[].field "multidouble" -override[].command "attribute" -override[].arguments "multidouble" -override[].field "wsdouble" -override[].command "attribute" -override[].arguments "wsdouble" -override[].field "singlestring" -override[].command "attribute" -override[].arguments "singlestring" -override[].field "multistring" -override[].command "attribute" -override[].arguments "multistring" -override[].field "wsstring" -override[].command "attribute" -override[].arguments "wsstring" +override[0].field "rankfeatures" +override[0].command "rankfeatures" +override[0].arguments "" +override[1].field "summaryfeatures" +override[1].command "summaryfeatures" +override[1].arguments "" +override[2].field "singlebyte" +override[2].command "attribute" +override[2].arguments "singlebyte" +override[3].field "singleint" +override[3].command "attribute" +override[3].arguments "singleint" +override[4].field "singlelong" +override[4].command "attribute" +override[4].arguments "singlelong" +override[5].field "singlefloat" +override[5].command "attribute" +override[5].arguments "singlefloat" +override[6].field "singledouble" +override[6].command "attribute" +override[6].arguments "singledouble" +override[7].field "singlestring" +override[7].command "attribute" +override[7].arguments "singlestring"
\ No newline at end of file diff --git a/config-model/src/test/derived/attributes/ilscripts.cfg b/config-model/src/test/derived/attributes/ilscripts.cfg index 08bf52021d7..536394ab11d 100644 --- a/config-model/src/test/derived/attributes/ilscripts.cfg +++ b/config-model/src/test/derived/attributes/ilscripts.cfg @@ -26,13 +26,13 @@ ilscript[].content[] "clear_state | guard { input a1 | attribute a1 | summary a1 ilscript[].content[] "clear_state | guard { input a2 | attribute a2 | summary a2; }" ilscript[].content[] "clear_state | guard { input a3 | attribute a3 | summary a3; }" ilscript[].content[] "clear_state | guard { input a4 | summary a4; }" -ilscript[].content[] "clear_state | guard { input a5 | tokenize normalize stem:\"SHORTEST\" | attribute a5 | summary a5 | index a5; }" -ilscript[].content[] "clear_state | guard { input a6 | tokenize normalize stem:\"SHORTEST\" | attribute a6 | summary a6 | index a6; }" +ilscript[].content[] "clear_state | guard { input a5 | tokenize normalize stem:\"BEST\" | attribute a5 | summary a5 | index a5; }" +ilscript[].content[] "clear_state | guard { input a6 | tokenize normalize stem:\"BEST\" | attribute a6 | summary a6 | index a6; }" ilscript[].content[] "clear_state | guard { input a7 | summary a7; }" ilscript[].content[] "clear_state | guard { input a8 | summary a8; }" ilscript[].content[] "clear_state | guard { input b1 | attribute b1 | summary b1; }" -ilscript[].content[] "clear_state | guard { input b2 | tokenize normalize stem:\"SHORTEST\" | attribute b2 | summary b2 | index b2; }" -ilscript[].content[] "clear_state | guard { input b3 | tokenize normalize stem:\"SHORTEST\" | attribute b3 | summary b3 | index b3; }" +ilscript[].content[] "clear_state | guard { input b2 | tokenize normalize stem:\"BEST\" | attribute b2 | summary b2 | index b2; }" +ilscript[].content[] "clear_state | guard { input b3 | tokenize normalize stem:\"BEST\" | attribute b3 | summary b3 | index b3; }" ilscript[].content[] "clear_state | guard { input b4 | attribute b4 | summary b4; }" ilscript[].content[] "clear_state | guard { input b5 | summary b5 | attribute b5; }" ilscript[].content[] "clear_state | guard { input b6 | summary b6 | attribute b6; }" diff --git a/config-model/src/test/derived/attributes/index-info.cfg b/config-model/src/test/derived/attributes/index-info.cfg index ef96a1dd310..df33a41b066 100644 --- a/config-model/src/test/derived/attributes/index-info.cfg +++ b/config-model/src/test/derived/attributes/index-info.cfg @@ -22,7 +22,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "a5" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "a5" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "a5" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "a5" @@ -32,7 +32,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "a6" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "a6" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "a6" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "a6" @@ -52,7 +52,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "b2" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "b2" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "b2" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "b2" @@ -62,7 +62,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "b3" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "b3" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "b3" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "b3" @@ -144,7 +144,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "default" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "default" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "default" indexinfo[].command[].command "normalize" indexinfo[].alias[].alias "date" diff --git a/config-model/src/test/derived/combinedattributeandindexsearch/index-info.cfg b/config-model/src/test/derived/combinedattributeandindexsearch/index-info.cfg index 85939113adf..d34b6557e3b 100644 --- a/config-model/src/test/derived/combinedattributeandindexsearch/index-info.cfg +++ b/config-model/src/test/derived/combinedattributeandindexsearch/index-info.cfg @@ -8,7 +8,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "index1" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "index1" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "index1" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "index1" @@ -18,7 +18,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "index2" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "index2" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "index2" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "index2" @@ -42,6 +42,6 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "default" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "default" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "default" indexinfo[].command[].command "normalize" diff --git a/config-model/src/test/derived/complex/ilscripts.cfg b/config-model/src/test/derived/complex/ilscripts.cfg index dfef7058936..f7f6c9dd720 100644 --- a/config-model/src/test/derived/complex/ilscripts.cfg +++ b/config-model/src/test/derived/complex/ilscripts.cfg @@ -23,11 +23,11 @@ ilscript[].docfield[] "combineda" ilscript[].docfield[] "combinedb" ilscript[].docfield[] "category" ilscript[].content[] "clear_state | guard { input title . input category | tokenize | summary exact | index exact; }" -ilscript[].content[] "clear_state | guard { input location | tokenize normalize stem:\"SHORTEST\" | summary woe | index woe; }" +ilscript[].content[] "clear_state | guard { input location | tokenize normalize stem:\"BEST\" | summary woe | index woe; }" ilscript[].content[] "clear_state | guard { input yEaR | to_array | attribute year_arr; }" ilscript[].content[] "clear_state | guard { input yEaR - 1900 | attribute year_sub; }" -ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"SHORTEST\" | index title | summary title; }" -ilscript[].content[] "clear_state | guard { input dyntitle | tokenize normalize stem:\"SHORTEST\" | summary dyntitle; }" +ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | index title | summary title; }" +ilscript[].content[] "clear_state | guard { input dyntitle | tokenize normalize stem:\"BEST\" | summary dyntitle; }" ilscript[].content[] "clear_state | guard { input special1 | tokenize normalize | index special1; }" ilscript[].content[] "clear_state | guard { input special2 | tokenize normalize | index special2; }" ilscript[].content[] "clear_state | guard { input special3 | tokenize normalize | index special3; }" @@ -38,12 +38,12 @@ ilscript[].content[] "clear_state | guard { input fleeting | attribute fleeting; ilscript[].content[] "clear_state | guard { input fleeting2 | attribute fleeting2; }" ilscript[].content[] "clear_state | guard { input foundat | attribute foundat; }" ilscript[].content[] "clear_state | guard { input collapseby | attribute collapseby; }" -ilscript[].content[] "clear_state | guard { input stringfield | tokenize normalize stem:\"SHORTEST\" | summary stringfield | index stringfield; }" -ilscript[].content[] "clear_state | guard { input exactemento | tokenize normalize stem:\"SHORTEST\" | index exactemento; }" -ilscript[].content[] "clear_state | guard { input exactagain | tokenize normalize stem:\"SHORTEST\" | index exactagain; }" +ilscript[].content[] "clear_state | guard { input stringfield | tokenize normalize stem:\"BEST\" | summary stringfield | index stringfield; }" +ilscript[].content[] "clear_state | guard { input exactemento | tokenize normalize stem:\"BEST\" | index exactemento; }" +ilscript[].content[] "clear_state | guard { input exactagain | tokenize normalize stem:\"BEST\" | index exactagain; }" ilscript[].content[] "clear_state | guard { input ts | attribute ts; }" ilscript[].content[] "clear_state | guard { input combineda | attribute combineda; }" -ilscript[].content[] "clear_state | guard { input combinedb | tokenize normalize stem:\"SHORTEST\" | index combinedb; }" +ilscript[].content[] "clear_state | guard { input combinedb | tokenize normalize stem:\"BEST\" | index combinedb; }" ilscript[].content[] "input category | passthrough category" ilscript[].content[] "input location | passthrough location" ilscript[].content[] "input yEaR | passthrough yEaR" diff --git a/config-model/src/test/derived/emptydefault/ilscripts.cfg b/config-model/src/test/derived/emptydefault/ilscripts.cfg index 66dbf01d214..b17920fcaab 100644 --- a/config-model/src/test/derived/emptydefault/ilscripts.cfg +++ b/config-model/src/test/derived/emptydefault/ilscripts.cfg @@ -3,5 +3,5 @@ fieldmatchmaxlength 1000000 ilscript[].doctype "emptydefault" ilscript[].docfield[] "one" ilscript[].docfield[] "two" -ilscript[].content[] "clear_state | guard { input one | tokenize normalize stem:\"SHORTEST\" | index one; }" -ilscript[].content[] "clear_state | guard { input two | tokenize normalize stem:\"SHORTEST\" | index two; }" +ilscript[].content[] "clear_state | guard { input one | tokenize normalize stem:\"BEST\" | index one; }" +ilscript[].content[] "clear_state | guard { input two | tokenize normalize stem:\"BEST\" | index two; }" diff --git a/config-model/src/test/derived/emptydefault/index-info.cfg b/config-model/src/test/derived/emptydefault/index-info.cfg index d773f544abb..fbb4c3ebbea 100644 --- a/config-model/src/test/derived/emptydefault/index-info.cfg +++ b/config-model/src/test/derived/emptydefault/index-info.cfg @@ -8,7 +8,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "one" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "one" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "one" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "one" @@ -18,7 +18,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "two" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "two" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "two" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "two" diff --git a/config-model/src/test/derived/imported_position_field/imported-fields.cfg b/config-model/src/test/derived/imported_position_field/imported-fields.cfg index 51b371827c5..5ce853e17f2 100644 --- a/config-model/src/test/derived/imported_position_field/imported-fields.cfg +++ b/config-model/src/test/derived/imported_position_field/imported-fields.cfg @@ -1,5 +1,3 @@ attribute[0].name "my_pos_zcurve" attribute[0].referencefield "parent_ref" attribute[0].targetfield "pos_zcurve" -attribute[0].datatype NONE -attribute[0].collectiontype SINGLE
\ No newline at end of file diff --git a/config-model/src/test/derived/imported_struct_fields/imported-fields.cfg b/config-model/src/test/derived/imported_struct_fields/imported-fields.cfg index 772e5cffec9..699a3045075 100644 --- a/config-model/src/test/derived/imported_struct_fields/imported-fields.cfg +++ b/config-model/src/test/derived/imported_struct_fields/imported-fields.cfg @@ -1,35 +1,21 @@ attribute[0].name "my_elem_array.name" attribute[0].referencefield "parent_ref" attribute[0].targetfield "elem_array.name" -attribute[0].datatype NONE -attribute[0].collectiontype SINGLE attribute[1].name "my_elem_array.weight" attribute[1].referencefield "parent_ref" attribute[1].targetfield "elem_array.weight" -attribute[1].datatype NONE -attribute[1].collectiontype SINGLE attribute[2].name "my_elem_map.key" attribute[2].referencefield "parent_ref" attribute[2].targetfield "elem_map.key" -attribute[2].datatype NONE -attribute[2].collectiontype SINGLE attribute[3].name "my_elem_map.value.name" attribute[3].referencefield "parent_ref" attribute[3].targetfield "elem_map.value.name" -attribute[3].datatype NONE -attribute[3].collectiontype SINGLE attribute[4].name "my_elem_map.value.weight" attribute[4].referencefield "parent_ref" attribute[4].targetfield "elem_map.value.weight" -attribute[4].datatype NONE -attribute[4].collectiontype SINGLE attribute[5].name "my_str_int_map.key" attribute[5].referencefield "parent_ref" attribute[5].targetfield "str_int_map.key" -attribute[5].datatype NONE -attribute[5].collectiontype SINGLE attribute[6].name "my_str_int_map.value" attribute[6].referencefield "parent_ref" attribute[6].targetfield "str_int_map.value" -attribute[6].datatype NONE -attribute[6].collectiontype SINGLE
\ No newline at end of file diff --git a/config-model/src/test/derived/importedfields/imported-fields.cfg b/config-model/src/test/derived/importedfields/imported-fields.cfg index d1895d02290..c8de7d4fb7e 100644 --- a/config-model/src/test/derived/importedfields/imported-fields.cfg +++ b/config-model/src/test/derived/importedfields/imported-fields.cfg @@ -1,25 +1,15 @@ attribute[].name "my_int_field" attribute[].referencefield "a_ref" attribute[].targetfield "int_field" -attribute[].datatype NONE -attribute[].collectiontype SINGLE attribute[].name "my_string_field" attribute[].referencefield "b_ref" attribute[].targetfield "string_field" -attribute[].datatype NONE -attribute[].collectiontype SINGLE attribute[].name "my_int_array_field" attribute[].referencefield "a_ref" attribute[].targetfield "int_array_field" -attribute[].datatype NONE -attribute[].collectiontype SINGLE attribute[].name "my_int_wset_field" attribute[].referencefield "a_ref" attribute[].targetfield "int_wset_field" -attribute[].datatype NONE -attribute[].collectiontype SINGLE attribute[].name "my_ancient_int_field" attribute[].referencefield "a_ref" attribute[].targetfield "ancient_int_field" -attribute[].datatype NONE -attribute[].collectiontype SINGLE diff --git a/config-model/src/test/derived/indexinfo_fieldsets/index-info.cfg b/config-model/src/test/derived/indexinfo_fieldsets/index-info.cfg index aad17d1dbf2..3e0bad5e172 100644 --- a/config-model/src/test/derived/indexinfo_fieldsets/index-info.cfg +++ b/config-model/src/test/derived/indexinfo_fieldsets/index-info.cfg @@ -24,7 +24,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "nonormalizing1" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "nonormalizing1" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "nonormalizing1" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "nonormalizing2" @@ -32,7 +32,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "nonormalizing2" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "nonormalizing2" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "nonormalizing2" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "exact1" @@ -66,7 +66,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "nonormalizing" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "nonormalizing" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "exact" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "exact" diff --git a/config-model/src/test/derived/indexinfo_lowercase/index-info.cfg b/config-model/src/test/derived/indexinfo_lowercase/index-info.cfg index f78daad6f58..d397e37804c 100644 --- a/config-model/src/test/derived/indexinfo_lowercase/index-info.cfg +++ b/config-model/src/test/derived/indexinfo_lowercase/index-info.cfg @@ -20,7 +20,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "nc_index" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "nc_index" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "nc_index" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "nc_index" @@ -40,7 +40,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "lc_index" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "lc_index" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "lc_index" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "lc_index" @@ -68,7 +68,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "lc_set2" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "lc_set2" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "lc_set2" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "lc_set3" @@ -84,7 +84,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "lc_set4" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "lc_set4" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "lc_set4" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "lc_set5" @@ -102,7 +102,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "lc_set6" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "lc_set6" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "lc_set6" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "lc_set7" @@ -112,7 +112,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "lc_set7" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "lc_set7" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "lc_set7" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "nc_set1" @@ -128,7 +128,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "nc_set2" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "nc_set2" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "nc_set2" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "nc_set3" @@ -142,7 +142,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "nc_set4" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "nc_set4" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "nc_set4" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "nc_set5" @@ -158,7 +158,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "nc_set6" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "nc_set6" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "nc_set6" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "nc_set7" @@ -168,7 +168,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "nc_set7" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "nc_set7" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "nc_set7" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "nc_set8" @@ -178,7 +178,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "nc_set8" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "nc_set8" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "nc_set8" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "nc_set9" @@ -188,7 +188,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "nc_set9" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "nc_set9" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "nc_set9" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "nc_set10" @@ -214,7 +214,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "nc_set12" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "nc_set12" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "nc_set12" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "nc_set13" @@ -224,7 +224,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "nc_set13" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "nc_set13" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "nc_set13" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "nc_set14" @@ -234,7 +234,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "nc_set14" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "nc_set14" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "nc_set14" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "nc_set15" @@ -244,7 +244,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "nc_set15" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "nc_set15" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "nc_set15" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "nc_set16" @@ -254,7 +254,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "nc_set16" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "nc_set16" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "nc_set16" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "nc_set17" @@ -264,7 +264,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "nc_set17" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "nc_set17" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "nc_set17" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "nc_set18" @@ -274,7 +274,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "nc_set18" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "nc_set18" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "nc_set18" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "nc_set19" @@ -284,6 +284,6 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "nc_set19" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "nc_set19" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "nc_set19" indexinfo[].command[].command "normalize" diff --git a/config-model/src/test/derived/indexschema/index-info.cfg b/config-model/src/test/derived/indexschema/index-info.cfg index 6b05a34f2bd..2ba3a5a99ee 100644 --- a/config-model/src/test/derived/indexschema/index-info.cfg +++ b/config-model/src/test/derived/indexschema/index-info.cfg @@ -8,7 +8,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "sa" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "sa" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "sa" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "sa" @@ -18,7 +18,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "sb" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "sb" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "sb" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "sb" @@ -28,7 +28,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "sc" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "sc" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "sc" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "sc" @@ -38,7 +38,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "sd" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "sd" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "sd" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "sd" @@ -70,7 +70,7 @@ indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "sf" indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "sf" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "sf" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "sf" @@ -82,7 +82,7 @@ indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "sg" indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "sg" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "sg" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "sg" @@ -116,7 +116,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "si" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "si" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "si" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "si" @@ -188,7 +188,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "fs9" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "fs9" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "fs9" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "fs9" @@ -198,7 +198,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "f10.text" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "f10.text" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "f10.text" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "f10.text" @@ -266,7 +266,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "fs1" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "fs1" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "fs1" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "fs2" @@ -276,7 +276,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "fs2" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "fs2" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "fs2" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "fs3" @@ -290,7 +290,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "fs4" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "fs4" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "fs4" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "onlyattrib" @@ -320,7 +320,7 @@ indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "gram" indexinfo[].command[].command "index" indexinfo[].command[].indexname "gram" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "gram" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "gram" @@ -348,6 +348,6 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "default" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "default" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "default" indexinfo[].command[].command "normalize" diff --git a/config-model/src/test/derived/indexschema/indexschema.cfg b/config-model/src/test/derived/indexschema/indexschema.cfg index 7d081869b22..d1f43c4a81e 100644 --- a/config-model/src/test/derived/indexschema/indexschema.cfg +++ b/config-model/src/test/derived/indexschema/indexschema.cfg @@ -1,5 +1,4 @@ indexfield[].name "sa" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype SINGLE indexfield[].prefix false @@ -7,7 +6,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "sb" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype SINGLE indexfield[].prefix false @@ -15,7 +13,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "sc" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype SINGLE indexfield[].prefix false @@ -23,7 +20,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "sd" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype SINGLE indexfield[].prefix false @@ -31,7 +27,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "sf" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype ARRAY indexfield[].prefix false @@ -39,7 +34,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "sg" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype WEIGHTEDSET indexfield[].prefix false @@ -47,7 +41,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "sh" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype SINGLE indexfield[].prefix false @@ -55,7 +48,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "si" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype SINGLE indexfield[].prefix true @@ -63,7 +55,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "exact1" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype SINGLE indexfield[].prefix false @@ -71,7 +62,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "exact2" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype SINGLE indexfield[].prefix false @@ -79,7 +69,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "nostemstring1" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype SINGLE indexfield[].prefix false @@ -87,7 +76,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "nostemstring2" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype SINGLE indexfield[].prefix false @@ -95,7 +83,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "nostemstring3" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype SINGLE indexfield[].prefix false @@ -103,7 +90,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "nostemstring4" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype SINGLE indexfield[].prefix false @@ -111,7 +97,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "fs9" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype SINGLE indexfield[].prefix false @@ -119,7 +104,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "sd_literal" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype SINGLE indexfield[].prefix false @@ -127,7 +111,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "sh.fragment" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype SINGLE indexfield[].prefix false @@ -135,7 +118,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "sh.host" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype SINGLE indexfield[].prefix false @@ -143,7 +125,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "sh.hostname" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype SINGLE indexfield[].prefix false @@ -151,7 +132,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "sh.path" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype SINGLE indexfield[].prefix false @@ -159,7 +139,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "sh.port" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype SINGLE indexfield[].prefix false @@ -167,7 +146,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "sh.query" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype SINGLE indexfield[].prefix false @@ -175,7 +153,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "sh.scheme" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype SINGLE indexfield[].prefix false diff --git a/config-model/src/test/derived/indexswitches/ilscripts.cfg b/config-model/src/test/derived/indexswitches/ilscripts.cfg index 078895a8a48..77ac18e3261 100644 --- a/config-model/src/test/derived/indexswitches/ilscripts.cfg +++ b/config-model/src/test/derived/indexswitches/ilscripts.cfg @@ -5,6 +5,6 @@ ilscript[].docfield[] "title" ilscript[].docfield[] "descr" ilscript[].docfield[] "source_src" ilscript[].content[] "clear_state | guard { input source_src | switch { case \"theweb\": input source_src | tokenize normalize | summary source | index source; case \"amg\": input source_src | tokenize normalize | summary source; default: input source_src . \" partner\" | tokenize normalize | summary source | index source; }; }" -ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"SHORTEST\" | summary title | index title; }" -ilscript[].content[] "clear_state | guard { input descr | tokenize normalize stem:\"SHORTEST\" | summary descr | index descr; }" +ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }" +ilscript[].content[] "clear_state | guard { input descr | tokenize normalize stem:\"BEST\" | summary descr | index descr; }" ilscript[].content[] "input source_src | passthrough source_src" diff --git a/config-model/src/test/derived/indexswitches/index-info.cfg b/config-model/src/test/derived/indexswitches/index-info.cfg index eec20bdbff6..3af5f311d8b 100644 --- a/config-model/src/test/derived/indexswitches/index-info.cfg +++ b/config-model/src/test/derived/indexswitches/index-info.cfg @@ -8,7 +8,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "title" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "title" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "title" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "title" @@ -18,7 +18,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "descr" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "descr" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "descr" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "descr" @@ -44,6 +44,6 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "default" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "default" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "default" indexinfo[].command[].command "normalize" diff --git a/config-model/src/test/derived/inheritance/ilscripts.cfg b/config-model/src/test/derived/inheritance/ilscripts.cfg index 5856da6992e..91410489de8 100644 --- a/config-model/src/test/derived/inheritance/ilscripts.cfg +++ b/config-model/src/test/derived/inheritance/ilscripts.cfg @@ -9,5 +9,5 @@ ilscript[].docfield[] "onlychild" ilscript[].content[] "clear_state | guard { input onlygrandparent | attribute onlygrandparent; }" ilscript[].content[] "clear_state | guard { input overridden | attribute overridden; }" ilscript[].content[] "clear_state | guard { input onlyfather | summary onlyfather; }" -ilscript[].content[] "clear_state | guard { input onlymother | tokenize normalize stem:\"SHORTEST\" | attribute onlymother | index onlymother; }" -ilscript[].content[] "clear_state | guard { input onlychild | tokenize normalize stem:\"SHORTEST\" | index onlychild; }" +ilscript[].content[] "clear_state | guard { input onlymother | tokenize normalize stem:\"BEST\" | attribute onlymother | index onlymother; }" +ilscript[].content[] "clear_state | guard { input onlychild | tokenize normalize stem:\"BEST\" | index onlychild; }" diff --git a/config-model/src/test/derived/inheritance/index-info.cfg b/config-model/src/test/derived/inheritance/index-info.cfg index fe7bf334fcc..e0be7b7d71f 100644 --- a/config-model/src/test/derived/inheritance/index-info.cfg +++ b/config-model/src/test/derived/inheritance/index-info.cfg @@ -22,7 +22,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "onlymother" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "onlymother" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "onlymother" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "onlymother" @@ -32,7 +32,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "onlychild" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "onlychild" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "onlychild" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "onlychild" diff --git a/config-model/src/test/derived/inheritstruct/index-info.cfg b/config-model/src/test/derived/inheritstruct/index-info.cfg index f0476c4bbb8..e0f3fdb5dde 100644 --- a/config-model/src/test/derived/inheritstruct/index-info.cfg +++ b/config-model/src/test/derived/inheritstruct/index-info.cfg @@ -8,7 +8,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "child_struct_field.my_str" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "child_struct_field.my_str" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "child_struct_field.my_str" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "child_struct_field.my_str" diff --git a/config-model/src/test/derived/music/ilscripts.cfg b/config-model/src/test/derived/music/ilscripts.cfg index bbac0fff10c..7a02d836db5 100644 --- a/config-model/src/test/derived/music/ilscripts.cfg +++ b/config-model/src/test/derived/music/ilscripts.cfg @@ -39,39 +39,39 @@ ilscript[].docfield[] "powermetalvalue" ilscript[].docfield[] "progvalue" ilscript[].content[] "clear_state | guard { input hiphopvalue | split \";\" | attribute hiphopvalue_arr; }" ilscript[].content[] "clear_state | guard { input metalvalue | split \";\" | attribute metalvalue_arr; }" -ilscript[].content[] "clear_state | guard { input bgndata | tokenize normalize stem:\"SHORTEST\" | summary bgndata; }" +ilscript[].content[] "clear_state | guard { input bgndata | tokenize normalize stem:\"BEST\" | summary bgndata; }" ilscript[].content[] "clear_state | guard { input sales | summary sales | attribute sales; }" ilscript[].content[] "clear_state | guard { input pto | summary pto | attribute pto; }" -ilscript[].content[] "clear_state | guard { input keys | tokenize normalize stem:\"SHORTEST\" | index keys; }" +ilscript[].content[] "clear_state | guard { input keys | tokenize normalize stem:\"BEST\" | index keys; }" ilscript[].content[] "clear_state | guard { input mid | summary mid | attribute mid; }" -ilscript[].content[] "clear_state | guard { input ew | tokenize normalize stem:\"SHORTEST\" | summary ew | index ew; }" +ilscript[].content[] "clear_state | guard { input ew | tokenize normalize stem:\"BEST\" | summary ew | index ew; }" ilscript[].content[] "clear_state | guard { input surl | summary surl; }" ilscript[].content[] "clear_state | guard { input userrate | summary userrate; }" ilscript[].content[] "clear_state | guard { input pid | summary pid; }" ilscript[].content[] "clear_state | guard { input weight | summary weight | attribute weight; }" ilscript[].content[] "clear_state | guard { input url | summary url; }" ilscript[].content[] "clear_state | guard { input isbn | summary isbn; }" -ilscript[].content[] "clear_state | guard { input fmt | tokenize normalize stem:\"SHORTEST\" | summary fmt | index fmt; }" +ilscript[].content[] "clear_state | guard { input fmt | tokenize normalize stem:\"BEST\" | summary fmt | index fmt; }" ilscript[].content[] "clear_state | guard { input albumid | summary albumid; }" ilscript[].content[] "clear_state | guard { input disp_song | summary disp_song; }" -ilscript[].content[] "clear_state | guard { input song | tokenize normalize stem:\"SHORTEST\" | summary song | index song; }" +ilscript[].content[] "clear_state | guard { input song | tokenize normalize stem:\"BEST\" | summary song | index song; }" ilscript[].content[] "clear_state | guard { input pfrom | summary pfrom; }" ilscript[].content[] "clear_state | guard { input bgnpfrom | summary bgnpfrom | attribute bgnpfrom; }" -ilscript[].content[] "clear_state | guard { input categories | tokenize normalize stem:\"SHORTEST\" | summary categories | index categories; }" +ilscript[].content[] "clear_state | guard { input categories | tokenize normalize stem:\"BEST\" | summary categories | index categories; }" ilscript[].content[] "clear_state | guard { input data | summary data; }" ilscript[].content[] "clear_state | guard { input numreview | summary numreview; }" ilscript[].content[] "clear_state | guard { input bgnsellers | summary bgnsellers; }" ilscript[].content[] "clear_state | guard { input image | summary image; }" -ilscript[].content[] "clear_state | guard { input artist | tokenize normalize stem:\"SHORTEST\" | summary artist | index artist; }" +ilscript[].content[] "clear_state | guard { input artist | tokenize normalize stem:\"BEST\" | summary artist | index artist; }" ilscript[].content[] "clear_state | guard { input artistspid | summary artistspid; }" -ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"SHORTEST\" | summary title | index title; }" +ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }" ilscript[].content[] "clear_state | guard { input newestedition | summary newestedition | attribute newestedition; }" -ilscript[].content[] "clear_state | guard { input bgnpto | tokenize normalize stem:\"SHORTEST\" | summary bgnpto; }" +ilscript[].content[] "clear_state | guard { input bgnpto | tokenize normalize stem:\"BEST\" | summary bgnpto; }" ilscript[].content[] "clear_state | guard { input year | summary year | attribute year; }" ilscript[].content[] "clear_state | guard { input did | summary did | attribute did; }" ilscript[].content[] "clear_state | guard { input scorekey | summary scorekey; }" ilscript[].content[] "clear_state | guard { input cbid | summary cbid | attribute cbid; }" ilscript[].content[] "clear_state | guard { input metalvalue | summary metalvalue; }" ilscript[].content[] "clear_state | guard { input hiphopvalue | summary hiphopvalue; }" -ilscript[].content[] "clear_state | guard { input powermetalvalue | tokenize normalize stem:\"SHORTEST\" | index powermetalvalue | summary powermetalvalue; }" -ilscript[].content[] "clear_state | guard { input progvalue | tokenize normalize stem:\"SHORTEST\" | index progvalue | summary progvalue; }" +ilscript[].content[] "clear_state | guard { input powermetalvalue | tokenize normalize stem:\"BEST\" | index powermetalvalue | summary powermetalvalue; }" +ilscript[].content[] "clear_state | guard { input progvalue | tokenize normalize stem:\"BEST\" | index progvalue | summary progvalue; }" diff --git a/config-model/src/test/derived/music/index-info.cfg b/config-model/src/test/derived/music/index-info.cfg index b9264e81e14..5d4cb8133e5 100644 --- a/config-model/src/test/derived/music/index-info.cfg +++ b/config-model/src/test/derived/music/index-info.cfg @@ -22,7 +22,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "keys" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "keys" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "keys" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "keys" @@ -38,7 +38,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "ew" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "ew" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "ew" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "ew" @@ -66,7 +66,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "fmt" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "fmt" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "fmt" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "fmt" @@ -80,7 +80,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "song" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "song" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "song" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "song" @@ -100,7 +100,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "categories" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "categories" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "categories" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "categories" @@ -122,7 +122,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "artist" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "artist" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "artist" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "artist" @@ -134,7 +134,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "title" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "title" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "title" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "title" @@ -178,7 +178,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "powermetalvalue" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "powermetalvalue" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "powermetalvalue" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "powermetalvalue" @@ -188,7 +188,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "progvalue" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "progvalue" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "progvalue" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "progvalue" @@ -230,6 +230,6 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "default" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "default" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "default" indexinfo[].command[].command "normalize" diff --git a/config-model/src/test/derived/newrank/ilscripts.cfg b/config-model/src/test/derived/newrank/ilscripts.cfg index 6194e66ee33..6986f12f62a 100644 --- a/config-model/src/test/derived/newrank/ilscripts.cfg +++ b/config-model/src/test/derived/newrank/ilscripts.cfg @@ -33,34 +33,34 @@ ilscript[].docfield[] "year" ilscript[].docfield[] "did" ilscript[].docfield[] "scorekey" ilscript[].docfield[] "cbid" -ilscript[].content[] "clear_state | guard { input bgndata | tokenize normalize stem:\"SHORTEST\" | summary bgndata; }" +ilscript[].content[] "clear_state | guard { input bgndata | tokenize normalize stem:\"BEST\" | summary bgndata; }" ilscript[].content[] "clear_state | guard { input sales | summary sales | attribute sales; }" ilscript[].content[] "clear_state | guard { input pto | summary pto | attribute pto; }" -ilscript[].content[] "clear_state | guard { input keys | tokenize normalize stem:\"SHORTEST\" | index keys; }" +ilscript[].content[] "clear_state | guard { input keys | tokenize normalize stem:\"BEST\" | index keys; }" ilscript[].content[] "clear_state | guard { input mid | summary mid | attribute mid; }" -ilscript[].content[] "clear_state | guard { input ew | tokenize normalize stem:\"SHORTEST\" | summary ew | index ew; }" +ilscript[].content[] "clear_state | guard { input ew | tokenize normalize stem:\"BEST\" | summary ew | index ew; }" ilscript[].content[] "clear_state | guard { input surl | summary surl; }" ilscript[].content[] "clear_state | guard { input userrate | summary userrate; }" ilscript[].content[] "clear_state | guard { input pid | summary pid; }" ilscript[].content[] "clear_state | guard { input weight | summary weight | attribute weight; }" ilscript[].content[] "clear_state | guard { input url | summary url; }" ilscript[].content[] "clear_state | guard { input isbn | summary isbn; }" -ilscript[].content[] "clear_state | guard { input fmt | tokenize normalize stem:\"SHORTEST\" | summary fmt | index fmt; }" +ilscript[].content[] "clear_state | guard { input fmt | tokenize normalize stem:\"BEST\" | summary fmt | index fmt; }" ilscript[].content[] "clear_state | guard { input albumid | summary albumid; }" ilscript[].content[] "clear_state | guard { input disp_song | summary disp_song; }" -ilscript[].content[] "clear_state | guard { input song | tokenize normalize stem:\"SHORTEST\" | summary song | index song; }" +ilscript[].content[] "clear_state | guard { input song | tokenize normalize stem:\"BEST\" | summary song | index song; }" ilscript[].content[] "clear_state | guard { input pfrom | summary pfrom; }" ilscript[].content[] "clear_state | guard { input bgnpfrom | summary bgnpfrom | attribute bgnpfrom; }" -ilscript[].content[] "clear_state | guard { input categories | tokenize normalize stem:\"SHORTEST\" | summary categories | index categories; }" +ilscript[].content[] "clear_state | guard { input categories | tokenize normalize stem:\"BEST\" | summary categories | index categories; }" ilscript[].content[] "clear_state | guard { input data | summary data; }" ilscript[].content[] "clear_state | guard { input numreview | summary numreview; }" ilscript[].content[] "clear_state | guard { input bgnsellers | summary bgnsellers; }" ilscript[].content[] "clear_state | guard { input image | summary image; }" -ilscript[].content[] "clear_state | guard { input artist | tokenize normalize stem:\"SHORTEST\" | summary artist | index artist; }" +ilscript[].content[] "clear_state | guard { input artist | tokenize normalize stem:\"BEST\" | summary artist | index artist; }" ilscript[].content[] "clear_state | guard { input artistspid | summary artistspid; }" -ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"SHORTEST\" | summary title | index title; }" +ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }" ilscript[].content[] "clear_state | guard { input newestedition | summary newestedition | attribute newestedition; }" -ilscript[].content[] "clear_state | guard { input bgnpto | tokenize normalize stem:\"SHORTEST\" | summary bgnpto; }" +ilscript[].content[] "clear_state | guard { input bgnpto | tokenize normalize stem:\"BEST\" | summary bgnpto; }" ilscript[].content[] "clear_state | guard { input year | summary year | attribute year; }" ilscript[].content[] "clear_state | guard { input did | summary did | attribute did; }" ilscript[].content[] "clear_state | guard { input scorekey | summary scorekey | attribute scorekey; }" diff --git a/config-model/src/test/derived/newrank/index-info.cfg b/config-model/src/test/derived/newrank/index-info.cfg index 26d2b92fd10..96d34ee2548 100644 --- a/config-model/src/test/derived/newrank/index-info.cfg +++ b/config-model/src/test/derived/newrank/index-info.cfg @@ -22,7 +22,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "keys" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "keys" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "keys" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "keys" @@ -38,7 +38,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "ew" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "ew" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "ew" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "ew" @@ -66,7 +66,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "fmt" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "fmt" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "fmt" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "fmt" @@ -80,7 +80,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "song" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "song" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "song" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "song" @@ -100,7 +100,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "categories" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "categories" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "categories" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "categories" @@ -122,7 +122,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "artist" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "artist" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "artist" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "artist" @@ -134,7 +134,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "title" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "title" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "title" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "title" @@ -192,6 +192,6 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "default" indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "default" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "default" indexinfo[].command[].command "normalize" diff --git a/config-model/src/test/derived/orderilscripts/ilscripts.cfg b/config-model/src/test/derived/orderilscripts/ilscripts.cfg index 801c7e4d790..c41939b34d1 100644 --- a/config-model/src/test/derived/orderilscripts/ilscripts.cfg +++ b/config-model/src/test/derived/orderilscripts/ilscripts.cfg @@ -3,4 +3,4 @@ fieldmatchmaxlength 1000000 ilscript[].doctype "orderilscripts" ilscript[].docfield[] "foo" ilscript[].content[] "clear_state | guard { input foo | summary bar; }" -ilscript[].content[] "clear_state | guard { input foo | tokenize normalize stem:\"SHORTEST\" | summary foo | index foo; }" +ilscript[].content[] "clear_state | guard { input foo | tokenize normalize stem:\"BEST\" | summary foo | index foo; }" diff --git a/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg b/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg index 4663ef92fcf..a2ada9792a0 100644 --- a/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg +++ b/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg @@ -6,7 +6,7 @@ ilscript[].docfield[] "attributefield1" ilscript[].docfield[] "attributefield2" ilscript[].docfield[] "indexfield1" ilscript[].docfield[] "indexfield2" -ilscript[].content[] "clear_state | guard { input indexfield0 | tokenize normalize stem:\"SHORTEST\" max-length:79 | index indexfield0; }" +ilscript[].content[] "clear_state | guard { input indexfield0 | tokenize normalize stem:\"BEST\" max-length:79 | index indexfield0; }" ilscript[].content[] "clear_state | guard { input attributefield1 | attribute attributefield1; }" ilscript[].content[] "clear_state | guard { input attributefield2 | attribute attributefield2; }" ilscript[].content[] "clear_state | guard { input indexfield1 | exact | index indexfield1; }" diff --git a/config-model/src/test/derived/prefixexactattribute/index-info.cfg b/config-model/src/test/derived/prefixexactattribute/index-info.cfg index 6429d34c5f9..7a84f0515f8 100644 --- a/config-model/src/test/derived/prefixexactattribute/index-info.cfg +++ b/config-model/src/test/derived/prefixexactattribute/index-info.cfg @@ -8,7 +8,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "indexfield0" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "indexfield0" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "indexfield0" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "indexfield0" diff --git a/config-model/src/test/derived/ranktypes/ilscripts.cfg b/config-model/src/test/derived/ranktypes/ilscripts.cfg index 21fe564c922..9f82e75986c 100644 --- a/config-model/src/test/derived/ranktypes/ilscripts.cfg +++ b/config-model/src/test/derived/ranktypes/ilscripts.cfg @@ -6,7 +6,7 @@ ilscript[].docfield[] "descr" ilscript[].docfield[] "keywords" ilscript[].docfield[] "identity" ilscript[].content[] "clear_state | guard { input identity | tokenize | index identity_literal; }" -ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"SHORTEST\" | summary title | index title; }" -ilscript[].content[] "clear_state | guard { input descr | tokenize normalize stem:\"SHORTEST\" | summary descr | index descr; }" -ilscript[].content[] "clear_state | guard { input keywords | tokenize normalize stem:\"SHORTEST\" | index keywords; }" -ilscript[].content[] "clear_state | guard { input identity | tokenize normalize stem:\"SHORTEST\" | index identity; }" +ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }" +ilscript[].content[] "clear_state | guard { input descr | tokenize normalize stem:\"BEST\" | summary descr | index descr; }" +ilscript[].content[] "clear_state | guard { input keywords | tokenize normalize stem:\"BEST\" | index keywords; }" +ilscript[].content[] "clear_state | guard { input identity | tokenize normalize stem:\"BEST\" | index identity; }" diff --git a/config-model/src/test/derived/ranktypes/index-info.cfg b/config-model/src/test/derived/ranktypes/index-info.cfg index 3d4ee9e9239..0b9777ca488 100644 --- a/config-model/src/test/derived/ranktypes/index-info.cfg +++ b/config-model/src/test/derived/ranktypes/index-info.cfg @@ -8,7 +8,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "title" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "title" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "title" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "title" @@ -18,7 +18,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "descr" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "descr" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "descr" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "descr" @@ -28,7 +28,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "keywords" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "keywords" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "keywords" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "keywords" @@ -38,7 +38,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "identity" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "identity" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "identity" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "identity" diff --git a/config-model/src/test/derived/types/index-info.cfg b/config-model/src/test/derived/types/index-info.cfg index 576a95de06f..d6b914e5b1b 100644 --- a/config-model/src/test/derived/types/index-info.cfg +++ b/config-model/src/test/derived/types/index-info.cfg @@ -84,7 +84,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "stringmapfield.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stringmapfield.key" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "stringmapfield.key" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "stringmapfield.key" @@ -94,7 +94,7 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "stringmapfield.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stringmapfield.value" -indexinfo[].command[].command "stem:SHORTEST" +indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "stringmapfield.value" indexinfo[].command[].command "normalize" indexinfo[].command[].indexname "stringmapfield.value" diff --git a/config-model/src/test/derived/uri_array/indexschema.cfg b/config-model/src/test/derived/uri_array/indexschema.cfg index be7bf609579..8593c594e3c 100644 --- a/config-model/src/test/derived/uri_array/indexschema.cfg +++ b/config-model/src/test/derived/uri_array/indexschema.cfg @@ -1,5 +1,4 @@ indexfield[].name "my_uri" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype ARRAY indexfield[].prefix false @@ -7,7 +6,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "my_uri.fragment" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype ARRAY indexfield[].prefix false @@ -15,7 +13,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "my_uri.host" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype ARRAY indexfield[].prefix false @@ -23,7 +20,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "my_uri.hostname" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype ARRAY indexfield[].prefix false @@ -31,7 +27,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "my_uri.path" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype ARRAY indexfield[].prefix false @@ -39,7 +34,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "my_uri.port" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype ARRAY indexfield[].prefix false @@ -47,7 +41,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "my_uri.query" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype ARRAY indexfield[].prefix false @@ -55,7 +48,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "my_uri.scheme" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype ARRAY indexfield[].prefix false diff --git a/config-model/src/test/derived/uri_wset/indexschema.cfg b/config-model/src/test/derived/uri_wset/indexschema.cfg index 5997a47c48a..a432556bb6b 100644 --- a/config-model/src/test/derived/uri_wset/indexschema.cfg +++ b/config-model/src/test/derived/uri_wset/indexschema.cfg @@ -1,5 +1,4 @@ indexfield[].name "my_uri" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype WEIGHTEDSET indexfield[].prefix false @@ -7,7 +6,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "my_uri.fragment" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype WEIGHTEDSET indexfield[].prefix false @@ -15,7 +13,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "my_uri.host" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype WEIGHTEDSET indexfield[].prefix false @@ -23,7 +20,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "my_uri.hostname" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype WEIGHTEDSET indexfield[].prefix false @@ -31,7 +27,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "my_uri.path" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype WEIGHTEDSET indexfield[].prefix false @@ -39,7 +34,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "my_uri.port" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype WEIGHTEDSET indexfield[].prefix false @@ -47,7 +41,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "my_uri.query" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype WEIGHTEDSET indexfield[].prefix false @@ -55,7 +48,6 @@ indexfield[].phrases false indexfield[].positions true indexfield[].averageelementlen 512 indexfield[].name "my_uri.scheme" -indexfield[].indextype VESPA indexfield[].datatype STRING indexfield[].collectiontype WEIGHTEDSET indexfield[].prefix false diff --git a/config-model/src/test/examples/disksummary.sd b/config-model/src/test/examples/disksummary.sd deleted file mode 100644 index 766f0e0d2e1..00000000000 --- a/config-model/src/test/examples/disksummary.sd +++ /dev/null @@ -1,14 +0,0 @@ -search disksummary { - - document disksummary { - - field inmemory type string { - indexing: attribute | summary - } - field ondisk type string { - indexing: index | summary - } - - } - -}
\ No newline at end of file diff --git a/config-model/src/test/examples/disksummaryexplicit.sd b/config-model/src/test/examples/disksummaryexplicit.sd deleted file mode 100644 index b0179e04801..00000000000 --- a/config-model/src/test/examples/disksummaryexplicit.sd +++ /dev/null @@ -1,18 +0,0 @@ -search disksummary { - - document disksummary { - - field inmemory type string { - indexing: attribute | summary - } - field ondisk type string { - indexing: index | summary - } - - } - - document-summary default { - from-disk - } - -}
\ No newline at end of file diff --git a/config-model/src/test/examples/indexsettings.sd b/config-model/src/test/examples/indexsettings.sd index 6832527f86e..9f6b02d9a4d 100644 --- a/config-model/src/test/examples/indexsettings.sd +++ b/config-model/src/test/examples/indexsettings.sd @@ -6,7 +6,7 @@ search indexsettings { document indexsettings { field allstemmed type string { - stemming: all + stemming: shortest } field usingdefault type string { diff --git a/config-model/src/test/examples/memorysummary.sd b/config-model/src/test/examples/memorysummary.sd deleted file mode 100644 index 79be30bc0fd..00000000000 --- a/config-model/src/test/examples/memorysummary.sd +++ /dev/null @@ -1,14 +0,0 @@ -search memorysummary { - - document memorysummary { - - field inmemory type string { - indexing: attribute | summary - } - field ondisk type string { - indexing: index # no summary, so ignored - } - - } - -}
\ No newline at end of file diff --git a/config-model/src/test/examples/stemmingsetting.sd b/config-model/src/test/examples/stemmingsetting.sd index 3780dfb3699..51864193174 100644 --- a/config-model/src/test/examples/stemmingsetting.sd +++ b/config-model/src/test/examples/stemmingsetting.sd @@ -25,7 +25,7 @@ search stemmingsetting { field backward type string { indexing: index - stemming: all + stemming: shortest } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java index a3f3f9cccb2..8ccb1ed969a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java @@ -15,11 +15,13 @@ import static org.junit.Assert.assertEquals; * @author bratseth */ public class CommentTestCase extends SearchDefinitionTestCase { + @Test public void testComments() throws IOException, ParseException { Search search = SearchBuilder.buildFromFile("src/test/examples/comment.sd"); SDField field = search.getConcreteField("a"); - assertEquals("{ input a | tokenize normalize stem:\"SHORTEST\" | summary a | index a; }", + assertEquals("{ input a | tokenize normalize stem:\"BEST\" | summary a | index a; }", field.getIndexingScript().toString()); } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java index 0c464d668f7..89e551fa789 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java @@ -55,7 +55,7 @@ public class SearchImporterTestCase extends SearchDefinitionTestCase { // First field field=(SDField) document.getField("title"); assertEquals(DataType.STRING,field.getDataType()); - assertEquals("{ input title | tokenize normalize stem:\"SHORTEST\" | summary title | index title; }", field.getIndexingScript().toString()); + assertEquals("{ input title | tokenize normalize stem:\"BEST\" | summary title | index title; }", field.getIndexingScript().toString()); assertTrue(!search.getIndex("default").isPrefix()); assertTrue(search.getIndex("title").isPrefix()); Iterator<String> titleAliases=search.getIndex("title").aliasIterator(); @@ -105,19 +105,19 @@ public class SearchImporterTestCase extends SearchDefinitionTestCase { // Seventh field field= search.getConcreteField("categories"); - assertEquals("{ input categories_src | lowercase | normalize | tokenize normalize stem:\"SHORTEST\" | index categories; }", + assertEquals("{ input categories_src | lowercase | normalize | tokenize normalize stem:\"BEST\" | index categories; }", field.getIndexingScript().toString()); - assertTrue(!field.isHeader()); + assertTrue(field.isHeader()); // Eight field field= search.getConcreteField("categoriesagain"); - assertEquals("{ input categoriesagain_src | lowercase | normalize | tokenize normalize stem:\"SHORTEST\" | index categoriesagain; }", + assertEquals("{ input categoriesagain_src | lowercase | normalize | tokenize normalize stem:\"BEST\" | index categoriesagain; }", field.getIndexingScript().toString()); assertTrue(field.isHeader()); // Ninth field field= search.getConcreteField("exactemento"); - assertEquals("{ input exactemento_src | lowercase | tokenize normalize stem:\"SHORTEST\" | index exactemento | summary exactemento; }", + assertEquals("{ input exactemento_src | lowercase | tokenize normalize stem:\"BEST\" | index exactemento | summary exactemento; }", field.getIndexingScript().toString()); // Tenth field diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java index defff99d51e..9b27d338ced 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java @@ -42,10 +42,10 @@ public class StemmingSettingTestCase extends SearchDefinitionTestCase { } @Test - public void requireThatStemmingIsDefaultShortest() throws IOException, ParseException { + public void requireThatStemmingIsDefaultBest() throws IOException, ParseException { Search search = SearchBuilder.buildFromFile("src/test/examples/stemmingdefault.sd"); assertNull(search.getConcreteField("my_str").getStemming()); - assertEquals(Stemming.SHORTEST, search.getConcreteField("my_str").getStemming(search)); + assertEquals(Stemming.BEST, search.getConcreteField("my_str").getStemming(search)); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java index 6fa716d9b76..7e6eaa0683a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java @@ -19,28 +19,83 @@ import static org.junit.Assert.assertTrue; public class SummaryTestCase { @Test - public void testMemorySummary() throws IOException, ParseException { + public void testMemorySummary() throws ParseException { + String sd = + "search memorysummary {\n" + + "\n" + + " document memorysummary {\n" + + "\n" + + " field inmemory type string {\n" + + " indexing: attribute | summary\n" + + " }\n" + + " field ondisk type string {\n" + + " indexing: index # no summary, so ignored\n" + + " }\n" + + "\n" + + " }\n" + + "\n" + + "}"; DeployLoggerStub logger = new DeployLoggerStub(); - SearchBuilder.createFromFile("src/test/examples/memorysummary.sd", logger); + SearchBuilder.createFromString(sd, logger); assertTrue(logger.entries.isEmpty()); } @Test - public void testDiskSummary() throws IOException, ParseException { + public void testDiskSummary() throws ParseException { + String sd = + "search disksummary {\n" + + "\n" + + " document-summary foobar {\n" + + " summary foo1 type string { source: inmemory }\n" + + " summary foo2 type string { source: ondisk }\n" + + " }\n" + + " document disksummary {\n" + + "\n" + + " field inmemory type string {\n" + + " indexing: attribute | summary\n" + + " }\n" + + " field ondisk type string {\n" + + " indexing: index | summary\n" + + " }\n" + + "\n" + + " }\n" + + "\n" + + "}"; DeployLoggerStub logger = new DeployLoggerStub(); - SearchBuilder.createFromFile("src/test/examples/disksummary.sd", logger); + SearchBuilder.createFromString(sd, logger); assertEquals(1, logger.entries.size()); - assertEquals(Level.FINE, logger.entries.get(0).level); - assertEquals("summary field 'ondisk' in document summary 'default' references source field 'ondisk', " + + assertEquals(Level.WARNING, logger.entries.get(0).level); + assertEquals("summary field 'foo2' in document summary 'foobar' references source field 'ondisk', " + "which is not an attribute: Using this summary will cause disk accesses. " + "Set 'from-disk' on this summary class to silence this warning.", logger.entries.get(0).message); } @Test - public void testDiskSummaryExplicit() throws IOException, ParseException { + public void testDiskSummaryExplicit() throws ParseException { + String sd = + "search disksummary {\n" + + "\n" + + " document disksummary {\n" + + "\n" + + " field inmemory type string {\n" + + " indexing: attribute | summary\n" + + " }\n" + + " field ondisk type string {\n" + + " indexing: index | summary\n" + + " }\n" + + "\n" + + " }\n" + + "\n" + + " document-summary foobar {\n" + + " summary foo1 type string { source: inmemory }\n" + + " summary foo2 type string { source: ondisk }\n" + + " from-disk\n" + + " }\n" + + "\n" + + "}"; DeployLoggerStub logger = new DeployLoggerStub(); - SearchBuilder.createFromFile("src/test/examples/disksummaryexplicit.sd", logger); + SearchBuilder.createFromString(sd, logger); assertTrue(logger.entries.isEmpty()); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java index 21739705e30..be720cbe4dd 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java @@ -48,7 +48,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { derived.getAttributeFields(); // TODO: assert content // Check il script addition - assertIndexing(Arrays.asList("clear_state | guard { input a | tokenize normalize stem:\"SHORTEST\" | index a; }", + assertIndexing(Arrays.asList("clear_state | guard { input a | tokenize normalize stem:\"BEST\" | index a; }", "clear_state | guard { input a | tokenize | index a_literal; }"), search); @@ -76,7 +76,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { DerivedConfiguration derived = new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(),new ImportedMlModels()); // Check il script addition - assertIndexing(Arrays.asList("clear_state | guard { input a | tokenize normalize stem:\"SHORTEST\" | index a; }", + assertIndexing(Arrays.asList("clear_state | guard { input a | tokenize normalize stem:\"BEST\" | index a; }", "clear_state | guard { input a | tokenize | index a_literal; }"), search); @@ -101,8 +101,8 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { search = SearchBuilder.buildFromRawSearch(search, rankProfileRegistry, new QueryProfileRegistry()); new DerivedConfiguration(search, rankProfileRegistry, new QueryProfileRegistry(), new ImportedMlModels()); - assertIndexing(Arrays.asList("clear_state | guard { input title | tokenize normalize stem:\"SHORTEST\" | summary title | index title; }", - "clear_state | guard { input body | tokenize normalize stem:\"SHORTEST\" | summary body | index body; }", + assertIndexing(Arrays.asList("clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }", + "clear_state | guard { input body | tokenize normalize stem:\"BEST\" | summary body | index body; }", "clear_state | guard { input title | tokenize | index title_literal; }", "clear_state | guard { input body | tokenize | index body_literal; }"), search); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/OrderIlscriptsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/OrderIlscriptsTestCase.java index b678c407cf9..428522d3e05 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/OrderIlscriptsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/OrderIlscriptsTestCase.java @@ -7,11 +7,13 @@ import org.junit.Test; import java.io.IOException; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class OrderIlscriptsTestCase extends AbstractExportingTestCase { + @Test public void testOrderIlscripts() throws IOException, ParseException { assertCorrectDeriving("orderilscripts"); } + } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java index e078d91f248..d313c2391fd 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java @@ -46,12 +46,12 @@ public class IndexingScriptRewriterTestCase extends SearchDefinitionTestCase { public void testDynamicSummaryRewriting() { SDField field = createField("test", DataType.STRING, "{ summary }"); field.addSummaryField(createDynamicSummaryField(field, "dyn")); - assertIndexingScript("{ input test | tokenize normalize stem:\"SHORTEST\" | summary dyn; }", field); + assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | summary dyn; }", field); } @Test public void testSummaryRewritingWithIndexing() { - assertIndexingScript("{ input test | tokenize normalize stem:\"SHORTEST\" | summary test | index test; }", + assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | summary test | index test; }", createField("test", DataType.STRING, "{ summary | index }")); } @@ -62,7 +62,7 @@ public class IndexingScriptRewriterTestCase extends SearchDefinitionTestCase { field.addSummaryField(createStaticSummaryField(field, "test")); field.addSummaryField(createStaticSummaryField(field, "other")); field.addSummaryField(createDynamicSummaryField(field, "dyn2")); - assertIndexingScript("{ input test | tokenize normalize stem:\"SHORTEST\" | summary dyn | summary dyn2 | summary other | " + + assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | summary dyn | summary dyn2 | summary other | " + "summary test | index test; }", field); } @@ -89,7 +89,7 @@ public class IndexingScriptRewriterTestCase extends SearchDefinitionTestCase { public void requireThatOutputDefaultsToCurrentField() { assertIndexingScript("{ input test | attribute test; }", createField("test", DataType.STRING, "{ attribute; }")); - assertIndexingScript("{ input test | tokenize normalize stem:\"SHORTEST\" | index test; }", + assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | index test; }", createField("test", DataType.STRING, "{ index; }")); assertIndexingScript("{ input test | summary test; }", createField("test", DataType.STRING, "{ summary; }")); @@ -97,7 +97,7 @@ public class IndexingScriptRewriterTestCase extends SearchDefinitionTestCase { @Test public void testTokenizeComparisonDisregardsConfig() { - assertIndexingScript("{ input test | tokenize normalize stem:\"SHORTEST\" | summary test | index test; }", + assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" | summary test | index test; }", createField("test", DataType.STRING, "{ summary | tokenize | index; }")); } @@ -106,18 +106,18 @@ public class IndexingScriptRewriterTestCase extends SearchDefinitionTestCase { assertIndexing(Arrays.asList("clear_state | guard { input access | attribute access; }", "clear_state | guard { input category | split \";\" | attribute category_arr; }", "clear_state | guard { input category | tokenize | index category; }", - "clear_state | guard { input categories_src | lowercase | normalize | tokenize normalize stem:\"SHORTEST\" | index categories; }", - "clear_state | guard { input categoriesagain_src | lowercase | normalize | tokenize normalize stem:\"SHORTEST\" | index categoriesagain; }", - "clear_state | guard { input chatter | tokenize normalize stem:\"SHORTEST\" | index chatter; }", - "clear_state | guard { input description | tokenize normalize stem:\"SHORTEST\" | summary description | summary dyndesc | index description; }", - "clear_state | guard { input exactemento_src | lowercase | tokenize normalize stem:\"SHORTEST\" | index exactemento | summary exactemento; }", - "clear_state | guard { input longdesc | tokenize normalize stem:\"SHORTEST\" | summary dyndesc2 | summary dynlong | summary longdesc | summary longstat; }", + "clear_state | guard { input categories_src | lowercase | normalize | tokenize normalize stem:\"BEST\" | index categories; }", + "clear_state | guard { input categoriesagain_src | lowercase | normalize | tokenize normalize stem:\"BEST\" | index categoriesagain; }", + "clear_state | guard { input chatter | tokenize normalize stem:\"BEST\" | index chatter; }", + "clear_state | guard { input description | tokenize normalize stem:\"BEST\" | summary description | summary dyndesc | index description; }", + "clear_state | guard { input exactemento_src | lowercase | tokenize normalize stem:\"BEST\" | index exactemento | summary exactemento; }", + "clear_state | guard { input longdesc | tokenize normalize stem:\"BEST\" | summary dyndesc2 | summary dynlong | summary longdesc | summary longstat; }", "clear_state | guard { input measurement | attribute measurement | summary measurement; }", "clear_state | guard { input measurement | to_array | attribute measurement_arr; }", "clear_state | guard { input popularity | attribute popularity; }", "clear_state | guard { input popularity * input measurement | attribute popsiness; }", "clear_state | guard { input smallattribute | attribute smallattribute; }", - "clear_state | guard { input title | tokenize normalize stem:\"SHORTEST\" | summary title | index title; }", + "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }", "clear_state | guard { input title . \" \" . input category | tokenize | summary exact | index exact; }"), SearchBuilder.buildFromFile("src/test/examples/simple.sd")); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValidationTestCase.java index df323ddfe09..ad801ed50ab 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValidationTestCase.java @@ -63,8 +63,8 @@ public class IndexingValidationTestCase extends AbstractExportingTestCase { @Test public void testExtraField() throws IOException, ParseException { assertIndexing( - Arrays.asList("clear_state | guard { input my_index | tokenize normalize stem:\"SHORTEST\" | index my_index | summary my_index }", - "clear_state | guard { input my_input | tokenize normalize stem:\"SHORTEST\" | index my_extra | summary my_extra }"), + Arrays.asList("clear_state | guard { input my_index | tokenize normalize stem:\"BEST\" | index my_index | summary my_index }", + "clear_state | guard { input my_input | tokenize normalize stem:\"BEST\" | index my_extra | summary my_extra }"), SearchBuilder.buildFromFile("src/test/examples/indexing_extra.sd")); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java index 9cafe6541e4..93f87ef033b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java @@ -218,14 +218,8 @@ public class AdminTestCase { public void testContainerMetricsSnapshotInterval() { VespaModel vespaModel = getVespaModel(TESTDIR + "metricconfig"); - ContainerCluster docprocCluster = vespaModel.getContainerClusters().get("cluster.music.indexing"); - HealthMonitorConfig.Builder builder = new HealthMonitorConfig.Builder(); - docprocCluster.getConfig(builder); - HealthMonitorConfig docprocConfig = new HealthMonitorConfig(builder); - assertEquals(60, (int) docprocConfig.snapshot_interval()); - ContainerCluster qrCluster = vespaModel.getContainerClusters().get("container"); - builder = new HealthMonitorConfig.Builder(); + HealthMonitorConfig.Builder builder = new HealthMonitorConfig.Builder(); qrCluster.getConfig(builder); HealthMonitorConfig qrClusterConfig = new HealthMonitorConfig(builder); assertEquals(60, (int) qrClusterConfig.snapshot_interval()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java index 27839765930..0b9aeb9f188 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/DedicatedAdminV4Test.java @@ -53,19 +53,12 @@ public class DedicatedAdminV4Test { " <admin version='4.0'>" + " <slobroks><nodes count='2' dedicated='true'/></slobroks>" + " <logservers><nodes count='1' dedicated='true'/></logservers>" + - " <yamas systemname='vespa.routing' interval='60' />" + + " <monitoring systemname='vespa.routing' interval='60' />" + " <metrics>" + " <consumer id='slingstone'>" + " <metric id='foobar.count' display-name='foobar'/>" + " </consumer>" + " </metrics>" + - " <metric-consumers>" + - " <consumer name='yamas'>" + - " <metric name='upstreams_generated' />" + - " <metric name='upstreams_nginx_reloads' />" + - " <metric name='nginx.upstreams.down.last' output-name='nginx.upstreams.down'/>" + - " </consumer>" + - " </metric-consumers>" + " <identity>" + " <domain>mydomain</domain>" + " <service>myservice</service>" + @@ -90,13 +83,6 @@ public class DedicatedAdminV4Test { Metric metric = consumer.getMetrics().get("foobar.count"); assertNotNull(metric); assertEquals("foobar", metric.outputName); - - consumer = model.getAdmin().getLegacyUserMetricsConsumers().get(VESPA_CONSUMER_ID); - assertNotNull(consumer); - assertEquals(3, consumer.getMetrics().size()); - metric = consumer.getMetrics().get("nginx.upstreams.down.last"); - assertNotNull(metric); - assertEquals("nginx.upstreams.down", metric.outputName); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java index f9944f563c4..2456113f40d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java @@ -58,10 +58,9 @@ public class ConfigValueChangeValidatorTest { createVespaModel(createQrStartConfigSegment(true, 2096)), createVespaModel(createQrStartConfigSegment(false, 2096)) ); - assertEquals(3, changes.size()); + assertEquals(2, changes.size()); assertComponentsEquals(changes, "default/container.0", 0); assertComponentsEquals(changes, "admin/cluster-controllers/0", 1); - assertComponentsEquals(changes, "docproc/cluster.basicsearch.indexing/0", 2); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java index 339f8514f9f..43db1c65d37 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidatorTest.java @@ -48,7 +48,7 @@ public class DocumentDatabaseChangeValidatorTest { newRefeedAction("indexing-change", ValidationOverrides.empty, "Field 'f2' changed: add index aspect, indexing script: '{ input f2 | summary f2; }' -> " + - "'{ input f2 | tokenize normalize stem:\"SHORTEST\" | index f2 | summary f2; }'", Instant.now()), + "'{ input f2 | tokenize normalize stem:\"BEST\" | index f2 | summary f2; }'", Instant.now()), newRefeedAction("field-type-change", ValidationOverrides.empty, "Field 'f3' changed: data type: 'int' -> 'string'", Instant.now()))); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java index 3fe3c27a648..7a5b235737a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java @@ -66,7 +66,7 @@ public class IndexingScriptChangeValidatorTest { FIELD + " { indexing: index | summary }"). assertValidation(expectedAction("add index aspect", "{ input f1 | summary f1; }", - "{ input f1 | tokenize normalize stem:\"SHORTEST\" | index f1 | summary f1; }")); + "{ input f1 | tokenize normalize stem:\"BEST\" | index f1 | summary f1; }")); } @Test @@ -74,7 +74,7 @@ public class IndexingScriptChangeValidatorTest { new Fixture(FIELD + " { indexing: index | summary }", FIELD + " { indexing: summary }"). assertValidation(expectedAction("remove index aspect", - "{ input f1 | tokenize normalize stem:\"SHORTEST\" | index f1 | summary f1; }", + "{ input f1 | tokenize normalize stem:\"BEST\" | index f1 | summary f1; }", "{ input f1 | summary f1; }")); } @@ -82,8 +82,8 @@ public class IndexingScriptChangeValidatorTest { public void requireThatChangingStemmingRequireRefeed() throws Exception { new Fixture(FIELD + " { indexing: index }", FIELD + " { indexing: index \n stemming: none }"). - assertValidation(expectedAction("stemming: 'shortest' -> 'none'", - "{ input f1 | tokenize normalize stem:\"SHORTEST\" | index f1; }", + assertValidation(expectedAction("stemming: 'best' -> 'none'", + "{ input f1 | tokenize normalize stem:\"BEST\" | index f1; }", "{ input f1 | tokenize normalize | index f1; }")); } @@ -92,8 +92,8 @@ public class IndexingScriptChangeValidatorTest { new Fixture(FIELD + " { indexing: index }", FIELD + " { indexing: index \n normalizing: none }"). assertValidation(expectedAction("normalizing: 'ACCENT' -> 'NONE'", - "{ input f1 | tokenize normalize stem:\"SHORTEST\" | index f1; }", - "{ input f1 | tokenize stem:\"SHORTEST\" | index f1; }")); + "{ input f1 | tokenize normalize stem:\"BEST\" | index f1; }", + "{ input f1 | tokenize stem:\"BEST\" | index f1; }")); } @Test @@ -111,19 +111,19 @@ public class IndexingScriptChangeValidatorTest { FIELD + " { indexing: summary \n summary: dynamic }"). assertValidation(expectedAction("summary field 'f1' transform: 'none' -> 'dynamicteaser'", "{ input f1 | summary f1; }", - "{ input f1 | tokenize normalize stem:\"SHORTEST\" | summary f1; }")); + "{ input f1 | tokenize normalize stem:\"BEST\" | summary f1; }")); } @Test public void requireThatMultipleChangesRequireRefeed() throws Exception { new Fixture(FIELD + " { indexing: index } " + FIELD_F2 + " { indexing: index }", FIELD + " { indexing: index \n stemming: none } " + FIELD_F2 + " { indexing: index \n normalizing: none }"). - assertValidation(Arrays.asList(expectedAction("f1", "stemming: 'shortest' -> 'none'", - "{ input f1 | tokenize normalize stem:\"SHORTEST\" | index f1; }", + assertValidation(Arrays.asList(expectedAction("f1", "stemming: 'best' -> 'none'", + "{ input f1 | tokenize normalize stem:\"BEST\" | index f1; }", "{ input f1 | tokenize normalize | index f1; }"), expectedAction("f2", "normalizing: 'ACCENT' -> 'NONE'", - "{ input f2 | tokenize normalize stem:\"SHORTEST\" | index f2; }", - "{ input f2 | tokenize stem:\"SHORTEST\" | index f2; }"))); + "{ input f2 | tokenize normalize stem:\"BEST\" | index f2; }", + "{ input f2 | tokenize stem:\"BEST\" | index f2; }"))); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java index 38c7547c625..d7861a6f284 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java @@ -73,7 +73,7 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { " <configserver hostalias=\"mockhost\"/>" + " </configservers>" + " <adminserver hostalias=\"mockhost\"/>" + - " <yamas systemname=\"foo\"/>" + + " <monitoring systemname=\"foo\"/>" + "</admin>").getDocumentElement(); } @@ -101,7 +101,7 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { " <configserver hostalias=\"mockhost\"/>" + " </configservers>" + " <adminserver hostalias=\"mockhost\"/>" + - " <yamas systemname=\"foo\" interval=\"300\"/>" + + " <monitoring systemname=\"foo\" interval=\"300\"/>" + "</admin>").getDocumentElement(); } @@ -187,7 +187,7 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { Element servicesYamasIllegalInterval = XML.getDocument( "<admin version=\"2.0\">" + " <adminserver hostalias=\"mockhost\"/>" + - " <yamas interval=\"5\"/>" + + " <monitoring interval=\"5\"/>" + "</admin>").getDocumentElement(); Admin admin = buildAdmin(servicesYamasIllegalInterval); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java index 2ccf048df92..145715bdc16 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java @@ -184,7 +184,6 @@ public class DomSearchTuningBuilderTest extends DomBuilderTest { "<logstore>", "<maxfilesize>512</maxfilesize>", "<minfilesizefactor>0.3</minfilesizefactor>", - "<numthreads>7</numthreads>", "<chunk>", "<maxsize>256</maxsize>", "<compression>", @@ -204,7 +203,6 @@ public class DomSearchTuningBuilderTest extends DomBuilderTest { assertEquals(3, t.searchNode.summary.store.cache.compression.level.intValue()); assertEquals(512, t.searchNode.summary.store.logStore.maxFileSize.longValue()); assertEquals(0.3, t.searchNode.summary.store.logStore.minFileSizeFactor, DELTA); - assertEquals(7, t.searchNode.summary.store.logStore.numThreads.intValue()); assertEquals(256, t.searchNode.summary.store.logStore.chunk.maxSize.intValue()); assertEquals(Tuning.SearchNode.Summary.Store.Compression.Type.LZ4, t.searchNode.summary.store.logStore.chunk.compression.type); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilderTest.java deleted file mode 100755 index ff2f2054229..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilderTest.java +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.builder.xml.dom.chains.search; - -import com.yahoo.config.model.builder.xml.test.DomBuilderTest; -import com.yahoo.search.federation.ProviderConfig; -import com.yahoo.vespa.model.container.search.searchchain.HttpProvider; -import com.yahoo.vespa.model.container.search.searchchain.Provider; -import org.junit.Test; -import org.w3c.dom.Element; - -import java.util.HashMap; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.fail; - -/** - * @author gjoranv - */ -public class DomProviderBuilderTest extends DomBuilderTest { - - private static final Element noProxy = parse( - "<provider id='yca-provider' type='vespa' yca-application-id='my-app'>", - " <nodes>", - " <node host='sourcehost' port='12'/>", - " </nodes>", - "</provider>"); - - private static final Element defaultProxy = parse( - "<provider id='yca-provider' type='vespa' yca-application-id='my-app'>", - " <yca-proxy/>", - " <nodes>", - " <node host='sourcehost' port='12'/>", - " </nodes>", - "</provider>"); - - private static final Element proprietaryProxy = parse( - "<provider id='yca-provider' type='vespa' yca-application-id='my-app'>", - " <yca-proxy host='my-host' port='80'/>", - " <nodes>", - " <node host='sourcehost' port='12'/>", - " </nodes>", - "</provider>"); - - private static final Element illegal_proxyWithoutId= parse( - "<provider id='yca-provider' type='vespa'>", - " <yca-proxy host='my-host' port='80'/>", - " <nodes>", - " <node host='sourcehost' port='12'/>", - " </nodes>", - "</provider>"); - - private Provider provider; - - @Test - public void testCertificateConfig_noProxy() { - provider = new DomProviderBuilder(new HashMap<>()).doBuild(root.getDeployState(), root, noProxy); - - ProviderConfig.Builder providerBuilder = new ProviderConfig.Builder(); - ((HttpProvider)provider).getConfig(providerBuilder); - ProviderConfig providerConfig = new ProviderConfig(providerBuilder); - assertThat(providerConfig.yca().applicationId(), is("my-app")); - assertThat(providerConfig.yca().useProxy(), is(false)); - } - - @Test - public void testCertificatewConfig_defaultProxy() { - provider = new DomProviderBuilder(new HashMap<>()).doBuild(root.getDeployState(), root, defaultProxy); - - ProviderConfig.Builder providerBuilder = new ProviderConfig.Builder(); - ((HttpProvider)provider).getConfig(providerBuilder); - ProviderConfig providerConfig = new ProviderConfig(providerBuilder); - - assertThat(providerConfig.yca().applicationId(), is("my-app")); - assertThat(providerConfig.yca().useProxy(), is(true)); - assertThat(providerConfig.yca().host(), is("yca.host must be set explicitly")); // default from def-file - assertThat(providerConfig.yca().port(), is(3128)); // default from def-file - } - - @Test - public void testCertificateConfig_proprietaryProxy() { - provider = new DomProviderBuilder(new HashMap<>()).doBuild(root.getDeployState(), root, proprietaryProxy); - - ProviderConfig.Builder providerBuilder = new ProviderConfig.Builder(); - ((HttpProvider)provider).getConfig(providerBuilder); - ProviderConfig providerConfig = new ProviderConfig(providerBuilder); - - assertThat(providerConfig.yca().applicationId(), is("my-app")); - assertThat(providerConfig.yca().useProxy(), is(true)); - assertThat(providerConfig.yca().host(), is("my-host")); - assertThat(providerConfig.yca().port(), is(80)); - } - - @Test - public void testFail_ycaProxyWithoutId() { - try { - provider = new DomProviderBuilder(new HashMap<>()).doBuild(root.getDeployState(), root, illegal_proxyWithoutId); - fail("Expected exception upon illegal xml."); - } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), is("Provider 'yca-provider' must have a certificate application ID, since a certificate store proxy is given")); - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilderTest.java index cc139913536..ef09d8ff887 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilderTest.java @@ -43,7 +43,7 @@ public class DomSearchChainsBuilderTest extends DomBuilderTest { "<searchchains>", " <searcher id='searcher:1'/>", - " <provider id='provider:1' type='vespa' inherits='parentChain1 parentChain2' excludes='ExcludedSearcher1 ExcludedSearcher2'", + " <provider id='provider:1' inherits='parentChain1 parentChain2' excludes='ExcludedSearcher1 ExcludedSearcher2'", " cacheweight='2.3'>", " <federationoptions optional='true' timeout='2.3 s' />", " <nodes>", @@ -155,7 +155,6 @@ public class DomSearchChainsBuilderTest extends DomBuilderTest { ChainsConfig config = new ChainsConfig(builder); checkInnerSearcherIdIsNestedInSearchChainId(config, "federationSearcher", "default"); - checkInnerSearcherIdIsNestedInSearchChainId(config, "VespaSearcher", "provider"); } private void checkInnerSearcherIdIsNestedInSearchChainId(ChainsConfig config, diff --git a/config-model/src/test/java/com/yahoo/vespa/model/clients/test/SpoolerTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/clients/test/SpoolerTestCase.java deleted file mode 100644 index 7e0c31dd998..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/clients/test/SpoolerTestCase.java +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.clients.test; - -import com.yahoo.vespa.config.content.spooler.SpoolerConfig; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.test.utils.CommonVespaModelSetup; -import com.yahoo.vespaclient.config.FeederConfig; -import org.junit.Test; - -import java.util.*; - -import static org.junit.Assert.assertEquals; - -/** - * @author Thomas Gundersen - */ -public class SpoolerTestCase { - - @Test - public void testSimple() throws Exception { - VespaModel model = createModel("src/test/cfg/clients/simpleconfig.v2.docprocv3"); - - SpoolerConfig.Builder builder = new SpoolerConfig.Builder(); - SpoolerConfig.Parsers.Builder parserBuilder1 = createParserBuilder("com.yahoo.vespaspooler.XMLFileParser"); - SpoolerConfig.Parsers.Builder parserBuilder2 = createParserBuilder("com.yahoo.vespaspooler.MusicFileParser"); - LinkedHashMap<String, String> parameters = new LinkedHashMap<>(); - parameters.put("route", "default"); - parameters.put("foo", "bar"); - SpoolerConfig.Parsers.Builder parserBuilder3 = createParserBuilder("com.yahoo.vespaspooler.MusicParser", - parameters); - builder.maxfailuresize(100000). - maxfatalfailuresize(1000000). - threads(5). - parsers(Arrays.asList(parserBuilder1, parserBuilder2, parserBuilder3)); - final int spoolerIndex = 0; - testSpoolerConfigBuilder(model, spoolerIndex, builder); - - FeederConfig.Builder feederBuilder = new FeederConfig.Builder(). - abortondocumenterror(false). - maxpendingbytes(8000). - tracelevel(7); - testFeederConfigBuilder(model, spoolerIndex, feederBuilder); - } - - @Test - public void testAdvanced() throws Exception { - VespaModel model = createModel("src/test/cfg/clients/advancedconfig.v2"); - - SpoolerConfig.Builder builder = new SpoolerConfig.Builder(); - SpoolerConfig.Parsers.Builder parserBuilder1 = createParserBuilder("com.yahoo.vespaspooler.XMLFileParser"); - SpoolerConfig.Parsers.Builder parserBuilder2 = createParserBuilder("com.yahoo.vespaspooler.MusicFileParser"); - LinkedHashMap<String, String> parameters = new LinkedHashMap<>(); - parameters.put("route", "default"); - SpoolerConfig.Parsers.Builder parserBuilder3 = createParserBuilder("com.yahoo.vespaspooler.MusicParser", - parameters); - builder.keepsuccess(true). - parsers(Arrays.asList(parserBuilder1, parserBuilder2, parserBuilder3)); - int spoolerIndex = 0; - testSpoolerConfigBuilder(model, spoolerIndex, builder); - - FeederConfig.Builder feederBuilder = new FeederConfig.Builder(). - abortondocumenterror(false). - maxpendingbytes(8000). - timeout(90.0); - testFeederConfigBuilder(model, spoolerIndex, feederBuilder); - - builder = new SpoolerConfig.Builder(); - parameters = new LinkedHashMap<>(); - parameters.put("route", "othercluster"); - - parserBuilder1 = createParserBuilder("com.yahoo.vespaspooler.MusicParser", - parameters); - builder.keepsuccess(false). - parsers(parserBuilder1); - spoolerIndex = 1; - testSpoolerConfigBuilder(model, spoolerIndex, builder); - - feederBuilder = new FeederConfig.Builder(). - abortondocumenterror(false). - maxpendingbytes(4000). - timeout(50.0); - testFeederConfigBuilder(model, spoolerIndex, feederBuilder); - - builder = new SpoolerConfig.Builder(); - parserBuilder1 = new SpoolerConfig.Parsers.Builder(); - parserBuilder1.classname("com.yahoo.vespaspooler.MusicFileParser"); - builder.parsers(parserBuilder1); - String id = "plan9"; - testSpoolerConfigBuilder(model, "clients/spoolers/" + id, builder); - - feederBuilder = new FeederConfig.Builder(). - route("myroute"). - mbusport(14064). - timeout(90.0); - testFeederConfigBuilder(model, "clients/spoolers/" + id, feederBuilder); - } - - SpoolerConfig.Parsers.Builder createParserBuilder(String className) { - return createParserBuilder(className, new HashMap<String, String>()); - } - - SpoolerConfig.Parsers.Builder createParserBuilder(String className, Map<String, String> parameters) { - SpoolerConfig.Parsers.Builder builder = new SpoolerConfig.Parsers.Builder(); - builder.classname(className); - if (!parameters.isEmpty()) { - List<SpoolerConfig.Parsers.Parameters.Builder> parametersBuilders = new ArrayList<>(); - for (Map.Entry<String, String> entry : parameters.entrySet()) { - final SpoolerConfig.Parsers.Parameters.Builder parametersBuilder = new SpoolerConfig.Parsers.Parameters.Builder(); - parametersBuilder.key(entry.getKey()).value(entry.getValue()); - parametersBuilders.add(parametersBuilder); - } - builder.parameters(parametersBuilders); - } - return builder; - } - - private void testSpoolerConfigBuilder(VespaModel model, int index, SpoolerConfig.Builder expected) throws Exception { - testSpoolerConfigBuilder(model, "clients/spoolers/spooler." + index, expected); - } - - private void testSpoolerConfigBuilder(VespaModel model, String id, SpoolerConfig.Builder expected) throws Exception { - SpoolerConfig.Builder b = new SpoolerConfig.Builder(); - model.getConfig(b, id); - SpoolerConfig config = new SpoolerConfig(b); - SpoolerConfig expectedConfig = new SpoolerConfig(expected); - assertEquals(expectedConfig, config); - } - - private void testFeederConfigBuilder(VespaModel model, int index, FeederConfig.Builder expected) throws Exception { - testFeederConfigBuilder(model, "clients/spoolers/spooler." + index, expected); - } - - private void testFeederConfigBuilder(VespaModel model, String id, FeederConfig.Builder expected) throws Exception { - FeederConfig.Builder b = new FeederConfig.Builder(); - model.getConfig(b, id); - FeederConfig config = new FeederConfig(b); - FeederConfig expectedConfig = new FeederConfig(expected); - assertEquals(expectedConfig, config); - } - - private VespaModel createModel(String configFile) throws Exception { - return CommonVespaModelSetup.createVespaModelWithMusic(configFile); - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index b4dea09010d..121b1cd4f4d 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -13,7 +13,6 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.container.handler.ThreadpoolConfig; -import com.yahoo.container.jdisc.config.MetricDefaultsConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.Host; import com.yahoo.vespa.model.HostResource; @@ -29,6 +28,7 @@ import java.util.Collection; import java.util.Iterator; import java.util.Optional; +import static com.yahoo.vespa.model.container.ContainerCluster.G1GC; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -39,21 +39,6 @@ import static org.junit.Assert.assertTrue; public class ContainerClusterTest { @Test - public void requireThatDefaultMetricConsumerFactoryCanBeConfigured() { - ContainerCluster cluster = newContainerCluster(); - cluster.setDefaultMetricConsumerFactory(MetricDefaultsConfig.Factory.Enum.YAMAS_SCOREBOARD); - assertEquals(MetricDefaultsConfig.Factory.Enum.YAMAS_SCOREBOARD, - getMetricDefaultsConfig(cluster).factory()); - } - - @Test - public void requireThatDefaultMetricConsumerFactoryMatchesConfigDefault() { - ContainerCluster cluster = newContainerCluster(); - assertEquals(new MetricDefaultsConfig(new MetricDefaultsConfig.Builder()).factory(), - getMetricDefaultsConfig(cluster).factory()); - } - - @Test public void requireThatClusterInfoIsPopulated() { ContainerCluster cluster = newContainerCluster(); ClusterInfoConfig config = getClusterInfoConfig(cluster); @@ -227,7 +212,7 @@ public class ContainerClusterTest { MockRoot root = createRoot(false); ContainerCluster cluster = createClusterControllerCluster(root); addClusterController(root.deployLogger(), cluster, "host-c1"); - assertFalse(contains("com.yahoo.language.provider.SimpleLinguisticsProvider", cluster.getAllComponents())); + assertFalse(contains("com.yahoo.language.provider.DefaultLinguisticsProvider", cluster.getAllComponents())); } @Test @@ -235,7 +220,7 @@ public class ContainerClusterTest { MockRoot root = createRoot(false); ContainerCluster cluster = createContainerCluster(root, false); addClusterController(root.deployLogger(), cluster, "host-c1"); - assertTrue(contains("com.yahoo.language.provider.SimpleLinguisticsProvider", cluster.getAllComponents())); + assertTrue(contains("com.yahoo.language.provider.DefaultLinguisticsProvider", cluster.getAllComponents())); } private static boolean contains(String componentId, Collection<Component<?, ?>> componentList) { @@ -300,12 +285,6 @@ public class ContainerClusterTest { return cluster; } - private static MetricDefaultsConfig getMetricDefaultsConfig(ContainerCluster cluster) { - MetricDefaultsConfig.Builder builder = new MetricDefaultsConfig.Builder(); - cluster.getConfig(builder); - return new MetricDefaultsConfig(builder); - } - private static ClusterInfoConfig getClusterInfoConfig(ContainerCluster cluster) { ClusterInfoConfig.Builder builder = new ClusterInfoConfig.Builder(); cluster.getConfig(builder); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterBindingsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterBindingsTest.java index 643e65ffc61..4d04a1be614 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterBindingsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/http/FilterBindingsTest.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.container.http; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; -import com.yahoo.container.jdisc.config.HttpServerConfig; import com.yahoo.jdisc.http.ServerConfig; import com.yahoo.vespa.model.container.ContainerModel; import com.yahoo.vespa.model.container.component.chain.Chain; @@ -23,7 +22,6 @@ import static org.junit.Assert.assertNotNull; /** * @author gjoranv - * @since 5.1.25 */ public class FilterBindingsTest extends DomBuilderTest { @@ -85,7 +83,7 @@ public class FilterBindingsTest extends DomBuilderTest { @Test public void bindings_are_added_to_config_for_all_http_servers_with_jetty() throws Exception { final Element xml = parse( - "<jdisc version='1.0' jetty='true'>", + "<jdisc version='1.0'>", " <http>", " <filtering>", " <request-chain id='my-request-chain'>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SearchChainsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SearchChainsTest.java index 1d05a0d6d51..d81ffedef7f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SearchChainsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SearchChainsTest.java @@ -21,9 +21,11 @@ import static org.junit.Assert.*; /** * Test of search chains config * <p>TODO: examine the actual values in the configs.</p> + * * @author Tony Vaagenes */ public class SearchChainsTest extends SearchChainsTestBase { + private ChainsConfig chainsConfig; private ProviderConfig providerConfig; private ClusterConfig clusterConfig; @@ -34,10 +36,6 @@ public class SearchChainsTest extends SearchChainsTestBase { chainsBuilder = (ChainsConfig.Builder)root.getConfig(chainsBuilder, "searchchains"); chainsConfig = new ChainsConfig(chainsBuilder); - ProviderConfig.Builder providerBuilder = new ProviderConfig.Builder(); - providerBuilder = (ProviderConfig.Builder)root.getConfig(providerBuilder, "searchchains/chain/provider:1/component/com.yahoo.search.federation.vespa.VespaSearcher"); - providerConfig = new ProviderConfig(providerBuilder); - ClusterConfig.Builder clusterBuilder = new ClusterConfig.Builder(); clusterBuilder = (ClusterConfig.Builder)root.getConfig(clusterBuilder, "searchchains/chain/cluster2/component/" + ClusterSearcher.class.getName()); clusterConfig = new ClusterConfig(clusterBuilder); @@ -50,7 +48,7 @@ public class SearchChainsTest extends SearchChainsTestBase { "<searchchains>", " <searcher id='searcher:1' classId='classId1' />", - " <provider id='provider:1' type='vespa' inherits='parentChain1 parentChain2' excludes='ExcludedSearcher1 ExcludedSearcher2'", + " <provider id='provider:1' inherits='parentChain1 parentChain2' excludes='ExcludedSearcher1 ExcludedSearcher2'", " cacheweight='2.3'>", " <federationoptions optional='true' timeout='2.3 s' />", " <nodes>", @@ -67,7 +65,7 @@ public class SearchChainsTest extends SearchChainsTestBase { " <provider id='provider:2' type='local' cluster='cluster1' />", " <provider id='provider:3' />", - " <provider id='vespa-provider' type='vespa' >", + " <provider id='vespa-provider'>", " <nodes>", " <node host='localhost' port='" + Defaults.getDefaults().vespaWebServicePort() + "' />", " </nodes>", @@ -96,32 +94,15 @@ public class SearchChainsTest extends SearchChainsTestBase { } @Test - public void require_user_config_for_vespa_searcher_works() { - assertEquals(root.getConfig(ProviderConfig.class, "searchchains/chain/vespa-provider/component/com.yahoo.search.federation.vespa.VespaSearcher"). - queryType(), ProviderConfig.QueryType.PROGRAMMATIC); - } - - @Test public void require_that_source_chain_spec_id_is_namespaced_in_provider_id() { Source source = (Source) getSearchChains().allChains().getComponent("source:1@provider:1"); assertThat(source.getChainSpecification().componentId.getNamespace(), is(ComponentId.fromString("provider:1"))); } @Test - public void validateHttpProviderConfig() { - assertNotNull(providerConfig); - } - - @Test public void validateLocalProviderConfig() { assertEquals(2, clusterConfig.clusterId()); assertEquals("cluster2", clusterConfig.clusterName()); } - public static boolean verifyChainExists(List<ChainsConfig.Chains> chains, String componentId) { - for (ChainsConfig.Chains c : chains) { - if (c.id().equals(componentId)) return true; - } - return false; - } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java index 52029d4c434..fd1ea2ae7ab 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java @@ -35,13 +35,7 @@ public class AccessControlTest extends ContainerModelBuilderTestBase { private static final Set<String> REQUIRED_HANDLER_BINDINGS = ImmutableSet.of( "/custom-handler/", "/search/", - "/feed/", - "/remove/", - "/removelocation/", - "/get/", - "/visit/", "/document/", - "/feedstatus/", ContainerCluster.RESERVED_URI_PREFIX); private static final Set<String> FORBIDDEN_HANDLER_BINDINGS = ImmutableSet.of( diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java index a4a8eef74dd..d545723241e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java @@ -39,8 +39,10 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { createModel(root, cluster1Elem, cluster2Elem); - assertNotNull(getVespaAccessLog("cluster1")); - assertNull( getVespaAccessLog("cluster2")); + assertNotNull(getJsonAccessLog("cluster1")); + assertNull( getJsonAccessLog("cluster2")); + assertNull(getVespaAccessLog("cluster1")); + assertNull(getVespaAccessLog("cluster2")); } @Test @@ -55,27 +57,32 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { createModel(root, clusterElem); assertNull(getVespaAccessLog(jdiscClusterId)); + assertNull(getJsonAccessLog(jdiscClusterId)); } private Component<?, ?> getVespaAccessLog(String clusterName) { ContainerCluster cluster = (ContainerCluster) root.getChildren().get(clusterName); return cluster.getComponentsMap().get(ComponentId.fromString((VespaAccessLog.class.getName()))); } + private Component<?, ?> getJsonAccessLog(String clusterName) { + ContainerCluster cluster = (ContainerCluster) root.getChildren().get(clusterName); + return cluster.getComponentsMap().get(ComponentId.fromString((JSONAccessLog.class.getName()))); + } @Test public void access_log_can_be_configured() throws Exception { Element clusterElem = DomBuilderTest.parse( "<jdisc id='default' version='1.0'>", " <accesslog type='yapache' ", - " fileNamePattern='pattern' rotationInterval='interval'", - " rotationScheme='date' />", + " fileNamePattern='pattern' rotationInterval='interval' />", " <accesslog type='json' ", - " fileNamePattern='pattern' rotationInterval='interval'", - " rotationScheme='date' />", + " fileNamePattern='pattern' rotationInterval='interval' />", nodesXml, "</jdisc>" ); createModel(root, clusterElem); + assertNotNull(getJsonAccessLog("default")); + assertNull(getVespaAccessLog("default")); { // yapache Component<?, ?> accessLogComponent = getContainerComponent("default", YApacheAccessLog.class.getName()); @@ -85,7 +92,6 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { AccessLogConfig.FileHandler fileHandlerConfig = config.fileHandler(); assertEquals("pattern", fileHandlerConfig.pattern()); assertEquals("interval", fileHandlerConfig.rotation()); - assertEquals(AccessLogConfig.FileHandler.RotateScheme.DATE, fileHandlerConfig.rotateScheme()); } { // json @@ -96,7 +102,6 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { AccessLogConfig.FileHandler fileHandlerConfig = config.fileHandler(); assertEquals("pattern", fileHandlerConfig.pattern()); assertEquals("interval", fileHandlerConfig.rotation()); - assertEquals(AccessLogConfig.FileHandler.RotateScheme.DATE, fileHandlerConfig.rotateScheme()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java index 49a989a9474..428286c4794 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerDocumentApiBuilderTest.java @@ -21,8 +21,7 @@ import static org.hamcrest.Matchers.hasItem; import static org.junit.Assert.assertThat; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.11 + * @author Einar M R Rosenvinge */ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBase { @@ -38,7 +37,7 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa } @Test - public void document_api_config_is_added_to_container_cluster() throws Exception { + public void document_api_config_is_added_to_container_cluster() { Element elem = DomBuilderTest.parse( "<jdisc id='cluster1' version='1.0'>", " <document-api>", @@ -61,7 +60,7 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa } @Test - public void custom_bindings_are_allowed() throws Exception { + public void custom_bindings_are_allowed() { Element elem = DomBuilderTest.parse( "<jdisc id='cluster1' version='1.0'>", " <document-api>", @@ -72,13 +71,6 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa "</jdisc>"); createModel(root, elem); - verifyCustomBindings("com.yahoo.feedhandler.VespaFeedHandler", "feed"); - verifyCustomBindings("com.yahoo.feedhandler.VespaFeedHandlerCompatibility", "document"); - verifyCustomBindings("com.yahoo.feedhandler.VespaFeedHandlerGet", "get"); - verifyCustomBindings("com.yahoo.feedhandler.VespaFeedHandlerRemove", "remove"); - verifyCustomBindings("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation", "removelocation"); - verifyCustomBindings("com.yahoo.feedhandler.VespaFeedHandlerStatus", "feedstatus"); - verifyCustomBindings("com.yahoo.feedhandler.VespaFeedHandlerVisit", "visit"); verifyCustomBindings("com.yahoo.vespa.http.server.FeedHandler", ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"); } @@ -94,7 +86,7 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa } @Test - public void requireThatHandlersAreSetup() throws Exception { + public void requireThatHandlersAreSetup() { Element elem = DomBuilderTest.parse( "<jdisc id='cluster1' version='1.0'>", " <document-api />", @@ -108,60 +100,6 @@ public class ContainerDocumentApiBuilderTest extends ContainerModelBuilderTestBa assertThat(handlerMap.get("com.yahoo.container.handler.observability.ApplicationStatusHandler"), not(nullValue())); assertThat(handlerMap.get("com.yahoo.container.jdisc.state.StateHandler"), not(nullValue())); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandler"), not(nullValue())); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandler").getServerBindings().contains("http://*/feed"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandler").getServerBindings().contains("https://*/feed"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandler").getServerBindings().contains("http://*/feed/"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandler").getServerBindings().contains("https://*/feed/"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandler").getServerBindings().size(), equalTo(4)); - - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerCompatibility").getComponentId().toString(), not(nullValue())); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerCompatibility").getServerBindings().contains("http://*/document"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerCompatibility").getServerBindings().contains("https://*/document"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerCompatibility").getServerBindings().contains("http://*/document/"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerCompatibility").getServerBindings().contains("https://*/document/"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerCompatibility").getServerBindings().size(), equalTo(4)); - - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerGet"), not(nullValue())); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerGet").getServerBindings().contains("http://*/get"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerGet").getServerBindings().contains("https://*/get"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerGet").getServerBindings().contains("http://*/get/"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerGet").getServerBindings().contains("https://*/get/"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerGet").getServerBindings().size(), equalTo(4)); - - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemove"), not(nullValue())); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemove").getServerBindings().contains("http://*/remove"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemove").getServerBindings().contains("https://*/remove"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemove").getServerBindings().contains("http://*/remove/"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemove").getServerBindings().contains("https://*/remove/"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemove").getServerBindings().size(), equalTo(4)); - - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation"), not(nullValue())); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation").getServerBindings().contains("http://*/removelocation"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation").getServerBindings().contains("https://*/removelocation"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation").getServerBindings().contains("http://*/removelocation/"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation").getServerBindings().contains("https://*/removelocation/"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation").getServerBindings().size(), equalTo(4)); - - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerStatus"), not(nullValue())); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerStatus").getServerBindings().contains("http://*/feedstatus"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerStatus").getServerBindings().contains("https://*/feedstatus"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerStatus").getServerBindings().contains("http://*/feedstatus/"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerStatus").getServerBindings().contains("https://*/feedstatus/"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerStatus").getServerBindings().size(), equalTo(4)); - - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerVisit"), not(nullValue())); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerVisit").getServerBindings().contains("http://*/visit"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerVisit").getServerBindings().contains("https://*/visit"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerVisit").getServerBindings().contains("http://*/visit/"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerVisit").getServerBindings().contains("https://*/visit/"), is(true)); - assertThat(handlerMap.get("com.yahoo.feedhandler.VespaFeedHandlerVisit").getServerBindings().size(), equalTo(4)); - - assertThat(handlerMap.get("com.yahoo.search.handler.SearchHandler"), not(nullValue())); - assertThat(handlerMap.get("com.yahoo.search.handler.SearchHandler").getServerBindings().contains("http://*/search/*"), is(true)); - assertThat(handlerMap.get("com.yahoo.search.handler.SearchHandler").getServerBindings().contains("https://*/search/*"), is(true)); - assertThat(handlerMap.get("com.yahoo.search.handler.SearchHandler").getServerBindings().size(), equalTo(2)); - assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler"), not(nullValue())); assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().contains("http://*/" + ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"), is(true)); assertThat(handlerMap.get("com.yahoo.vespa.http.server.FeedHandler").getServerBindings().contains("https://*/" + ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"), is(true)); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 0030392f318..ee829607da1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -94,7 +94,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } private static void verifyIgnoreJvmGCOptions(boolean isHosted) throws IOException, SAXException { - verifyIgnoreJvmGCOptionsIfJvmArgs(isHosted, "jvmargs", ContainerCluster.CMS); + verifyIgnoreJvmGCOptionsIfJvmArgs(isHosted, "jvmargs", ContainerCluster.G1GC); verifyIgnoreJvmGCOptionsIfJvmArgs(isHosted, "jvm-options", "-XX:+UseG1GC"); } @@ -162,7 +162,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test public void requireThatJvmGCOptionsIsHonoured() throws IOException, SAXException { - verifyJvmGCOptions(false, Zone.defaultZone(),ContainerCluster.CMS); + verifyJvmGCOptions(false, Zone.defaultZone(),ContainerCluster.G1GC); verifyJvmGCOptions(true, Zone.defaultZone(), ContainerCluster.G1GC); } @@ -483,21 +483,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test - public void legacy_yca_filter_and_its_config_provider_are_included_in_components_config() { - Element clusterElem = DomBuilderTest.parse( - "<jdisc id='default' version='1.0'>", - " <filter id='YcaFilter' /> ", - "</jdisc>"); - - createModel(root, clusterElem); - assertThat(componentsConfig().toString(), containsString(".id \"YcaFilter\"")); - - String providerId = HttpFilter.configProviderId(ComponentId.fromString("YcaFilter")).stringValue(); - assertThat(componentsConfig().toString(), containsString(".id \"" + providerId + "\"")); - } - - @Test - public void nested_components_are_injected_to_handlers() { + public void nested_components_are_injected_to_handlers() throws Exception { Element clusterElem = DomBuilderTest.parse( "<jdisc id='default' version='1.0'>", " <handler id='myHandler'>", @@ -638,7 +624,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { @Test(expected = IllegalArgumentException.class) public void renderers_named_DefaultRenderer_are_not_allowed() { - createModel(root, generateContainerElementWithRenderer("DefaultRenderer")); + createModel(root, generateContainerElementWithRenderer("XmlRenderer")); } @Test @@ -699,7 +685,6 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { assertTrue(config.rpc().enabled()); assertEquals("", config.rpc().host()); assertFalse(config.restartOnDeploy()); - assertFalse(config.coveragereports()); assertEquals("filedistribution/" + hostname, config.filedistributor().configid()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java index 5acfe9312f6..e0dcd3fcc47 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java @@ -217,7 +217,7 @@ public class DocprocBuilderTest extends DomBuilderTest { QrStartConfig.Jvm jvm = qrStartConfig.jvm(); assertThat(jvm.server(), is(true)); assertThat(jvm.verbosegc(), is(true)); - assertThat(jvm.gcopts(), is("-XX:+UseConcMarkSweepGC -XX:MaxTenuringThreshold=15 -XX:NewRatio=1")); + assertThat(jvm.gcopts(), is("-XX:+UseG1GC -XX:MaxTenuringThreshold=15")); assertThat(jvm.heapsize(), is(1536)); assertThat(jvm.stacksize(), is(512)); assertThat(qrStartConfig.ulimitv(), is("")); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java index ceff9f3d4bb..5525b75373d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java @@ -16,7 +16,6 @@ import org.w3c.dom.Element; import java.util.List; import java.util.Optional; -import static com.yahoo.jdisc.http.ConnectorConfig.Ssl.KeyStoreType; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; @@ -33,7 +32,7 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas @Test public void verify_that_overriding_connector_options_works() throws Exception { Element clusterElem = DomBuilderTest.parse( - "<jdisc id='default' version='1.0' jetty='true'>\n" + + "<jdisc id='default' version='1.0'>\n" + " <http>\n" + " <server id='bananarama' port='4321'>\n" + " <config name='jdisc.http.connector'>\n" + @@ -46,7 +45,6 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas "</jdisc>\n" ); createModel(root, clusterElem); - ConnectorConfig.Builder connectorConfigBuilder = new ConnectorConfig.Builder(); ConnectorConfig cfg = root.getConfig(ConnectorConfig.class, "default/http/jdisc-jetty/bananarama"); assertThat(cfg.requestHeaderSize(), is(300000)); assertThat(cfg.headerCacheSize(), is(300000)); @@ -55,7 +53,7 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas @Test public void verify_that_enabling_jetty_works() throws Exception { Element clusterElem = DomBuilderTest.parse( - "<jdisc id='default' version='1.0' jetty='true'>" + + "<jdisc id='default' version='1.0'>" + nodesXml + "</jdisc>" ); @@ -66,7 +64,7 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas @Test public void verify_that_enabling_jetty_works_for_custom_http_servers() throws Exception { Element clusterElem = DomBuilderTest.parse( - "<jdisc id='default' version='1.0' jetty='true'>", + "<jdisc id='default' version='1.0'>", " <http>", " <server port='9000' id='foo' />", " </http>", @@ -79,7 +77,7 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas @Test public void verifyThatJettyHttpServerHasFilterBindingsProvider() throws Exception { final Element clusterElem = DomBuilderTest.parse( - "<jdisc id='default' version='1.0' jetty='true'>", + "<jdisc id='default' version='1.0'>", nodesXml, "</jdisc>" ); createModel(root, clusterElem); @@ -100,7 +98,7 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas @Test public void verifyThatJettyHttpServerHasFilterBindingsProviderForCustomHttpServers() throws Exception { final Element clusterElem = DomBuilderTest.parse( - "<jdisc id='default' version='1.0' jetty='true'>", + "<jdisc id='default' version='1.0'>", " <http>", " <server port='9000' id='foo' />", " </http>", @@ -122,73 +120,7 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas } @Test - public void verify_that_old_http_config_override_inside_server_tag_works() throws Exception { - Element clusterElem = DomBuilderTest.parse( - "<jdisc id='default' version='1.0' jetty='true'>", - " <http>", - " <server port='9000' id='foo'>", - " <config name=\"container.jdisc.config.http-server\">", - " <tcpKeepAliveEnabled>true</tcpKeepAliveEnabled>", - " <tcpNoDelayEnabled>false</tcpNoDelayEnabled>", - " <tcpListenBacklogLength>2</tcpListenBacklogLength>", - " <idleConnectionTimeout>34.1</idleConnectionTimeout>", - " <soLinger>42.2</soLinger>", - " <sendBufferSize>1234</sendBufferSize>", - " <maxHeaderSize>4321</maxHeaderSize>", - " <ssl>", - " <enabled>true</enabled>", - " <keyStoreType>JKS</keyStoreType>", - " <keyStorePath>apple</keyStorePath>", - " <trustStorePath>grape</trustStorePath>", - " <keyDBKey>tomato</keyDBKey>", - " <algorithm>onion</algorithm>", - " <protocol>carrot</protocol>", - " </ssl>", - " </config>", - " </server>", - " </http>", - nodesXml, - "</jdisc>" ); - createModel(root, clusterElem); - ContainerCluster cluster = (ContainerCluster) root.getChildren().get("default"); - List<JettyHttpServer> jettyServers = cluster.getChildrenByTypeRecursive(JettyHttpServer.class); - - assertThat(jettyServers.size(), is(1)); - - JettyHttpServer server = jettyServers.get(0); - assertThat(server.model.bundleInstantiationSpec.classId.toString(), - is(com.yahoo.jdisc.http.server.jetty.JettyHttpServer.class.getName())); - assertThat(server.model.bundleInstantiationSpec.bundle.toString(), is("jdisc_http_service")); - assertThat(server.getConnectorFactories().size(), is(1)); - - ConnectorConfig.Builder connectorConfigBuilder = new ConnectorConfig.Builder(); - server.getConnectorFactories().get(0).getConfig(connectorConfigBuilder); - ConnectorConfig connector = new ConnectorConfig(connectorConfigBuilder); - assertThat(connector.name(), equalTo("foo")); - assertThat(connector.tcpKeepAliveEnabled(), equalTo(true)); - assertThat(connector.tcpNoDelay(), equalTo(false)); - assertThat(connector.acceptQueueSize(), equalTo(2)); - assertThat(connector.idleTimeout(), equalTo(34.1)); - assertThat(connector.soLingerTime(), equalTo(42.2)); - assertThat(connector.outputBufferSize(), equalTo(1234)); - assertThat(connector.headerCacheSize(), equalTo(4321)); - assertThat(connector.ssl().enabled(), equalTo(true)); - assertThat(connector.ssl().keyStoreType(), equalTo(KeyStoreType.Enum.JKS)); - assertThat(connector.ssl().keyStorePath(), equalTo("apple")); - assertThat(connector.ssl().trustStorePath(), equalTo("grape")); - assertThat(connector.ssl().keyDbKey(), equalTo("tomato")); - assertThat(connector.ssl().sslKeyManagerFactoryAlgorithm(), equalTo("onion")); - assertThat(connector.ssl().protocol(), equalTo("carrot")); - - assertThat( - extractComponentByClassName( - clusterComponentsConfig(), - com.yahoo.jdisc.http.server.jetty.JettyHttpServer.class.getName()), - is(not(nullValue()))); - } - - @Test - public void verify_that_ssl_element_generates_connector_config_and_inject_provider_component() { + public void ssl_element_generates_connector_config_and_injects_provider_component() { Element clusterElem = DomBuilderTest.parse( "<jdisc id='default' version='1.0' jetty='true'>", " <http>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index a003e0122a5..057ef0efbd6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -676,18 +676,6 @@ public class ContentClusterTest extends ContentBaseTest { "<admin version=\"2.0\">" + " <logserver hostalias=\"node0\"/>" + " <adminserver hostalias=\"node0\"/>" + - " <metric-consumers>" + - " <consumer name=\"foobar\">" + - " <metric name=\"storage.foo.bar\"/>" + - " </consumer>" + - " <consumer name=\"log\">" + - " <metric name=\"extralogmetric\"/>" + - " <metric name=\"extralogmetric3\"/>" + - " </consumer>" + - " <consumer name=\"fleetcontroller\">" + - " <metric name=\"extraextra\"/>" + - " </consumer>" + - " </metric-consumers>" + "</admin>" + "</services>"; @@ -700,11 +688,8 @@ public class ContentClusterTest extends ContentBaseTest { model.getConfig(builder, "storage/storage/0"); MetricsmanagerConfig config = new MetricsmanagerConfig(builder); - assertEquals("[storage.foo.bar]", getConsumer("foobar", config).addedmetrics().toString()); String expected = - "[extralogmetric\n" + - "extralogmetric3\n" + - "vds.filestor.alldisks.allthreads.put.sum\n" + + "[vds.filestor.alldisks.allthreads.put.sum\n" + "vds.filestor.alldisks.allthreads.get.sum\n" + "vds.filestor.alldisks.allthreads.remove.sum\n" + "vds.filestor.alldisks.allthreads.update.sum\n" + @@ -720,8 +705,7 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(expected, actual); assertEquals("[logdefault]", getConsumer("log", config).tags().toString()); expected = - "[extraextra\n" + - "vds.datastored.alldisks.docs\n" + + "[vds.datastored.alldisks.docs\n" + "vds.datastored.alldisks.bytes\n" + "vds.datastored.alldisks.buckets]"; actual = getConsumer("fleetcontroller", config).addedmetrics().toString().replaceAll(", ", "\n"); @@ -733,10 +717,7 @@ public class ContentClusterTest extends ContentBaseTest { model.getConfig(builder, "storage/distributor/0"); MetricsmanagerConfig config = new MetricsmanagerConfig(builder); - assertEquals("[storage.foo.bar]", getConsumer("foobar", config).addedmetrics().toString()); - assertEquals("[extralogmetric, extralogmetric3, vds.distributor.docsstored, vds.distributor.bytesstored, vds.idealstate.delete_bucket.done_ok, vds.idealstate.merge_bucket.done_ok, vds.idealstate.split_bucket.done_ok, vds.idealstate.join_bucket.done_ok, vds.idealstate.buckets_rechecking]", getConsumer("log", config).addedmetrics().toString()); assertEquals("[logdefault]", getConsumer("log", config).tags().toString()); - assertEquals("[extraextra]", getConsumer("fleetcontroller", config).addedmetrics().toString()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java index 731544bb30e..11abe13712a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java @@ -151,8 +151,8 @@ public class IndexedTest extends ContentBaseTest { DocumentProtocol protocol = (DocumentProtocol) routing.getProtocols().get(0); RoutingTableSpec spec = protocol.getRoutingTableSpec(); assertEquals(2, spec.getNumHops()); - assertEquals("docproc/cluster.test.indexing/chain.indexing", spec.getHop(0).getName()); - assertEquals("indexing", spec.getHop(1).getName()); + assertEquals("indexing", spec.getHop(0).getName()); + assertEquals("jdisc/chain.indexing", spec.getHop(1).getName()); RouteSpec r; r = spec.getRoute(0); @@ -174,7 +174,7 @@ public class IndexedTest extends ContentBaseTest { r = spec.getRoute(4); assertEquals("test-index", r.getName()); assertEquals(2, r.getNumHops()); - assertEquals("docproc/cluster.test.indexing/chain.indexing", r.getHop(0)); + assertEquals("jdisc/chain.indexing", r.getHop(0)); assertEquals("[Content:cluster=test]", r.getHop(1)); } @Test @@ -281,7 +281,7 @@ public class IndexedTest extends ContentBaseTest { @Test public void requireThatIndexingDocprocGetsConfigIdBasedOnDistributionKey() { VespaModel model = getIndexedVespaModel(); - ContainerCluster cluster = model.getContainerClusters().get("cluster.test.indexing"); - assertEquals("docproc/cluster.test.indexing/3", cluster.getContainers().get(0).getConfigId()); + ContainerCluster cluster = model.getContainerClusters().get("jdisc"); + assertEquals("jdisc/container.0", cluster.getContainers().get(0).getConfigId()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java index 9cdb2606241..76a4a83c4fc 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java @@ -42,8 +42,8 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { SearchClusterSpec searchCluster = new SearchClusterSpec(CLUSTERNAME, null, null); searchCluster.searchDefs.add(new SearchDefSpec("music", "artist", "album")); VespaModel model = getIndexedContentVespaModel(Collections.<DocprocClusterSpec>emptyList(), Arrays.asList(searchCluster)); - assertIndexing(model, new DocprocClusterSpec(CLUSTERNAME + ".indexing", new DocprocChainSpec("docproc/cluster." + CLUSTERNAME + ".indexing/chain.indexing"))); - assertFeedingRoute(model, CLUSTERNAME, "docproc/cluster." + CLUSTERNAME + ".indexing/chain.indexing"); + assertIndexing(model, new DocprocClusterSpec("jdisc", new DocprocChainSpec("jdisc/chain.indexing"))); + assertFeedingRoute(model, CLUSTERNAME, "jdisc/chain.indexing"); } @Test @@ -54,8 +54,8 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { searchCluster.searchDefs.add(new SearchDefSpec("music", "artist", "album")); searchCluster.searchDefs.add(new SearchDefSpec("book", "author", "title")); VespaModel model = getIndexedContentVespaModel(Collections.<DocprocClusterSpec>emptyList(), Arrays.asList(searchCluster)); - assertIndexing(model, new DocprocClusterSpec(CLUSTERNAME + ".indexing", new DocprocChainSpec("docproc/cluster." + CLUSTERNAME + ".indexing/chain.indexing"))); - assertFeedingRoute(model, CLUSTERNAME, "docproc/cluster." + CLUSTERNAME + ".indexing/chain.indexing"); + assertIndexing(model, new DocprocClusterSpec("jdisc", new DocprocChainSpec("jdisc/chain.indexing"))); + assertFeedingRoute(model, CLUSTERNAME, "jdisc/chain.indexing"); } @Test @@ -72,11 +72,10 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { VespaModel model = getIndexedContentVespaModel(Collections.<DocprocClusterSpec>emptyList(), Arrays.asList(musicCluster, booksCluster)); assertIndexing(model, - new DocprocClusterSpec(MUSIC + ".indexing", new DocprocChainSpec("docproc/cluster." + MUSIC + ".indexing/chain.indexing")), - new DocprocClusterSpec(BOOKS + ".indexing", new DocprocChainSpec("docproc/cluster." + BOOKS + ".indexing/chain.indexing"))); + new DocprocClusterSpec("jdisc", new DocprocChainSpec("jdisc/chain.indexing"))); - assertFeedingRoute(model, MUSIC, "docproc/cluster." + MUSIC + ".indexing/chain.indexing"); - assertFeedingRoute(model, BOOKS, "docproc/cluster." + BOOKS + ".indexing/chain.indexing"); + assertFeedingRoute(model, MUSIC, "jdisc/chain.indexing"); + assertFeedingRoute(model, BOOKS, "jdisc/chain.indexing"); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/MonitoringConfigSnoopTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/MonitoringConfigSnoopTest.java index 6e0ab655128..dea86616f06 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/MonitoringConfigSnoopTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/MonitoringConfigSnoopTest.java @@ -26,7 +26,7 @@ public class MonitoringConfigSnoopTest { return "" + "<admin version='2.0'>" + " <adminserver hostalias='mockhost' />" - + " <yamas interval='" + interval + "' systemname='test' />" + + " <monitoring interval='" + interval + "' systemname='test' />" + "</admin>"; } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java index fd97bf1f7b2..62d2bc51830 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java @@ -59,8 +59,7 @@ public class ContentClusterUtils { public static ContentCluster createCluster(String clusterXml, MockRoot root) { Document doc = XML.getDocument(clusterXml); - Admin admin = new Admin(root, new DefaultMonitoring("vespa", 60), new Metrics(), - Collections.emptyMap(), false, + Admin admin = new Admin(root, new DefaultMonitoring("vespa", 60), new Metrics(), false, new FileDistributionConfigProducer(root, new MockFileRegistry(), null), root.getDeployState().isHosted()); ConfigModelContext context = ConfigModelContext.create(null, root.getDeployState(), diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/TldTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/TldTest.java index c3cbc58860b..4d766b776b3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/TldTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/TldTest.java @@ -59,17 +59,17 @@ public class TldTest { assertEquals(1, config.dataset(0).engine().size()); } - @Test - public void requireThatUseLocalPolicyIsOk() { - ApplicationPackage app = new MockApplicationPackage.Builder() + @Test + public void requireThatUseLocalPolicyIsOk() { + ApplicationPackage app = new MockApplicationPackage.Builder() .withHosts( - "<hosts>" + - "<host name='search.node1'><alias>search1</alias></host>" + - "<host name='search.node2'><alias>search2</alias></host>" + - "<host name='jdisc.host.other'><alias>gateway</alias></host>" + - "</hosts>") + "<hosts>" + + " <host name='search.node1'><alias>search1</alias></host>" + + " <host name='search.node2'><alias>search2</alias></host>" + + " <host name='jdisc.host.other'><alias>gateway</alias></host>" + + "</hosts>") .withServices( - "<services>" + + "<services>" + " <admin version='2.0'>" + " <adminserver hostalias='gateway' />" + " </admin>" + @@ -110,43 +110,37 @@ public class TldTest { .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) .build(); - PartitionsConfig.Builder builder = new PartitionsConfig.Builder(); - new TestDriver(true).buildModel(app).getConfig(builder, "foo/search/cluster.foo/tlds/gw.0.tld.0"); - PartitionsConfig config = new PartitionsConfig(builder); - - assertEquals(1, config.dataset().size()); - //gateway TLD with no local search node gets all search nodes - assertEquals(2, config.dataset(0).engine().size()); + PartitionsConfig.Builder builder = new PartitionsConfig.Builder(); + new TestDriver(true).buildModel(app).getConfig(builder, "foo/search/cluster.foo/tlds/gw.0.tld.0"); + PartitionsConfig config = new PartitionsConfig(builder); - assertEquals("rowid not equal 0",0,config.dataset(0).engine(0).rowid()); //Load Balance row 0 - assertEquals("partid not equal 0",0,config.dataset(0).engine(0).partid()); - assertTrue("Not configured with correct search node",config.dataset(0).engine(0).name_and_port().contains("search.node1")); + // No tld if no search + assertEquals(0, config.dataset().size()); - assertEquals("rowid not equal to 1",1,config.dataset(0).engine(1).rowid()); //Load Balance row 1 - assertEquals("partid no equal to 0",0,config.dataset(0).engine(1).partid()); - assertTrue("Not configured with correct search node",config.dataset(0).engine(1).name_and_port().contains("search.node2")); + // First container with a local search node + builder = new PartitionsConfig.Builder(); + new TestDriver(true).buildModel(app).getConfig(builder, "foo/search/cluster.foo/tlds/default.0.tld.0"); + config = new PartitionsConfig(builder); - //First container with a local search node - builder = new PartitionsConfig.Builder(); - new TestDriver(true).buildModel(app).getConfig(builder, "foo/search/cluster.foo/tlds/default.0.tld.0"); - config = new PartitionsConfig(builder); + assertEquals(1, config.dataset().size()); + assertEquals(1, config.dataset(0).engine().size()); + assertEquals(0,config.dataset(0).engine(0).rowid()); + assertEquals(0,config.dataset(0).engine(0).partid()); + assertTrue("Configured with local search node as engine", + config.dataset(0).engine(0).name_and_port().contains("search.node1")); - assertEquals(1, config.dataset().size()); - assertEquals(1, config.dataset(0).engine().size()); - assertEquals(0,config.dataset(0).engine(0).rowid()); - assertEquals(0,config.dataset(0).engine(0).partid()); - assertTrue("Not configured with local search node as engine",config.dataset(0).engine(0).name_and_port().contains("search.node1")); + // Second container with a local search node + builder = new PartitionsConfig.Builder(); + new TestDriver(true).buildModel(app).getConfig(builder, "foo/search/cluster.foo/tlds/default.1.tld.1"); + config = new PartitionsConfig(builder); - //Second container with a local search node - builder = new PartitionsConfig.Builder(); - new TestDriver(true).buildModel(app).getConfig(builder, "foo/search/cluster.foo/tlds/default.1.tld.1"); - config = new PartitionsConfig(builder); + assertEquals(1, config.dataset().size()); + assertEquals(1, config.dataset(0).engine().size()); - assertEquals(1, config.dataset().size()); - assertEquals(1, config.dataset(0).engine().size()); - assertEquals(0,config.dataset(0).engine(0).rowid()); - assertEquals(0,config.dataset(0).engine(0).partid()); - assertTrue("Not configured with local search node as engine",config.dataset(0).engine(0).name_and_port().contains("search.node2")); + assertEquals("rowid equals 0",0, config.dataset(0).engine(0).rowid()); // Load Balance row 0 + assertEquals("partid equals 0",0, config.dataset(0).engine(0).partid()); + assertTrue("Configured with correct search node", + config.dataset(0).engine(0).name_and_port().contains("search.node2")); + } - } } diff --git a/config-model/src/test/schema-test-files/deployment.xml b/config-model/src/test/schema-test-files/deployment.xml index 15f8c969f04..ea65dbec661 100644 --- a/config-model/src/test/schema-test-files/deployment.xml +++ b/config-model/src/test/schema-test-files/deployment.xml @@ -5,7 +5,6 @@ <staging/> <block-change revision='true' version='false' days="mon,tue" hours="14,15"/> <block-change days="mon,tue" hours="14,15" time-zone="CET"/> - <block-upgrade days="wed" hours="16" time-zone="CET"/><!-- Tests legacy name. Remove in Vespa 7 --> <prod global-service-id='qrs' athenz-service='other-service'> <region active='true'>us-west-1</region> <delay hours='3'/> diff --git a/config-model/src/test/schema-test-files/services-hosted-infrastructure.xml b/config-model/src/test/schema-test-files/services-hosted-infrastructure.xml index 3ff6758880b..7a77a10a381 100644 --- a/config-model/src/test/schema-test-files/services-hosted-infrastructure.xml +++ b/config-model/src/test/schema-test-files/services-hosted-infrastructure.xml @@ -6,16 +6,16 @@ <slobroks><nodes count="3" flavor="small"/></slobroks> <logservers><nodes count="1" dedicated="true"/></logservers> <monitoring systemname="vespa.routing" interval="60" /> - <metric-consumers> - <consumer name="yamas"> - <metric name="upstreams_generated" /> - <metric name="upstreams_nginx_reloads" /> - <metric name="upstreams_nginx_reloads_succeeded" /> - <metric name="upstreams_configuration_age.last" /> - <metric name="nginx.upstreams.up.last" output-name="nginx.upstreams.up"/> - <metric name="nginx.upstreams.down.last" output-name="nginx.upstreams.down"/> + <metrics> + <consumer id="yamas"> + <metric id="upstreams_generated" /> + <metric id="upstreams_nginx_reloads" /> + <metric id="upstreams_nginx_reloads_succeeded" /> + <metric id="upstreams_configuration_age.last" /> + <metric id="nginx.upstreams.up.last" display-name="nginx.upstreams.up"/> + <metric id="nginx.upstreams.down.last" display-name="nginx.upstreams.down"/> </consumer> - </metric-consumers> + </metrics> </admin> <jdisc id="container" version="1.0"> diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 43a8d31e2ee..880d49fa460 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -113,11 +113,7 @@ </filtering> <server port="4080" id="myServer"/> - <server port="4081" id="anotherServer"> - <config name="container.jdisc.config.http-server"> - <maxChunkSize>9999</maxChunkSize> - </config> - </server> + <server port="4081" id="anotherServer"/> <server port="4082" id="defaultSsl"> <ssl> <private-key-file>/foo/key</private-key-file> diff --git a/config-model/src/test/sh/test-schema.sh b/config-model/src/test/sh/test-schema.sh index 18d35f2fbf6..e037cdf3c28 100755 --- a/config-model/src/test/sh/test-schema.sh +++ b/config-model/src/test/sh/test-schema.sh @@ -2,32 +2,29 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -e -pushd src/main && make clean && make all -popd - jar="target/jing.jar" mainclass="com/thaiopensource/relaxng/util/Driver" -cmd="java -jar $jar src/main/resources/schema/services.rng src/test/schema-test-files/services.xml" +cmd="java -jar $jar target/generated-sources/trang/resources/schema/services.rng src/test/schema-test-files/services.xml" echo $cmd $cmd -cmd="java -jar $jar src/main/resources/schema/services.rng src/test/schema-test-files/standalone-container.xml" +cmd="java -jar $jar target/generated-sources/trang/resources/schema/services.rng src/test/schema-test-files/standalone-container.xml" echo $cmd $cmd -cmd="java -jar $jar src/main/resources/schema/services.rng src/test/schema-test-files/services-hosted.xml" +cmd="java -jar $jar target/generated-sources/trang/resources/schema/services.rng src/test/schema-test-files/services-hosted.xml" echo $cmd $cmd -cmd="java -jar $jar src/main/resources/schema/services.rng src/test/schema-test-files/services-hosted-infrastructure.xml" +cmd="java -jar $jar target/generated-sources/trang/resources/schema/services.rng src/test/schema-test-files/services-hosted-infrastructure.xml" echo $cmd $cmd -cmd="java -jar $jar src/main/resources/schema/deployment.rng src/test/schema-test-files/deployment.xml" +cmd="java -jar $jar target/generated-sources/trang/resources/schema/deployment.rng src/test/schema-test-files/deployment.xml" echo $cmd $cmd -cmd="java -jar $jar src/main/resources/schema/validation-overrides.rng src/test/schema-test-files/validation-overrides.xml" +cmd="java -jar $jar target/generated-sources/trang/resources/schema/validation-overrides.rng src/test/schema-test-files/validation-overrides.xml" echo $cmd $cmd diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index 9d5c1a0d798..7b8207700a3 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -628,32 +628,6 @@ ], "fields": [] }, - "com.yahoo.config.provision.Version": { - "superClass": "java.lang.Object", - "interfaces": [ - "java.lang.Comparable" - ], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public com.yahoo.component.Version toVersion()", - "public java.lang.String toSerializedForm()", - "public static com.yahoo.config.provision.Version fromIntValues(int, int, int)", - "public static com.yahoo.config.provision.Version fromString(java.lang.String)", - "public static com.yahoo.config.provision.Version from(com.yahoo.component.Version)", - "public java.lang.String toString()", - "public int getMajor()", - "public int getMinor()", - "public int getMicro()", - "public int hashCode()", - "public boolean equals(java.lang.Object)", - "public int compareTo(com.yahoo.config.provision.Version)", - "public bridge synthetic int compareTo(java.lang.Object)" - ], - "fields": [] - }, "com.yahoo.config.provision.Zone": { "superClass": "java.lang.Object", "interfaces": [], diff --git a/config-provisioning/pom.xml b/config-provisioning/pom.xml index 46e07786ab0..2915a94d371 100644 --- a/config-provisioning/pom.xml +++ b/config-provisioning/pom.xml @@ -5,13 +5,13 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <groupId>com.yahoo.vespa</groupId> <artifactId>config-provisioning</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>config-provisioning</name> <description> Provisioning APIs. diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Version.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Version.java deleted file mode 100644 index bd6e7d62833..00000000000 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Version.java +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.provision; - -/** - * The {@link Version} class is used in providing versioned config for applications. - * - * A {@link Version} object has three components: - * - * * Major version. A non-negative integer. - * * Minor version. A non-negative integer. - * * Micro version. A non-negative integer. - * - * @author Vegard Sjonfjell - * @since 5.39 - * Loosely based on component/Version.java - * {@link Version} objects are immutable. - * @deprecated use com.yahoo.component.Version - */ -// TODO: Replace usage of this by com.yahoo.component.Version -@Deprecated -public final class Version implements Comparable<Version> { - - private final int major; - private final int minor; - private final int micro; - private final String stringValue; - - /** - * @see #fromIntValues - */ - private Version(int major, int minor, int micro) { - this.major = major; - this.minor = minor; - this.micro = micro; - stringValue = toSerializedForm(); - verify(); - } - - /** - * @see #fromString - */ - private Version(String versionString) { - try { - String[] components = versionString.split("\\.", 3); - assert (components.length == 3); - major = Integer.parseInt(components[0]); - minor = Integer.parseInt(components[1]); - micro = Integer.parseInt(components[2]); - stringValue = toSerializedForm(); - verify(); - } catch (AssertionError | ArrayIndexOutOfBoundsException | IllegalArgumentException e) { - throw new IllegalArgumentException(String.format("Invalid version specification: \"%s\": %s", versionString, e.getMessage())); - } - } - - public com.yahoo.component.Version toVersion() { - return new com.yahoo.component.Version(major, minor, micro); - } - - /** - * Verifies that the numerical components in a version are legal. - * Must be called on construction after the component values are set - * - * @throws IllegalArgumentException if one of the numerical components are negative. - */ - private void verify() { - if (major < 0) - throw new IllegalArgumentException("Negative major value"); - if (minor < 0) - throw new IllegalArgumentException("Negative minor value"); - if (micro < 0) - throw new IllegalArgumentException("Negative micro value"); - } - - public String toSerializedForm() { - return String.format("%d.%d.%d", major, minor, micro); - } - - /** - * Creates a {@link Version} object from the specified components. - * - * @param major major component of the version identifier. - * @param minor minor component of the version identifier. - * @param micro micro component of the version identifier. - * @throws IllegalArgumentException if one of the numerical components are negative. - * @return {@link Version} identifier object constructed from integer components. - */ - public static Version fromIntValues(int major, int minor, int micro) { - return new Version(major, minor, micro); - } - - /** - * Creates a version object from the specified string. Version strings are in the format major.minor.micro - * - * @param versionString String representation of the version. - * @throws IllegalArgumentException If version string is improperly formatted. - * @return {@link Version} object constructed from string representation. - */ - public static Version fromString(String versionString) { - return new Version(versionString); - } - - public static Version from(com.yahoo.component.Version version) { - return new Version(version.getMajor(), version.getMinor(), version.getMicro()); - } - - /** - * Returns a string representation of this version identifier, encoded as major.minor.micro - */ - public String toString() { return stringValue; } - - /** - * Returns major version component - */ - public int getMajor() { return major; } - - /** - * Returns minor version component - */ - public int getMinor() { return minor; } - - /** - * Returns micro version component - */ - public int getMicro() { return micro; } - - @Override - public int hashCode() { return stringValue.hashCode(); } - - /** - * Performs an equality test between this {@link Version} object and another. - * - * A version is considered to be equal to another version if the - * major, minor and micro components are equal. - * - * @param object The {@link Version} object to be compared to this version. - * @return <code>true</code> if object is a - * {@link Version} and is equal to this object; - * <code>false</code> otherwise. - */ - @Override - public boolean equals(Object object) { - if (object == null || object.getClass() != this.getClass()) { - return false; - } - - Version other = (Version)object; - return this.major == other.major && this.minor == other.minor && this.micro == other.micro; - } - - /** - * Compares this {@link Version} object to another. - * - * A version is considered to be less than another version if its - * major component is less than the other version's major component, or the - * major components are equal and its minor component is less than the other - * version's minor component, or the major and minor components are equal - * and its micro component is less than the other version's micro component. - * - * A version is considered to be equal to another version if the - * major, minor and micro components are equal. - * - * @param other the {@link Version} object to be compared to this version. - * @return A negative integer, zero, or a positive integer if this object is - * less than, equal to, or greater than the specified {@link Version} object. - * @throws ClassCastException if the specified object is not a {@link Version}. - */ - @Override - public int compareTo(Version other) { - if (this == other) return 0; - - int comparison = Integer.compare(getMajor(), other.getMajor()); - if (comparison != 0) { - return comparison; - } - - comparison = Integer.compare(getMinor(), other.getMinor()); - if (comparison != 0) { - return comparison; - } - - return Integer.compare(getMicro(), other.getMicro()); - } -} diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/VersionTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/VersionTest.java deleted file mode 100644 index e5d9cb2c8c9..00000000000 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/VersionTest.java +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.provision; - -import com.yahoo.test.TotalOrderTester; -import org.junit.Test; -import static org.junit.Assert.assertThat; -import static org.hamcrest.Matchers.*; -import com.google.common.testing.EqualsTester; - -/** - * @author Vegard Sjonfjell - * @since 5.39 - */ -public class VersionTest { - @Test - public void testConstructFromIntegers() { - Version exampleVersion = Version.fromIntValues(3, 2, 1); - assertThat(exampleVersion.getMajor(), is(3)); - assertThat(exampleVersion.getMinor(), is(2)); - assertThat(exampleVersion.getMicro(), is(1)); - } - - @Test (expected = IllegalArgumentException.class) - public void testConstructFromIntegersNegativesShouldFail() throws IllegalArgumentException { - Version.fromIntValues(2, -1, 1); - } - - @Test (expected = IllegalArgumentException.class) - public void testConstructFromStringTooLongVersionStringShouldFail() throws IllegalArgumentException { - Version.fromString("3.2.1.4"); - } - - @Test (expected = IllegalArgumentException.class) - public void testConstructFromStringTooShortVersionStringShouldFail() throws IllegalArgumentException { - Version.fromString("3.2"); - } - - @Test (expected = IllegalArgumentException.class) - public void testConstructFromStringInvalidVersionStringShouldFail() throws IllegalArgumentException { - Version.fromString("4.34.3a"); - } - - @Test - public void testEncodeToStringRepresentation() { - assertThat(Version.fromIntValues(3, 2, 1).toSerializedForm(), is("3.2.1")); - assertThat(Version.fromIntValues(0, 0, 0).toSerializedForm(), is("0.0.0")); - } - - @Test - public void testEqualityAndHashCode() { - new EqualsTester() - .addEqualityGroup(Version.fromIntValues(3, 2, 1), Version.fromIntValues(3, 2, 1)) - .addEqualityGroup(Version.fromIntValues(1, 2, 3), Version.fromString("1.2.3")) - .addEqualityGroup(Version.fromString("1.5.1")) - .addEqualityGroup(Version.fromIntValues(1, 2, 1)) - .addEqualityGroup(Version.fromString("0.0.0")) - .testEquals(); - } - - @Test - public void testCompareTo() { - new TotalOrderTester<Version>() - .theseObjects(Version.fromIntValues(1, 1, 1), Version.fromIntValues(1, 1, 1)) - .areLessThan(Version.fromIntValues(2, 1, 1)) - .areLessThan(Version.fromIntValues(2, 2, 1)) - .areLessThan(Version.fromIntValues(2, 2, 2)) - .areLessThan(Version.fromIntValues(3, 0, 0)) - .testOrdering(); - } -} diff --git a/config-proxy/pom.xml b/config-proxy/pom.xml index e5498aae5ec..89d66241e79 100644 --- a/config-proxy/pom.xml +++ b/config-proxy/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config-proxy</artifactId> <packaging>jar</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>junit</groupId> diff --git a/config/pom.xml b/config/pom.xml index ea5948a30bb..40ad965a410 100755 --- a/config/pom.xml +++ b/config/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>config</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>junit</groupId> diff --git a/config/src/test/java/com/yahoo/vespa/config/configsglobal/qr-logging.cfg b/config/src/test/java/com/yahoo/vespa/config/configsglobal/qr-logging.cfg deleted file mode 100755 index cc640619e3a..00000000000 --- a/config/src/test/java/com/yahoo/vespa/config/configsglobal/qr-logging.cfg +++ /dev/null @@ -1,44 +0,0 @@ -logger com.yahoo -speciallog[6] -speciallog[0].name QueryAccessLog -speciallog[0].type file -speciallog[0].filehandler.name QueryAccessLog -speciallog[0].filehandler.pattern logs/vespa/qrs/QueryAccessLog.%Y%m%d%H%M%S -speciallog[0].filehandler.rotation "0 60 ..." -speciallog[0].cachehandler.name QueryAccessLog -speciallog[0].cachehandler.size 1000 -speciallog[1].name QueryResultLog -speciallog[1].type cache -speciallog[1].filehandler.name QueryResultLog -speciallog[1].filehandler.pattern logs/vespa/qrs/QueryResultLog.%Y%m%d%H%M%S -speciallog[1].filehandler.rotation "0 60 ..." -speciallog[1].cachehandler.name QueryResultLog -speciallog[1].cachehandler.size 1000 -speciallog[2].name ResultImpressionLog -speciallog[2].type file -speciallog[2].filehandler.name ResultImpressionLog -speciallog[2].filehandler.pattern logs/vespa/qrs/ResultImpressionLog.%Y%m%d%H%M%S -speciallog[2].filehandler.rotation "0 60 ..." -speciallog[2].cachehandler.name ResultImpressionLog -speciallog[2].cachehandler.size 1000 -speciallog[3].name ServiceEventLog -speciallog[3].type cache -speciallog[3].filehandler.name ServiceEventLog -speciallog[3].filehandler.pattern logs/vespa/qrs/ServiceEventLog.%Y%m%d%H%M%S -speciallog[3].filehandler.rotation "0 60 ..." -speciallog[3].cachehandler.name ServiceEventLog -speciallog[3].cachehandler.size 1000 -speciallog[4].name ServiceStatusLog -speciallog[4].type off -speciallog[4].filehandler.name ServiceStatusLog -speciallog[4].filehandler.pattern logs/vespa/qrs/ServiceStatusLog.%Y%m%d%H%M%S -speciallog[4].filehandler.rotation "0 60 ..." -speciallog[4].cachehandler.name ServiceStatusLog -speciallog[4].cachehandler.size 1000 -speciallog[5].name ServiceTraceLog -speciallog[5].type parent -speciallog[5].filehandler.name ServiceTraceLog -speciallog[5].filehandler.pattern logs/vespa/qrs/ServiceTraceLog.%Y%m%d%H%M%S -speciallog[5].filehandler.rotation "0 60 ..." -speciallog[5].cachehandler.name ServiceTraceLog -speciallog[5].cachehandler.size 1000 diff --git a/config_test/pom.xml b/config_test/pom.xml index ad96b5135c3..ed52589c6af 100644 --- a/config_test/pom.xml +++ b/config_test/pom.xml @@ -6,7 +6,7 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>config_test</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>junit</groupId> diff --git a/configdefinitions/pom.xml b/configdefinitions/pom.xml index 487c43ed3b3..97aa75ee90b 100644 --- a/configdefinitions/pom.xml +++ b/configdefinitions/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>configdefinitions</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>configdefinitions</name> <dependencies> <dependency> diff --git a/configdefinitions/src/vespa/imported-fields.def b/configdefinitions/src/vespa/imported-fields.def index 051226f1615..0947ff63e2a 100644 --- a/configdefinitions/src/vespa/imported-fields.def +++ b/configdefinitions/src/vespa/imported-fields.def @@ -9,12 +9,3 @@ attribute[].referencefield string # The name of the target attribute field in the parent document type that is imported into this document type. attribute[].targetfield string - -# Deprecated. TODO: Remove when going to Vespa 7. -# The data type of the target attribute field. This enum should match the one in attributes.def. -attribute[].datatype enum { STRING, UINT1, UINT2, UINT4, INT8, INT16, INT32, INT64, FLOAT, DOUBLE, PREDICATE, TENSOR, REFERENCE, NONE } default=NONE - -# Deprecated. TODO: Remove when going to Vespa 7. -# The collection type of the target attribute field. This enum should match the one in attributes.def. -attribute[].collectiontype enum { SINGLE, ARRAY, WEIGHTEDSET } default=SINGLE - diff --git a/configdefinitions/src/vespa/indexschema.def b/configdefinitions/src/vespa/indexschema.def index 5352e4d9009..7153164bfab 100644 --- a/configdefinitions/src/vespa/indexschema.def +++ b/configdefinitions/src/vespa/indexschema.def @@ -4,10 +4,8 @@ namespace=vespa.config.search ## The name of the index field. indexfield[].name string -## This is the type of an index. Currently only rise and vespa are supported -indexfield[].indextype enum {VESPA, RISE} default=VESPA ## The data type of the index field. -indexfield[].datatype enum { STRING, INT64, BOOLEANTREE } default=STRING +indexfield[].datatype enum { STRING, INT64 } default=STRING ## The collection type of the index field. indexfield[].collectiontype enum { SINGLE, ARRAY, WEIGHTEDSET } default=SINGLE ## Whether the index should support prefix searches. diff --git a/configdefinitions/src/vespa/slobroks.def b/configdefinitions/src/vespa/slobroks.def index 2dd17b307e1..b4e5536d2df 100644 --- a/configdefinitions/src/vespa/slobroks.def +++ b/configdefinitions/src/vespa/slobroks.def @@ -2,8 +2,7 @@ namespace=cloud.config ### All params must be flagged as 'restart' because this config is manually -### retrieved by ConfiguredApplication.start to init the rpc server in -### com.yahoo.container.Server. +### retrieved by ConfiguredApplication.start to init the rpc server ## The connectionspec for a slobrok is used for connecting to it using ## the FNET Remote Tools framework. It is normally on the form diff --git a/configgen/pom.xml b/configgen/pom.xml index fc7be700373..7d316061a99 100644 --- a/configgen/pom.xml +++ b/configgen/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>configgen</artifactId> <packaging>jar</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>configgen</name> <description>Config java code generation from config definition files for Java Vespa components.</description> <dependencies> diff --git a/configserver-flags/pom.xml b/configserver-flags/pom.xml index c6fdbe89b95..a55a8db5f56 100644 --- a/configserver-flags/pom.xml +++ b/configserver-flags/pom.xml @@ -8,13 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>configserver-flags</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> - <name>${project.artifactId}</name> <description>Config Server Flags.</description> <dependencies> diff --git a/configserver/pom.xml b/configserver/pom.xml index 210e584fd85..cf75eb3b999 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>configserver</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>org.hamcrest</groupId> diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index cacbd45ed8e..c3f3e7b6fff 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -14,7 +14,7 @@ <initiallyInRotation>false</initiallyInRotation> </config> - <accesslog type="vespa" fileNamePattern="logs/vespa/configserver/access.log.%Y%m%d%H%M%S" rotationScheme="date" compressOnRotation="true" symlinkName="access.log" /> + <accesslog type="vespa" fileNamePattern="logs/vespa/configserver/access.log.%Y%m%d%H%M%S" compressOnRotation="true" symlinkName="access.log" /> <preprocess:include file='access-logging.xml' required='false' /> <component id="com.yahoo.vespa.config.server.ConfigServerBootstrap" bundle="configserver" /> @@ -57,9 +57,6 @@ <preprocess:include file='hosted-vespa/routing-status.xml' required='false' /> <preprocess:include file='model-integration.xml' required='true' /> - <!-- TODO Vespa 7: Remove scoreboard.xml, replaced by metrics-packets.xml --> - <preprocess:include file='hosted-vespa/scoreboard.xml' required='false' /> - <component id="com.yahoo.vespa.configserver.flags.ConfigServerFlagSource" bundle="configserver-flags"/> <component id="com.yahoo.vespa.configserver.flags.db.FlagsDbImpl" bundle="configserver-flags"/> diff --git a/configserver/src/main/sh/start-configserver b/configserver/src/main/sh/start-configserver index 4248372013e..b3fab139cfe 100755 --- a/configserver/src/main/sh/start-configserver +++ b/configserver/src/main/sh/start-configserver @@ -171,6 +171,11 @@ vespa-run-as-vespa-user vespa-runserver -s configserver -r 30 -p $pidfile -- \ -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${VESPA_HOME}/var/crash \ -XX:+ExitOnOutOfMemoryError \ $jvmargs \ + --illegal-access=warn \ + --add-opens=java.base/java.io=ALL-UNNAMED \ + --add-opens=java.base/java.lang=ALL-UNNAMED \ + --add-opens=java.base/java.net=ALL-UNNAMED \ + --add-opens=java.base/jdk.internal.loader=ALL-UNNAMED \ -Djava.library.path=${VESPA_HOME}/lib64 \ -Djava.awt.headless=true \ -Dsun.rmi.dgc.client.gcInterval=3600000 \ diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java index 0595be3230f..1310c9d7bf5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java @@ -145,16 +145,18 @@ public class SuperModelControllerTest { Applications.Hosts hosts = app.hosts(host); assertThat(hosts.hostname(), is(host)); for (Map.Entry<String, Applications.Hosts.Services> e : app.hosts(host).services().entrySet()) { - System.out.println(e); + System.out.println(e.getKey()); if ("qrserver".equals(e.getKey())) { Applications.Hosts.Services s = e.getValue(); + System.out.println(s); assertThat(s.type(), is("qrserver")); assertThat(s.ports().size(), is(4)); + assertThat(s.ports().get(0).number(), is(8000)); assertThat(s.index(), is(0)); return; } } - org.junit.Assert.fail("No qrserver service in config"); + org.junit.Assert.fail("No container service in config"); } private DeployState createDeployState(File applicationPackage, ApplicationId applicationId) { @@ -170,6 +172,3 @@ public class SuperModelControllerTest { } } - - - diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/spooler.def b/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/spooler.def deleted file mode 100644 index 2721af7e278..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/configdefs/spooler.def +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -# Which directory to find spool files in. -directory string default="/home/vespa/var/spool/vespa" - -# If true, move successfully processed files to <directory>/success -keepsuccess bool default=false - -# Trace level on error messages from messagebus -tracelevel int default=5 - -# Which parsers to use and config for each of them. -parsers[].classname string -parsers[].parameters[].key string -parsers[].parameters[].value string diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/spooler.clients-spooler-spooler.cfg b/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/spooler.clients-spooler-spooler.cfg deleted file mode 100644 index 038d655e83c..00000000000 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/modelconfigs/spooler.clients-spooler-spooler.cfg +++ /dev/null @@ -1,13 +0,0 @@ -directory /home/vespa/var/spool/vespa/ -keepsuccess false -parsers[4] -parsers[0].classname com.yahoo.vespaspooler.XMLFileParser -parsers[0].parameters[0] -parsers[1].classname com.yahoo.mail.vespa.spooler.MailFileParser -parsers[1].parameters[0] -parsers[2].classname com.yahoo.mail.vespa.spooler.UserDeleteParser -parsers[2].parameters[0] -parsers[3].classname com.yahoo.mail.vespa.spooler.VespaGrimParser -parsers[3].parameters[1] -parsers[3].parameters[0].key chunksize -parsers[3].parameters[0].value 5 diff --git a/container-accesslogging/pom.xml b/container-accesslogging/pom.xml index b2c9bd7db8b..62dd9cf170c 100644 --- a/container-accesslogging/pom.xml +++ b/container-accesslogging/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-accesslogging</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java index 452466d7d05..46e5aa95dab 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java @@ -61,7 +61,6 @@ public class AccessLogEntry { private long timeStampMillis; private long durationBetweenRequestResponseMillis; private long numBytesReturned; - private URI uri; private String remoteAddress; @@ -600,27 +599,6 @@ public class AccessLogEntry { } } - /** - * @deprecated Use {@link #setRawPath(String)} and {@link #setRawQuery(String)} instead. - */ - @Deprecated - public void setURI(final URI uri) { - synchronized (monitor) { - requireNull(this.uri); - this.uri = uri; - } - } - - /** - * @deprecated Use {@link #getRawPath()} and {@link #getRawQuery()} instead. This method may return wrong path. - */ - @Deprecated - public URI getURI() { - synchronized (monitor) { - return uri; - } - } - public void setRemoteAddress(String remoteAddress) { synchronized (monitor) { requireNull(this.remoteAddress); diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogHandler.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogHandler.java index a31e7b3b849..488a6137cc2 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogHandler.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogHandler.java @@ -3,8 +3,6 @@ package com.yahoo.container.logging; import com.yahoo.container.core.AccessLogConfig; -import static com.yahoo.container.core.AccessLogConfig.FileHandler.RotateScheme.DATE; - import java.util.logging.Logger; /** @@ -18,14 +16,12 @@ class AccessLogHandler { public AccessLogHandler(AccessLogConfig.FileHandler config) { access.setUseParentHandlers(false); - logFileHandler = new LogFileHandler(config.rotateScheme(), config.compressOnRotation()); + logFileHandler = new LogFileHandler(config.compressOnRotation()); logFileHandler.setFilePattern(config.pattern()); logFileHandler.setRotationTimes(config.rotation()); - if (config.rotateScheme() == DATE) - createSymlink(config, logFileHandler); - + createSymlink(config, logFileHandler); LogFormatter lf = new LogFormatter(); lf.messageOnly(true); diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java index 1f56523bce6..3912a3be853 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java @@ -38,8 +38,6 @@ import java.util.zip.GZIPOutputStream; public class LogFileHandler extends StreamHandler { private final static Logger logger = Logger.getLogger(LogFileHandler.class.getName()); - /** True to use the sequence file name scheme, false (default) to use the date scheme */ - private final boolean useSequenceNameScheme; private final boolean compressOnRotation; private long[] rotationTimes = {0}; //default to one log per day, at midnight private String filePattern = "./log.%T"; // default to current directory, ms time stamp @@ -101,17 +99,12 @@ public class LogFileHandler extends StreamHandler { private final LogThread logThread; LogFileHandler() { - this(AccessLogConfig.FileHandler.RotateScheme.Enum.DATE, false); + this(false); } - LogFileHandler(boolean compressOnRotation) { - this(AccessLogConfig.FileHandler.RotateScheme.Enum.DATE, compressOnRotation); - } - - LogFileHandler(AccessLogConfig.FileHandler.RotateScheme.Enum rotateScheme, boolean compressOnRotation) + LogFileHandler(boolean compressOnRotation) { super(); - this.useSequenceNameScheme = (rotateScheme == AccessLogConfig.FileHandler.RotateScheme.Enum.SEQUENCE); this.compressOnRotation = compressOnRotation; logThread = new LogThread(this); logThread.start(); @@ -237,22 +230,18 @@ public class LogFileHandler extends StreamHandler { super.flush(); super.close(); - if (useSequenceNameScheme) - moveCurrentFile(); - try { checkAndCreateDir(fileName); FileOutputStream os = new FileOutputStream(fileName, true); // append mode, for safety super.setOutputStream(os); currentOutputStream = os; - if (! useSequenceNameScheme) LogFileDb.nowLoggingTo(fileName); + LogFileDb.nowLoggingTo(fileName); } catch (IOException e) { throw new RuntimeException("Couldn't open log file '" + fileName + "'", e); } - if ( ! useSequenceNameScheme) - createSymlinkToCurrentFile(); + createSymlinkToCurrentFile(); numberOfRecords = 0; lastRotationTime = now; diff --git a/container-accesslogging/src/main/resources/configdefinitions/access-log.def b/container-accesslogging/src/main/resources/configdefinitions/access-log.def index 9df9299ae19..788fadcdf90 100644 --- a/container-accesslogging/src/main/resources/configdefinitions/access-log.def +++ b/container-accesslogging/src/main/resources/configdefinitions/access-log.def @@ -9,28 +9,9 @@ fileHandler.pattern string # Could just configure "every N minutes" instead fileHandler.rotation string default="0 60 ..." -# TODO remove in Vespa 7, always use DATE -# -# Defines how file rotation is done. There are two options: -# -# DATE: -# The active log file is given the name resulting from pattern (but in this case "pattern" must yield a -# time-dependent name. In addition, a symlink is created pointing to the newest file. -# The symlink is given the name of the symlink parameter (or the name of this service -# if no parameter is given. -# -# SEQUENCE: -# The active log file is given the name -# defined by "pattern" (which in this case will likely just be a constant string). -# At rotation, this file is given the name pattern.N where N is 1 + the largest integer found by -# extracting the integers from all files ending by .Integer in the same directory -# -fileHandler.rotateScheme enum {DATE, SEQUENCE} default=DATE - # Use this as the name of the symlink created pointing to the newest file in the "date" naming scheme. # This is ignored if the sequence naming scheme is used. fileHandler.symlink string default="" # compress the previous access log after rotation -# TODO change to "true" for Vespa 7 -fileHandler.compressOnRotation bool default=false +fileHandler.compressOnRotation bool default=true diff --git a/container-accesslogging/src/test/java/com/yahoo/container/logging/YApacheLogTestCase.java b/container-accesslogging/src/test/java/com/yahoo/container/logging/YApacheLogTestCase.java index a643ba77703..30442089f18 100644 --- a/container-accesslogging/src/test/java/com/yahoo/container/logging/YApacheLogTestCase.java +++ b/container-accesslogging/src/test/java/com/yahoo/container/logging/YApacheLogTestCase.java @@ -187,26 +187,6 @@ public class YApacheLogTestCase { return entry; } - /** - * author someone-else. Please rewrite this. - */ - @Test - public void testYApacheAccessLogWithSequenceNamingScheme() throws IOException, InterruptedException { - // try without existing files - assertCorrectSequenceBehavior(1); - - // try with existing files - try { - new File("yapachetest2").mkdir(); - new File("yapachetest2/access.1").createNewFile(); - new File("yapachetest2/access.2").createNewFile(); - assertCorrectSequenceBehavior(3); - } - finally { - deleteDirectory("yapachetest2"); - } - } - // Prefixes that don't collide with any in the specified log format. private static final char FIELD_KEY_REQUEST_EXTRA = '@'; private static final char FIELD_KEY_RESPONSE_EXTRA = '#'; @@ -232,8 +212,7 @@ public class YApacheLogTestCase { AccessLogConfig.Builder builder = new AccessLogConfig.Builder(). fileHandler(new AccessLogConfig.FileHandler.Builder(). pattern("yapachetest2/access"). - compressOnRotation(false). - rotateScheme(AccessLogConfig.FileHandler.RotateScheme.Enum.SEQUENCE)); + compressOnRotation(false)); AccessLogConfig config = new AccessLogConfig(builder); YApacheAccessLog accessLog = new YApacheAccessLog(config); diff --git a/container-core/CMakeLists.txt b/container-core/CMakeLists.txt index 2d5723865eb..aa9291b20d5 100644 --- a/container-core/CMakeLists.txt +++ b/container-core/CMakeLists.txt @@ -2,14 +2,11 @@ install_config_definition(src/main/resources/configdefinitions/application-metadata.def container.core.application-metadata.def) install_config_definition(src/main/resources/configdefinitions/container-document.def container.core.document.container-document.def) install_config_definition(src/main/resources/configdefinitions/container-http.def container.core.container-http.def) -install_config_definition(src/main/resources/configdefinitions/diagnostics.def container.core.diagnostics.def) install_config_definition(src/main/resources/configdefinitions/health-monitor.def container.jdisc.config.health-monitor.def) install_config_definition(src/main/resources/configdefinitions/http-filter.def container.core.http.http-filter.def) install_config_definition(src/main/resources/configdefinitions/metrics-presentation.def metrics.metrics-presentation.def) install_config_definition(src/main/resources/configdefinitions/mockservice.def container.handler.test.mockservice.def) -install_config_definition(src/main/resources/configdefinitions/qr-logging.def container.core.qr-logging.def) install_config_definition(src/main/resources/configdefinitions/qr-searchers.def container.qr-searchers.def) -install_config_definition(src/main/resources/configdefinitions/qr-templates.def container.core.qr-templates.def) install_config_definition(src/main/resources/configdefinitions/qr.def container.qr.def) install_config_definition(src/main/resources/configdefinitions/servlet-config.def container.servlet.servlet-config.def) install_config_definition(src/main/resources/configdefinitions/threadpool.def container.handler.threadpool.def) diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 8c8c241a5dd..dc13bd1807d 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -595,7 +595,6 @@ ], "fields": [ "public static final java.lang.String CONTENT_TYPE", - "public static final java.lang.String DATE", "protected final java.util.logging.Logger log" ] }, @@ -671,21 +670,6 @@ "public final int code" ] }, - "com.yahoo.container.protect.FreezeDetector": { - "superClass": "com.yahoo.component.AbstractComponent", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.container.core.DiagnosticsConfig)", - "public void register(com.yahoo.concurrent.ThreadLocalDirectory)", - "public boolean isBreakdown()", - "public void unRegister(com.yahoo.concurrent.ThreadLocalDirectory)", - "public void deconstruct()" - ], - "fields": [] - }, "com.yahoo.container.protect.ProcessTerminator": { "superClass": "java.lang.Object", "interfaces": [], @@ -699,40 +683,6 @@ ], "fields": [] }, - "com.yahoo.container.protect.TimeoutCollector": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.concurrent.ThreadLocalDirectory$Updater" - ], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>()", - "public com.yahoo.container.protect.TimeoutRate createGenerationInstance(com.yahoo.container.protect.TimeoutRate)", - "public com.yahoo.container.protect.TimeoutRate update(com.yahoo.container.protect.TimeoutRate, java.lang.Boolean)", - "public bridge synthetic java.lang.Object update(java.lang.Object, java.lang.Object)", - "public bridge synthetic java.lang.Object createGenerationInstance(java.lang.Object)" - ], - "fields": [] - }, - "com.yahoo.container.protect.TimeoutRate": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>()", - "public void addQuery(java.lang.Boolean)", - "public void merge(com.yahoo.container.protect.TimeoutRate)", - "public double timeoutFraction()", - "public int getTotal()" - ], - "fields": [] - }, "com.yahoo.processing.handler.AbstractProcessingHandler": { "superClass": "com.yahoo.container.jdisc.LoggingRequestHandler", "interfaces": [], diff --git a/container-core/pom.xml b/container-core/pom.xml index 7f616699c8e..e43a5b5df81 100644 --- a/container-core/pom.xml +++ b/container-core/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-core</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> diff --git a/container-core/src/main/java/com/yahoo/container/Container.java b/container-core/src/main/java/com/yahoo/container/Container.java index e84c8b340a4..031d4a26d05 100755 --- a/container-core/src/main/java/com/yahoo/container/Container.java +++ b/container-core/src/main/java/com/yahoo/container/Container.java @@ -3,13 +3,11 @@ package com.yahoo.container; import com.yahoo.component.AbstractComponent; import com.yahoo.component.provider.ComponentRegistry; -import com.yahoo.container.core.config.BundleLoader; import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.filedistribution.fileacquirer.FileAcquirerFactory; import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.service.ClientProvider; import com.yahoo.jdisc.service.ServerProvider; -import com.yahoo.osgi.Osgi; import com.yahoo.vespa.config.ConfigTransformer; import com.yahoo.vespa.config.UrlDownloader; @@ -34,8 +32,6 @@ public class Container { private volatile FileAcquirer fileAcquirer; private volatile UrlDownloader urlDownloader; - private volatile BundleLoader bundleLoader; - private static Logger logger = Logger.getLogger(Container.class.getName()); // TODO: Make this final again. @@ -43,12 +39,6 @@ public class Container { public static Container get() { return instance; } - /** @deprecated do not use */ - @Deprecated // TODO: Remove - public void setOsgi(Osgi osgi) { - bundleLoader = new BundleLoader(osgi); - } - public void shutdown() { if (fileAcquirer != null) fileAcquirer.shutdown(); @@ -61,14 +51,6 @@ public class Container { return fileAcquirer; } - /** @deprecated do not use */ - @Deprecated // TODO: Remove - public BundleLoader getBundleLoader() { - if (bundleLoader == null) - bundleLoader = new BundleLoader(null); - return bundleLoader; - } - /** * Hack. For internal use only, will be removed later * diff --git a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java index f7a3f49538e..cca3c5c4610 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java @@ -9,7 +9,6 @@ import com.yahoo.component.ComponentSpecification; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.config.FileReference; -import com.yahoo.container.core.DiagnosticsConfig; import com.yahoo.container.di.ComponentDeconstructor; import com.yahoo.container.di.Container; import com.yahoo.container.di.componentgraph.core.ComponentGraph; @@ -148,9 +147,6 @@ public class HandlersConfigurerDi { protected void configure() { bind(com.yahoo.container.Container.class).toInstance(vespaContainer); bind(com.yahoo.statistics.Statistics.class).toInstance(Statistics.nullImplementation); - bind(com.yahoo.container.protect.FreezeDetector.class).toInstance( - new com.yahoo.container.protect.FreezeDetector( - new DiagnosticsConfig(new DiagnosticsConfig.Builder().disabled(true)))); bind(AccessLog.class).toInstance(new AccessLog(new ComponentRegistry<>())); bind(Executor.class).toInstance(Executors.newCachedThreadPool(ThreadFactoryFactory.getThreadFactory("HandlersConfigurerDI"))); diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java index 60affddeb60..c8cf575dff3 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java @@ -38,7 +38,6 @@ public final class VipStatusHandler extends ThreadedHttpRequestHandler { private final boolean accessDisk; private final File statusFile; private final VipStatus vipStatus; - private final boolean noSearchBackendsImpliesOutOfService; private volatile boolean previouslyInRotation = true; @@ -57,7 +56,7 @@ public final class VipStatusHandler extends ThreadedHttpRequestHandler { private StatusResponse() { super(com.yahoo.jdisc.http.HttpResponse.Status.OK); // status may be overwritten below - if (noSearchBackendsImpliesOutOfService && !vipStatus.isInRotation()) { + if (vipStatus != null && ! vipStatus.isInRotation()) { searchContainerOutOfService(); } else if (accessDisk) { preSlurpFile(); @@ -179,7 +178,6 @@ public final class VipStatusHandler extends ThreadedHttpRequestHandler { super(executor, metric); this.accessDisk = vipConfig.accessdisk(); this.statusFile = new File(Defaults.getDefaults().underVespaHome(vipConfig.statusfile())); - this.noSearchBackendsImpliesOutOfService = vipConfig.noSearchBackendsImpliesOutOfService(); this.vipStatus = vipStatus; } @@ -187,9 +185,8 @@ public final class VipStatusHandler extends ThreadedHttpRequestHandler { public HttpResponse handle(HttpRequest request) { if (metric != null) metric.add(NUM_REQUESTS_METRIC, 1, null); - if (noSearchBackendsImpliesOutOfService) { + if (vipStatus != null) updateAndLogRotationState(); - } return new StatusResponse(); } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java index 73b56cd89f5..705092ef12e 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java @@ -295,7 +295,6 @@ public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { logEntry.setRemotePort(remoteAddress.getPort()); } URI uri = AccessLogUtil.getUri(httpRequest); - setDeprecatedUri(logEntry, uri); logEntry.setRawPath(uri.getRawPath()); logEntry.setRawQuery(uri.getRawQuery()); logEntry.setUserAgent(AccessLogUtil.getUserAgentHeader(httpRequest)); @@ -307,9 +306,4 @@ public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { } } - @SuppressWarnings("deprecation") - private static void setDeprecatedUri(AccessLogEntry logEntry, URI uri) { - logEntry.setURI(uri); - } - } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/MetricConsumerFactory.java b/container-core/src/main/java/com/yahoo/container/jdisc/MetricConsumerFactory.java index 50052bbf70c..d0cf07584e4 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/MetricConsumerFactory.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/MetricConsumerFactory.java @@ -4,7 +4,7 @@ package com.yahoo.container.jdisc; import com.yahoo.jdisc.application.MetricConsumer; /** - * <p>This is the interface to implement if one wishes to configure a non-default <tt>MetricConsumer</tt>. Simply + * <p>This is the interface to implement if one wishes to configure a non-default <code>MetricConsumer</code>. Simply * add the implementing class as a component in your services.xml file.</p> * * @author Simon Thoresen Hult diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java index 670c48bb339..dddde1205ca 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java @@ -30,7 +30,6 @@ import java.util.logging.Logger; public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler { public static final String CONTENT_TYPE = "Content-Type"; - public static final String DATE = "Date"; // TODO: Remove on Vespa 7 private static final String RENDERING_ERRORS = "rendering_errors"; /** Logger for subclasses */ diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/VespaHeaders.java b/container-core/src/main/java/com/yahoo/container/jdisc/VespaHeaders.java index 715038470b9..309405f1242 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/VespaHeaders.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/VespaHeaders.java @@ -54,6 +54,7 @@ public final class VespaHeaders { } } + /** Returns true if this is a benchmarking request, according to headers */ public static boolean benchmarkOutput(com.yahoo.container.jdisc.HttpRequest request) { return request.getHeader(BenchmarkingHeaders.REQUEST) != null; } @@ -69,8 +70,7 @@ public final class VespaHeaders { * @param coverage The Coverage to read data from. */ public static void benchmarkOutput(HeaderFields responseHeaders, boolean benchmarkCoverage, - Timing t, HitCounts c, int errorCount, Coverage coverage) - { + Timing t, HitCounts c, int errorCount, Coverage coverage) { final long renderStartTime = System.currentTimeMillis(); if (c != null) { // Fill inn response getHeaders diff --git a/container-core/src/main/java/com/yahoo/container/protect/FreezeDetector.java b/container-core/src/main/java/com/yahoo/container/protect/FreezeDetector.java deleted file mode 100644 index 6a8a3239c17..00000000000 --- a/container-core/src/main/java/com/yahoo/container/protect/FreezeDetector.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.protect; - -import java.util.Timer; - -import com.yahoo.component.AbstractComponent; -import com.yahoo.concurrent.ThreadLocalDirectory; -import com.yahoo.container.core.DiagnosticsConfig; - -/** - * Runs and initializes a {@link Watchdog} instance. - * - * @author Steinar Knutsen - * @deprecated this is not in use and will be removed in the next major release - */ -@Deprecated -// TODO: Remove on Vespa 7 -public class FreezeDetector extends AbstractComponent { - - private final Timer timeoutWatchdog; - private final Watchdog watchdog; - - public FreezeDetector(DiagnosticsConfig diagnosticsConfig) { - timeoutWatchdog = null; - watchdog = null; - } - - public void register(ThreadLocalDirectory<TimeoutRate, Boolean> timeouts) { - if (watchdog == null) { - return; - } - watchdog.addTimeouts(timeouts); - } - - public boolean isBreakdown() { - if (watchdog == null) { - return false; - } - return watchdog.isBreakdown(); - } - - public void unRegister(ThreadLocalDirectory<TimeoutRate, Boolean> timeouts) { - if (watchdog == null) { - return; - } - watchdog.removeTimeouts(timeouts); - } - - @Override - public void deconstruct() { - super.deconstruct(); - if (timeoutWatchdog != null) { - timeoutWatchdog.cancel(); - } - } - -} diff --git a/container-core/src/main/java/com/yahoo/container/protect/TimeoutCollector.java b/container-core/src/main/java/com/yahoo/container/protect/TimeoutCollector.java deleted file mode 100644 index e31716a292e..00000000000 --- a/container-core/src/main/java/com/yahoo/container/protect/TimeoutCollector.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.protect; - -import com.yahoo.concurrent.ThreadLocalDirectory.Updater; - -/** - * Allocator and glue for sampling timeouts in SearchHandler. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @deprecated this is not in use and will be removed in the next major release - */ -@Deprecated -public final class TimeoutCollector implements Updater<TimeoutRate, Boolean> { - - @Override - public TimeoutRate createGenerationInstance(TimeoutRate previous) { - return new TimeoutRate(); - } - - @Override - public TimeoutRate update(TimeoutRate current, Boolean x) { - current.addQuery(x); - return current; - } - -} diff --git a/container-core/src/main/java/com/yahoo/container/protect/TimeoutRate.java b/container-core/src/main/java/com/yahoo/container/protect/TimeoutRate.java deleted file mode 100644 index 4bb68fedb9b..00000000000 --- a/container-core/src/main/java/com/yahoo/container/protect/TimeoutRate.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.protect; - -/** - * Helper class to account for measuring how many queries times outs. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @deprecated this is not in use and will be removed in the next major release - */ -// TODO: Remove on Vespa 7 -@Deprecated -public final class TimeoutRate { - - private int timeouts = 0; - private int total = 0; - - public void addQuery(Boolean timeout) { - if (timeout) { - timeouts += 1; - } - total += 1; - } - - public void merge(TimeoutRate other) { - timeouts += other.timeouts; - total += other.total; - } - - public double timeoutFraction() { - if (total == 0) { - return 0.0d; - } else { - return ((double) timeouts) / ((double) total); - } - } - - public int getTotal() { - return total; - } - -} diff --git a/container-core/src/main/java/com/yahoo/container/protect/Watchdog.java b/container-core/src/main/java/com/yahoo/container/protect/Watchdog.java deleted file mode 100644 index ff425242587..00000000000 --- a/container-core/src/main/java/com/yahoo/container/protect/Watchdog.java +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.protect; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.TimerTask; -import java.util.logging.Level; -import java.util.logging.Logger; - -import com.yahoo.concurrent.ThreadLocalDirectory; -import com.yahoo.log.LogLevel; -import com.yahoo.protect.Process; - -/** - * Watchdog for a frozen process, too many timeouts, etc. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @deprecated this is not in use and will be removed in the next major release - */ -// TODO: Remove on Vespa 7 -@Deprecated -class Watchdog extends TimerTask { - - public static final String FREEZEDETECTOR_DISABLE = "vespa.freezedetector.disable"; - Logger log = Logger.getLogger(Watchdog.class.getName()); - private long lastRun = 0L; - private long lastQpsCheck = 0L; - // Local copy to avoid ever _reading_ the volatile version - private boolean breakdownCopy = false; - private volatile boolean breakdown; - // The fraction of queries which must time out to view the QRS as being - // in breakdown - private final double timeoutThreshold; - // The minimal QPS to care about timeoutThreshold - private final int minimalQps; - private final boolean disableSevereBreakdownCheck; - private final List<ThreadLocalDirectory<TimeoutRate, Boolean>> timeoutRegistry = new ArrayList<>(); - private final boolean shutdownIfFrozen; - - Watchdog(double timeoutThreshold, int minimalQps, boolean shutdownIfFrozen) { - this.timeoutThreshold = timeoutThreshold; - this.minimalQps = minimalQps; - if (System.getProperty(FREEZEDETECTOR_DISABLE) != null) { - disableSevereBreakdownCheck = true; - } else { - disableSevereBreakdownCheck = false; - } - this.shutdownIfFrozen = shutdownIfFrozen; - } - - @Override - public void run() { - long now = System.currentTimeMillis(); - if (lastRun != 0L) { - severeBreakdown(now); - queryTimeouts(now); - } else { - lastQpsCheck = now; - } - lastRun = now; - } - - private void severeBreakdown(final long now) { - if (disableSevereBreakdownCheck) { - return; - } - if (now - lastRun < 5000L) { - return; - } - - threadStackMessage(); - - if (shutdownIfFrozen) { - Process.logAndDie("Watchdog timer meant to run ten times per second" - + " not run for five seconds or more." - + " Assuming severe failure or overloaded node, shutting down container."); - } else { - log.log(LogLevel.ERROR, - "A watchdog meant to run 10 times a second has not been invoked for 5 seconds." - + " This usually means this machine is swapping or otherwise severely overloaded."); - } - } - - private void threadStackMessage() { - log.log(LogLevel.INFO, "System seems unresponsive, performing full thread dump for diagnostics."); - threadDump(); - log.log(LogLevel.INFO, "End of diagnostic thread dump."); - } - - private void threadDump() { - try { - Map<Thread, StackTraceElement[]> allStackTraces = Thread.getAllStackTraces(); - for (Map.Entry<Thread, StackTraceElement[]> e : allStackTraces.entrySet()) { - Thread t = e.getKey(); - StackTraceElement[] stack = e.getValue(); - StringBuilder forOneThread = new StringBuilder(); - int initLen; - forOneThread.append("Stack for thread: ").append(t.getName()).append(": "); - initLen = forOneThread.length(); - for (StackTraceElement s : stack) { - if (forOneThread.length() > initLen) { - forOneThread.append(" "); - } - forOneThread.append(s.toString()); - } - log.log(LogLevel.INFO, forOneThread.toString()); - } - } catch (Exception e) { - // just give up... - } - } - - private void queryTimeouts(final long now) { - // only check query timeout every 10s - if (now - lastQpsCheck < 10000L) { - return; - } else { - lastQpsCheck = now; - } - - final TimeoutRate globalState = new TimeoutRate(); - synchronized (timeoutRegistry) { - for (ThreadLocalDirectory<TimeoutRate, Boolean> timeouts : timeoutRegistry) { - final List<TimeoutRate> threadStates = timeouts.fetch(); - for (final TimeoutRate t : threadStates) { - globalState.merge(t); - } - } - } - if (globalState.timeoutFraction() > timeoutThreshold && globalState.getTotal() > (10 * minimalQps)) { - setBreakdown(true); - log.log(Level.WARNING, "Too many queries timed out. Assuming container is in breakdown."); - } else { - if (!breakdown()) { - return; - } - setBreakdown(false); - log.log(Level.WARNING, "Fewer queries timed out. Assuming container is no longer in breakdown."); - } - } - - private void setBreakdown(final boolean state) { - breakdown = state; - breakdownCopy = state; - } - - private boolean breakdown() { - return breakdownCopy; - } - - boolean isBreakdown() { - return breakdown; - } - - void addTimeouts(ThreadLocalDirectory<TimeoutRate, Boolean> t) { - synchronized (timeoutRegistry) { - timeoutRegistry.add(t); - } - } - - void removeTimeouts(ThreadLocalDirectory<TimeoutRate, Boolean> timeouts) { - synchronized (timeoutRegistry) { - timeoutRegistry.remove(timeouts); - } - } - -} diff --git a/container-core/src/main/java/com/yahoo/container/xml/bind/JAXBContextFactory.java b/container-core/src/main/java/com/yahoo/container/xml/bind/JAXBContextFactory.java deleted file mode 100644 index c2355ce74be..00000000000 --- a/container-core/src/main/java/com/yahoo/container/xml/bind/JAXBContextFactory.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.xml.bind; - -import javax.xml.bind.JAXBContext; -import javax.xml.bind.JAXBException; - -/** - * Container components can take an instance of this class as a constructor argument, - * to get a new instance injected by the container framework. There is usually no - * need to create an instance with this class' constructor. - * <p> - * This factory is needed because the JAXBContext needs a user defined context path, - * which means that it cannot be created at the time the container creates its - * component graph. - * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @author gjoranv - * @since 5.3 - * @deprecated Do not use! - */ -@Deprecated -public class JAXBContextFactory { - public static final String FACTORY_CLASS = "com.sun.xml.internal.bind.v2.ContextFactory"; - - /** - * Returns a new JAXBContext for the context path defined by the given list of classes. - * @return A new JAXBContext. - * @param classes One class per package that contains schema derived classes and/or - * java to schema (JAXB-annotated) mapped classes - */ - public JAXBContext newInstance(Class<?>... classes) { - return newInstance(getContextPath(classes), classes[0].getClassLoader()); - } - - // TODO: guard against adding the same package more than once - static String getContextPath(Class<?>... classes) { - if (classes == null || classes.length == 0) { - throw new IllegalArgumentException("Empty package list."); - } - StringBuilder contextPath = new StringBuilder(); - for (Class<?> clazz : classes) { - contextPath - .append(clazz.getPackage().getName()) - .append(':'); - } - contextPath.deleteCharAt(contextPath.length() - 1); - return contextPath.toString(); - } - - private static JAXBContext newInstance(String contextPath, ClassLoader classLoader) { - System.setProperty(JAXBContext.JAXB_CONTEXT_FACTORY, FACTORY_CLASS); - try { - return JAXBContext.newInstance(contextPath, classLoader); - } catch (JAXBException e) { - throw new IllegalStateException(e); - } - } -} diff --git a/container-core/src/main/java/com/yahoo/container/xml/providers/JAXBContextFactoryProvider.java b/container-core/src/main/java/com/yahoo/container/xml/providers/JAXBContextFactoryProvider.java deleted file mode 100644 index 6d82fa7af82..00000000000 --- a/container-core/src/main/java/com/yahoo/container/xml/providers/JAXBContextFactoryProvider.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.xml.providers; - -import com.yahoo.container.di.componentgraph.Provider; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.29 - * @deprecated Do not use! - */ -@Deprecated -@SuppressWarnings("deprecation") -public class JAXBContextFactoryProvider implements Provider<com.yahoo.container.xml.bind.JAXBContextFactory> { - public static final String FACTORY_CLASS = com.yahoo.container.xml.bind.JAXBContextFactory.class.getName(); - - @Override - public com.yahoo.container.xml.bind.JAXBContextFactory get() { - return new com.yahoo.container.xml.bind.JAXBContextFactory(); - } - - @Override - public void deconstruct() { } -} diff --git a/container-core/src/main/java/com/yahoo/language/provider/SimpleLinguisticsProvider.java b/container-core/src/main/java/com/yahoo/language/provider/DefaultLinguisticsProvider.java index 169c9f1a2a4..8f08e33a777 100644 --- a/container-core/src/main/java/com/yahoo/language/provider/SimpleLinguisticsProvider.java +++ b/container-core/src/main/java/com/yahoo/language/provider/DefaultLinguisticsProvider.java @@ -2,26 +2,23 @@ package com.yahoo.language.provider; import com.google.inject.Inject; -import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.language.opennlp.OpenNlpLinguistics; import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.language.Linguistics; -import com.yahoo.language.simple.SimpleLinguisticsConfig; /** - * Provides simple linguistics if no linguistics component has been explicitly configured + * Provides the default linguistics implementation if no linguistics component has been explicitly configured * (dependency injection will fallback to providers if no components of the requested type is found). * * @author bratseth */ -public class SimpleLinguisticsProvider implements Provider<Linguistics> { +public class DefaultLinguisticsProvider implements Provider<Linguistics> { private final Linguistics linguistics; @SuppressWarnings("deprecation") @Inject - public SimpleLinguisticsProvider(SimpleLinguisticsConfig config) { - linguistics = new SimpleLinguistics(config); - } + public DefaultLinguisticsProvider() { linguistics = new OpenNlpLinguistics(); } @Override public Linguistics get() { return linguistics; } diff --git a/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java b/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java index 2269bd1358b..72a5fbdce55 100644 --- a/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java +++ b/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java @@ -47,10 +47,9 @@ import static com.yahoo.component.chain.ChainsConfigurer.prepareChainRegistry; * <p> * COMPONENT: The type of the processing components of which this executes a chain * - * @author bratseth - * @author Tony Vaagenes - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @since 5.1.6 + * @author bratseth + * @author Tony Vaagenes + * @author Steinar Knutsen */ public abstract class AbstractProcessingHandler<COMPONENT extends Processor> extends LoggingRequestHandler { diff --git a/container-core/src/main/resources/configdefinitions/diagnostics.def b/container-core/src/main/resources/configdefinitions/diagnostics.def deleted file mode 100644 index 33f733d4386..00000000000 --- a/container-core/src/main/resources/configdefinitions/diagnostics.def +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=container.core - -## The fraction of queries to time out over a period of 10s to consider -## the system in breakdown. -timeoutfraction double default=0.1 - -## Minimum QPS to consider the system in breakdown. -minimumqps int default=1 - -## Whether to shut down process if in a deadlock situation -shutdown bool default=false - -## Whether to totally disable the detector. Alternative to system property -## which works better in tests. -disabled bool default=false diff --git a/container-core/src/main/resources/configdefinitions/qr-logging.def b/container-core/src/main/resources/configdefinitions/qr-logging.def deleted file mode 100644 index f828f321ae7..00000000000 --- a/container-core/src/main/resources/configdefinitions/qr-logging.def +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=container.core -logger string default="com.yahoo" -# Either QueryAccessLog for a regular Vespa access log, or YApacheAccessLog for a log on yApache format -speciallog[].name string - -# Leave as "" -speciallog[].type string - -speciallog[].filehandler.name string default="" - -# File name patterns supporting the expected time variables -speciallog[].filehandler.pattern string default=".%Y%m%d%H%M%S" - -speciallog[].filehandler.rotation string default="0 60 ..." - -# Defines how file rotation is done. There are two options: -# -# "date" : -# The active log file is given the name resulting from pattern (but in this case "pattern" must yield a -# time-dependent name. In addition, a symlink is created pointing to the newest file. -# The symlink is given the name of the symlink parameter (or the name of this service -# if no parameter is given. -# -# "sequence" : -# The active log file is given the name -# defined by "pattern" (which in this case will likely just be a constant string). -# At rotation, this file is given the name pattern.N where N is 1 + the largest integer found by -# extracting the integers from all files ending by .Integer in the same directory -# -speciallog[].filehandler.rotatescheme string default="date" - -# Use this as the name of the symlink created pointing to the newest file in the "date" naming scheme. -# This is ignored if the sequence naming scheme is used. -speciallog[].filehandler.symlink string default="" - -speciallog[].cachehandler.name string default="" -speciallog[].cachehandler.size int default=1000 diff --git a/container-core/src/main/resources/configdefinitions/qr-searchers.def b/container-core/src/main/resources/configdefinitions/qr-searchers.def index e2b4836c081..bb8f80052f1 100644 --- a/container-core/src/main/resources/configdefinitions/qr-searchers.def +++ b/container-core/src/main/resources/configdefinitions/qr-searchers.def @@ -14,6 +14,8 @@ com.yahoo.prelude.searcher.FieldCollapsingSearcher.extrafactor double default=2. com.yahoo.prelude.searcher.FieldCollapsingSearcher.collapsefield string default="mid" com.yahoo.prelude.searcher.BlendingSearcher.numthreads int default=200 + +# The field we'll unique on, use [id] to unique on the document id com.yahoo.prelude.searcher.BlendingSearcher.docid string default="" com.yahoo.prelude.searcher.JuniperSearcher.source string default="" @@ -74,8 +76,5 @@ searchcluster[].storagecluster.routespec string default="" searchcluster[].dispatcher[].host string searchcluster[].dispatcher[].port int -## Not used. TODO: Remove on Vespa 7 -searchcluster[].rowbits int default=0 - # Per dispatcher config-id might be nice to have, remove it until needed. # searchcluster[].dispatcher[].configid reference diff --git a/container-core/src/main/resources/configdefinitions/qr-templates.def b/container-core/src/main/resources/configdefinitions/qr-templates.def deleted file mode 100644 index 46b1a5cbd65..00000000000 --- a/container-core/src/main/resources/configdefinitions/qr-templates.def +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Not used -# TODO: Remove on Vespa 7 -namespace=container.core - -## Prefix to use in queries to choose a given template -templateset[].urlprefix string - -## The id of the Java class which the given templateset -## should be an instance of. This is only used for implementing -## templates in the Java API instead of Velocity. -templateset[].classid string default="" - -## The symbolic name of the Osgi bundle this template is located in. -## Assumed to be the same as the classid if not set, and is only used -## when classid is used. -templateset[].bundle string default="" - -## The MIME type of a given template -templateset[].mimetype string default="text/html" - -## The character set of a given template -templateset[].encoding string default="iso-8859-1" - -## Not used -templateset[].rankprofile int default=0 - - -## Not used in 1.0 -templateset[].keepalive bool default=false - -## Header template. Always rendered. -templateset[].headertemplate string default="" - -## Footer template. Always rendered. -templateset[].footertemplate string default="" - -## Nohits template. Rendered if there are no hits in the result. -templateset[].nohitstemplate string default="" - -## Hit template. Rendered if there are hits in the result. -templateset[].hittemplate string default="" - -## Error template. Rendered if there is an error condition. This is -## not mutually exclusive with the (no)hit templates as such. -templateset[].errortemplate string default="" - -## Aggregated groups header template. -## Default rendering is used if missing -templateset[].groupsheadertemplate string default="[DEFAULT]" - -## Aggregated range group template. -## Default rendering is used if missing -templateset[].rangegrouptemplate string default="[DEFAULT]" - -## Aggregated exact group template -## Default rendering is used if missing -templateset[].exactgrouptemplate string default="[DEFAULT]" - -## Aggregated groups footer template. -## Default rendering is used if missing -templateset[].groupsfootertemplate string default="[DEFAULT]" - -## Tags used to highlight results, starting a bolded section. -## An empty string means the template should no override what -## was inserted by the search chain. -templateset[].highlightstarttag string default="" -## Tags used to highlight results, ending a bolded section -## An empty string means the template should no override what -## was inserted by the search chain. -templateset[].highlightendtag string default="" -## Tags used to highlight results, separating dynamic snippets -## An empty string means the template should no override what -## was inserted by the search chain. -templateset[].highlightseptag string default="" - -## The summary class to use for this template if there is none -## defined in the query. -templateset[].defaultsummaryclass string default="" diff --git a/container-core/src/main/resources/configdefinitions/qr.def b/container-core/src/main/resources/configdefinitions/qr.def index d6549674137..fe44b04e9d5 100644 --- a/container-core/src/main/resources/configdefinitions/qr.def +++ b/container-core/src/main/resources/configdefinitions/qr.def @@ -2,8 +2,7 @@ namespace=container ### All params must be flagged as 'restart' because this config is manually -### retrieved by ConfiguredApplication.start to init the rpc server in -### com.yahoo.container.Server. +### retrieved by ConfiguredApplication.start to init the rpc server ## filedistributor rpc configuration filedistributor.configid reference default="" restart @@ -20,10 +19,6 @@ rpc.host string default="" restart ## The id this service should register itself with in slobrok rpc.slobrokId string default="" restart -## Whether to obtain coverage reports by default. This can be overridden with -## the 'reportcoverage' HTTP parameter. -coveragereports bool default=false restart - ## A unique identifier string for this QRS. The only guarantee given is ## this string will be unique for every QRS in a Vespa application. discriminator string default="qrserver.0" restart diff --git a/container-core/src/main/resources/configdefinitions/vip-status.def b/container-core/src/main/resources/configdefinitions/vip-status.def index 6aa10ff1b90..ed7ab3e4802 100644 --- a/container-core/src/main/resources/configdefinitions/vip-status.def +++ b/container-core/src/main/resources/configdefinitions/vip-status.def @@ -1,12 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=container.core -## If there is a Vespa search backend connected to this container, and that -## backend is out of service, automatically remove this container from VIP -## rotation, ignoring any status file. -## TODO VESPA 7: This is always true and can be removed -noSearchBackendsImpliesOutOfService bool default=true - ## Whether to return hard-coded reply or serve "status.html" from disk accessdisk bool default=false diff --git a/container-core/src/test/java/com/yahoo/component/provider/test/ComponentClassTestCase.java b/container-core/src/test/java/com/yahoo/component/provider/test/ComponentClassTestCase.java deleted file mode 100644 index 0db12ba2fea..00000000000 --- a/container-core/src/test/java/com/yahoo/component/provider/test/ComponentClassTestCase.java +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.component.provider.test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.lang.reflect.Constructor; -import java.util.HashMap; -import java.util.Map; - -import com.yahoo.component.AbstractComponent; -import org.junit.Test; - -import com.yahoo.component.ComponentId; -import com.yahoo.component.Version; -import com.yahoo.component.provider.ComponentClass; -import com.yahoo.config.ConfigInstance; -import com.yahoo.config.core.IntConfig; -import com.yahoo.config.core.StringConfig; -import com.yahoo.vespa.config.ConfigKey; - -/** - * @author <a href="gv@yahoo-inc.com">G. Voldengen</a> - */ -@SuppressWarnings("unused") -public class ComponentClassTestCase { - - @Test - public void testComponentConstructor() throws NoSuchMethodException { - ComponentClass<A> a = new ComponentClass<>(A.class); - assertEquals(A.preferred(), a.getPreferredConstructor().getConstructor()); - - ComponentClass<B> b = new ComponentClass<>(B.class); - assertEquals(B.preferred(), b.getPreferredConstructor().getConstructor()); - - ComponentClass<C> c = new ComponentClass<>(C.class); - assertEquals(C.preferred(), c.getPreferredConstructor().getConstructor()); - - ComponentClass<E> e = new ComponentClass<>(E.class); - assertEquals(E.preferred(), e.getPreferredConstructor().getConstructor()); - - ComponentClass<G> g = new ComponentClass<>(G.class); - assertEquals(G.preferred(), g.getPreferredConstructor().getConstructor()); - - try { - ComponentClass<H> h = new ComponentClass<>(H.class); - fail("Expected exception due to no legal public constructors."); - } catch (IllegalArgumentException expected) { - assertTrue(expected.getMessage().contains("must have at least one public constructor with an optional " + - "component ID followed by an optional FileAcquirer and zero or more config arguments")); - } - - try { - ComponentClass<I> i = new ComponentClass<>(I.class); - fail("Expected exception due to no public constructors."); - } catch (RuntimeException expected) { - assertTrue(expected.getMessage().contains("Class has no public constructors")); - } - - try { - ComponentClass<J> j = new ComponentClass<>(J.class); - fail("Expected exception due to no public constructors."); - } catch (RuntimeException expected) { - assertTrue(expected.getMessage().contains("Class has no public constructors")); - } - - ComponentClass<K> k = new ComponentClass<>(K.class); - assertEquals(K.preferred(), k.getPreferredConstructor().getConstructor()); - - ComponentClass<L> l = new ComponentClass<>(L.class); - assertEquals(L.preferred(), l.getPreferredConstructor().getConstructor()); - } - - @SuppressWarnings("unchecked") - @Test - public void testCreateComponent() throws NoSuchMethodException { - Map<ConfigKey, ConfigInstance> availableConfigs = new HashMap<>(); - String configId = "testConfigId"; - availableConfigs.put(new ConfigKey(StringConfig.class, configId), new StringConfig(new StringConfig.Builder())); - availableConfigs.put(new ConfigKey(IntConfig.class, configId), new IntConfig(new IntConfig.Builder())); - - ComponentClass<TestComponent> testClass = new ComponentClass<>(TestComponent.class); - TestComponent component = testClass. - createComponent(new ComponentId("test", new Version(1)), availableConfigs, configId); - assertEquals("test", component.getId().getName()); - assertEquals(1, component.getId().getVersion().getMajor()); - assertEquals(1, component.intVal); - assertEquals("_default_", component.stringVal); - } - - /** - * Verifies that ComponentClass sets the ComponentId when a component that takes a ComponentId as - * constructor argument fails to call super(id). - */ - @Test - public void testNullIdComponent() throws NoSuchMethodException { - ComponentClass<NullIdComponent> testClass = new ComponentClass<>(NullIdComponent.class); - NullIdComponent component = testClass.createComponent(new ComponentId("null-test", new Version(1)), new HashMap<ConfigKey, ConfigInstance>(), null); - assertEquals("null-test", component.getId().getName()); - assertEquals(1, component.getId().getVersion().getMajor()); - } - - public static class TestComponent extends AbstractComponent { - private int intVal = 0; - private String stringVal = ""; - public TestComponent(ComponentId id, IntConfig intConfig, StringConfig stringConfig) { - super(id); - intVal = intConfig.intVal(); - stringVal = stringConfig.stringVal(); - } - } - - /** - * This component takes a ComponentId as constructor arg, but "forgets" to call super(id). - */ - public static class NullIdComponent extends AbstractComponent { - private int intVal = 0; - private String stringVal = ""; - public NullIdComponent(ComponentId id) { - } - } - - private static class A extends AbstractComponent { - public A(IntConfig intConfig) { } - public A(IntConfig intConfig, StringConfig stringConfig) { } - static Constructor<A> preferred() throws NoSuchMethodException{ - return A.class.getConstructor(IntConfig.class, StringConfig.class); - } - } - - private static class B extends AbstractComponent { - public B(ComponentId id, IntConfig intConfig) { } - public B(IntConfig intConfig) { } - static Constructor<B> preferred() throws NoSuchMethodException{ - return B.class.getConstructor(ComponentId.class, IntConfig.class); - } - } - - private static class C extends AbstractComponent { - public C(IntConfig intConfig, ComponentId id) { } - public C(String id, IntConfig intConfig) { } - static Constructor<C> preferred() throws NoSuchMethodException{ - return C.class.getConstructor(IntConfig.class, ComponentId.class); - } - } - - private static class E extends AbstractComponent { - public E(IntConfig intConfig) { } - public E(String id, String illegal, IntConfig intConfig, StringConfig stringConfig) { } - static Constructor<E> preferred() throws NoSuchMethodException{ - return E.class.getConstructor(IntConfig.class); - } - } - - private static class G extends AbstractComponent { - public G(ComponentId id) { } - public G(String id) { } - static Constructor<G> preferred() throws NoSuchMethodException{ - return G.class.getConstructor(ComponentId.class); - } - } - - private static class H extends AbstractComponent { - public H(ComponentId id, String illegal) { } - public H(String id, String illegal) { } - } - - private static class I extends AbstractComponent { - protected I(ComponentId id) { } - } - - private static class J extends AbstractComponent { - } - - private static class K extends AbstractComponent { - public K() { } - public K(ComponentId id, String illegal) { } - static Constructor<K> preferred() throws NoSuchMethodException{ - return K.class.getConstructor(); - } - } - - private static class L extends AbstractComponent { - public L(long l, long ll, long lll) { } - public L(ComponentId id, IntConfig intConfig) { } - static Constructor<L> preferred() throws NoSuchMethodException{ - return L.class.getConstructor(ComponentId.class, IntConfig.class); - } - } -} diff --git a/container-core/src/test/java/com/yahoo/container/handler/VipStatusHandlerTestCase.java b/container-core/src/test/java/com/yahoo/container/handler/VipStatusHandlerTestCase.java index f8de32ee3ff..de9289a1e9d 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/VipStatusHandlerTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/handler/VipStatusHandlerTestCase.java @@ -53,8 +53,7 @@ public class VipStatusHandlerTestCase { @Test public void testHandleRequest() { - VipStatusConfig config = new VipStatusConfig(new VipStatusConfig.Builder().accessdisk(false) - .noSearchBackendsImpliesOutOfService(false)); + VipStatusConfig config = new VipStatusConfig(new VipStatusConfig.Builder().accessdisk(false)); VipStatusHandler handler = new VipStatusHandler(Executors.newCachedThreadPool(), config, metric); MockResponseHandler responseHandler = new MockResponseHandler(); HttpRequest request = createRequest(); @@ -81,8 +80,7 @@ public class VipStatusHandlerTestCase { @Test public void testFileNotFound() { VipStatusConfig config = new VipStatusConfig(new VipStatusConfig.Builder().accessdisk(true) - .statusfile("/VipStatusHandlerTestCaseFileThatReallyReallyShouldNotExist") - .noSearchBackendsImpliesOutOfService(false)); + .statusfile("/VipStatusHandlerTestCaseFileThatReallyReallyShouldNotExist")); VipStatusHandler handler = new VipStatusHandler(Executors.newCachedThreadPool(), config, metric); NotFoundResponseHandler responseHandler = new NotFoundResponseHandler(); HttpRequest request = createRequest(); @@ -104,8 +102,10 @@ public class VipStatusHandlerTestCase { String OK = "OK\n"; writer.write(OK); writer.close(); - VipStatusConfig config = new VipStatusConfig(new VipStatusConfig.Builder().accessdisk(true) - .statusfile(statusFile.getAbsolutePath()).noSearchBackendsImpliesOutOfService(false)); + VipStatusConfig config = new VipStatusConfig( + new VipStatusConfig.Builder() + .accessdisk(true) + .statusfile(statusFile.getAbsolutePath())); VipStatusHandler handler = new VipStatusHandler(Executors.newCachedThreadPool(), config, metric); MockResponseHandler responseHandler = new MockResponseHandler(); HttpRequest request = createRequest(); @@ -123,8 +123,7 @@ public class VipStatusHandlerTestCase { @Test public void testExplicitlyRotationControl() { VipStatus vipStatus = new VipStatus(); - VipStatusConfig config = new VipStatusConfig(new VipStatusConfig.Builder().accessdisk(false) - .noSearchBackendsImpliesOutOfService(true)); + VipStatusConfig config = new VipStatusConfig(new VipStatusConfig.Builder().accessdisk(false)); VipStatusHandler handler = new VipStatusHandler(Executors.newCachedThreadPool(), config, metric, vipStatus); vipStatus.setInRotation(false); diff --git a/container-core/src/test/java/com/yahoo/container/xml/bind/JAXBContextFactoryTest.java b/container-core/src/test/java/com/yahoo/container/xml/bind/JAXBContextFactoryTest.java deleted file mode 100644 index 670b69516bf..00000000000 --- a/container-core/src/test/java/com/yahoo/container/xml/bind/JAXBContextFactoryTest.java +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.xml.bind; - -import org.junit.Test; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.fail; - -/** - * @author einarmr - * @author gjoranv - * @since 5.3 - */ -@SuppressWarnings("deprecation") -public class JAXBContextFactoryTest { - @Test - public void testInstantiationAndDestruction() { - - com.yahoo.container.xml.providers.JAXBContextFactoryProvider provider = new com.yahoo.container.xml.providers.JAXBContextFactoryProvider(); - JAXBContextFactory factory = provider.get(); - assertThat(factory.getClass().getName(), equalTo(com.yahoo.container.xml.providers.JAXBContextFactoryProvider.FACTORY_CLASS)); - - try { - JAXBContextFactory.getContextPath((Class) null); - fail("Should have failed with null classes."); - } catch (Exception e) { } - - try { - JAXBContextFactory.getContextPath(); - fail("Should have failed with empty list."); - } catch (Exception e) { } - - assertThat(JAXBContextFactory.getContextPath(this.getClass()), - equalTo(this.getClass().getPackage().getName())); - - assertThat(JAXBContextFactory.getContextPath(this.getClass(), - String.class), - equalTo(this.getClass().getPackage().getName() + ":" + - String.class.getPackage().getName())); - - provider.deconstruct(); - - } -} diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml index d568d603adc..70c572185dc 100644 --- a/container-dependencies-enforcer/pom.xml +++ b/container-dependencies-enforcer/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-dependencies-enforcer</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>pom</packaging> <dependencies> @@ -90,7 +90,6 @@ <include>com.sun.activation:javax.activation:[1.2.0]:jar:provided</include> <include>com.sun.xml.bind:jaxb-core:[${jaxb.version}]:jar:provided</include> <include>com.sun.xml.bind:jaxb-impl:[${jaxb.version}]:jar:provided</include> - <include>commons-codec:commons-codec:[1.4]:jar:provided</include> <include>commons-daemon:commons-daemon:[1.0.3]:jar:provided</include> <include>commons-logging:commons-logging:[1.1.1]:jar:provided</include> <include>javax.annotation:javax.annotation-api:[${javax.annotation-api.version}]:jar:provided</include> @@ -104,8 +103,6 @@ <include>org.apache.felix:org.apache.felix.framework:[${felix.version}]:jar:provided</include> <include>org.apache.felix:org.apache.felix.log:[1.0.1]:jar:provided</include> <include>org.apache.felix:org.apache.felix.main:[${felix.version}]:jar:provided</include> - <include>org.apache.httpcomponents:httpclient:[${apache.httpclient.version}]:jar:provided</include> - <include>org.apache.httpcomponents:httpcore:[${apache.httpcore.version}]:jar:provided</include> <include>org.bouncycastle:bcpkix-jdk15on:[${bouncycastle.version}]:jar:provided</include> <include>org.bouncycastle:bcprov-jdk15on:[${bouncycastle.version}]:jar:provided</include> <include>org.eclipse.jetty:jetty-http:[${jetty.version}]:jar:provided</include> diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index fe37c08a947..b6af723670a 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -5,7 +5,7 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>container-dependency-versions</artifactId> <packaging>pom</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>container-dependency-versions</name> <description>Versions for all 3rd party dependencies provided from the Vespa container.</description> <url>https://github.com/vespa-engine</url> @@ -144,11 +144,6 @@ <version>${guice.version}</version> <classifier>no_aop</classifier> </dependency> - <dependency> - <groupId>commons-codec</groupId> - <artifactId>commons-codec</artifactId> - <version>1.4</version> - </dependency> <dependency> <groupId>commons-daemon</groupId> <artifactId>commons-daemon</artifactId> @@ -186,24 +181,27 @@ <version>${javax.ws.rs-api.version}</version> </dependency> - <!-- TODO: upgrade jaxb artifacts to >=2.3.0. - See https://stackoverflow.com/questions/50237516/proper-fix-for-java-10-complaining-about-illegal-reflection-access-by-jaxb-impl --> + <!-- jaxb start --> <dependency> + <!-- Needed by felix --> <groupId>javax.xml.bind</groupId> <artifactId>jaxb-api</artifactId> <version>${jaxb.version}</version> </dependency> <dependency> + <!-- Needed by jersey --> <groupId>com.sun.xml.bind</groupId> <artifactId>jaxb-core</artifactId> <version>${jaxb.version}</version> </dependency> <dependency> + <!-- Needed by jersey --> <groupId>com.sun.xml.bind</groupId> <artifactId>jaxb-impl</artifactId> <version>${jaxb.version}</version> </dependency> <dependency> + <!-- Needed by jersey --> <groupId>com.sun.activation</groupId> <artifactId>javax.activation</artifactId> <version>1.2.0</version> @@ -236,16 +234,6 @@ <version>${felix.version}</version> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - <version>${apache.httpclient.version}</version> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - <version>${apache.httpcore.version}</version> - </dependency> - <dependency> <groupId>org.bouncycastle</groupId> <artifactId>bcpkix-jdk15on</artifactId> <version>${bouncycastle.version}</version> @@ -458,12 +446,10 @@ </dependencyManagement> <properties> - <apache.httpclient.version>4.4.1</apache.httpclient.version> - <apache.httpcore.version>4.4.1</apache.httpcore.version> <bouncycastle.version>1.58</bouncycastle.version> - <felix.version>5.6.10</felix.version> + <felix.version>6.0.1</felix.version> <findbugs.version>1.3.9</findbugs.version> - <guava.version>18.0</guava.version> + <guava.version>20.0</guava.version> <guice.version>3.0</guice.version> <jaxb.version>2.3.0</jaxb.version> <jetty.version>9.4.14.v20181114</jetty.version> diff --git a/container-dev/pom.xml b/container-dev/pom.xml index 7d61882c085..e3837c08701 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -9,11 +9,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-dev</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>jar</packaging> <dependencies> <dependency> @@ -100,10 +100,6 @@ <groupId>commons-lang</groupId> <artifactId>commons-lang</artifactId> </exclusion> - <exclusion> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </exclusion> </exclusions> </dependency> <dependency> @@ -151,6 +147,10 @@ <artifactId>icu4j</artifactId> </exclusion> <exclusion> + <groupId>commons-codec</groupId> + <artifactId>commons-codec</artifactId> + </exclusion> + <exclusion> <groupId>commons-collections</groupId> <artifactId>commons-collections</artifactId> </exclusion> @@ -183,6 +183,10 @@ <version>${project.version}</version> <exclusions> <exclusion> + <groupId>commons-codec</groupId> + <artifactId>commons-codec</artifactId> + </exclusion> + <exclusion> <groupId>commons-collections</groupId> <artifactId>commons-collections</artifactId> </exclusion> diff --git a/container-di/pom.xml b/container-di/pom.xml index 38c6bd55822..216f9e9628b 100644 --- a/container-di/pom.xml +++ b/container-di/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-di</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>jar</packaging> <dependencies> <dependency> diff --git a/container-disc/CMakeLists.txt b/container-disc/CMakeLists.txt index 92f5b303d41..36c8f7e7799 100644 --- a/container-disc/CMakeLists.txt +++ b/container-disc/CMakeLists.txt @@ -3,9 +3,7 @@ install_fat_java_artifact(container-disc) vespa_install_script(src/main/sh/vespa-start-container-daemon.sh vespa-start-container-daemon bin) -install_config_definition(src/main/resources/configdefinitions/container.jdisc.config.http-server.def) install_config_definition(src/main/resources/configdefinitions/jdisc-bindings.def container.jdisc.jdisc-bindings.def) install_config_definition(src/main/resources/configdefinitions/jersey-connection.def container.config.jersey.jersey-connection.def) install_config_definition(src/main/resources/configdefinitions/jersey-web-app-pool.def container.config.jersey.jersey-web-app-pool.def) -install_config_definition(src/main/resources/configdefinitions/metric-defaults.def container.jdisc.config.metric-defaults.def) install_config_definition(src/main/resources/configdefinitions/score-board.def jdisc.metrics.yamasconsumer.cloud.score-board.def) diff --git a/container-disc/pom.xml b/container-disc/pom.xml index b0637393fba..1af9c69318a 100644 --- a/container-disc/pom.xml +++ b/container-disc/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-disc</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> @@ -137,12 +137,6 @@ <scope>provided</scope> </dependency> <!-- end WARNING --> - <!-- Athenz --> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - <scope>compile</scope> - </dependency> </dependencies> <build> <plugins> @@ -165,6 +159,7 @@ <configuration> <discApplicationClass>com.yahoo.container.jdisc.ConfiguredApplication</discApplicationClass> <discPreInstallBundle> + <!-- Vespa bundles --> config-bundle-jar-with-dependencies.jar, configdefinitions-jar-with-dependencies.jar, container-jersey2-jar-with-dependencies.jar, @@ -182,6 +177,9 @@ defaults-jar-with-dependencies.jar, component-jar-with-dependencies.jar, zkfacade-jar-with-dependencies.jar, + <!-- Misc 3rd party bundles --> + bcpkix-jdk15on-${bouncycastle.version}.jar, + bcprov-jdk15on-${bouncycastle.version}.jar, <!-- Jersey 2 + Jackson 2 --> aopalliance-repackaged-${hk2.version}.jar, hk2-api-${hk2.version}.jar, diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index 1cb4e1d4555..eae121604e4 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -39,7 +39,6 @@ import com.yahoo.jrt.slobrok.api.SlobrokList; import com.yahoo.log.LogLevel; import com.yahoo.log.LogSetup; import com.yahoo.net.HostName; -import com.yahoo.osgi.OsgiImpl; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.yolean.Exceptions; @@ -115,7 +114,6 @@ public final class ConfiguredApplication implements Application { } - @SuppressWarnings("deprecation") // TODO: Remove when the Container line below is removed @Inject public ConfiguredApplication(ContainerActivator activator, OsgiFramework osgiFramework, @@ -127,7 +125,6 @@ public final class ConfiguredApplication implements Application { this.subscriberFactory = subscriberFactory; this.configId = System.getProperty("config.id"); this.restrictedOsgiFramework = new DisableOsgiFramework(new RestrictedBundleContext(osgiFramework.bundleContext())); - Container.get().setOsgi(new OsgiImpl(osgiFramework)); // TODO: Remove, not necessary applicationWithLegacySetup = new ContainerDiscApplication(configId); } @@ -192,7 +189,6 @@ public final class ConfiguredApplication implements Application { try { Container.get().setupFileAcquirer(config.filedistributor()); Container.get().setupUrlDownloader(); - com.yahoo.container.Server.get().initialize(config); } catch (Exception e) { log.log(LogLevel.ERROR, "Caught exception when initializing server. Exiting.", e); Runtime.getRuntime().halt(1); diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/ForwardingMetricConsumer.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/ForwardingMetricConsumer.java index 7bdd2d9cc7b..c9caaa9d4b5 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/ForwardingMetricConsumer.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/ForwardingMetricConsumer.java @@ -8,8 +8,8 @@ import com.yahoo.jdisc.application.MetricConsumer; import java.util.Map; /** - * <p>If more than one {@link MetricConsumerFactory} is registered in a container, calls to <tt>Metric</tt> need to be - * forwarded to all the underlying <tt>MetricConsumers</tt>. That is the responsibility of this class. Instances of this + * <p>If more than one {@link MetricConsumerFactory} is registered in a container, calls to <code>Metric</code> need to be + * forwarded to all the underlying <code>MetricConsumers</code>. That is the responsibility of this class. Instances of this * class is created by the {@link MetricConsumerProvider} in those cases.</p> * * @author Simon Thoresen Hult diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProvider.java index 1e7a4e75385..a300364d848 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProvider.java @@ -11,14 +11,14 @@ import com.yahoo.metrics.MetricsPresentationConfig; /** - * <p>The purpose of this class it to be the only provider for the <tt>MetricConsumer</tt> interface in a component + * <p>The purpose of this class it to be the only provider for the <code>MetricConsumer</code> interface in a component * graph. This component is automatically registered in the graph by the config server. Configuring a different - * <tt>MetricConsumer</tt> is done by registering one or more {@link MetricConsumerFactory} in the services-file.</p> + * <code>MetricConsumer</code> is done by registering one or more {@link MetricConsumerFactory} in the services-file.</p> * * <p>Because this class depends on the {@link ComponentRegistry} of {@link MetricConsumerFactory}, any added or removed * {@link MetricConsumerFactory} will cause this component to be reconfigured. Because {@link MetricProvider} depends on - * this class, which means any component that uses <tt>Metric</tt> will be reconfigured. Any component that depends - * directly on <tt>MetricConsumer</tt> will also be reconfigured.</p> + * this class, which means any component that uses <code>Metric</code> will be reconfigured. Any component that depends + * directly on <code>MetricConsumer</code> will also be reconfigured.</p> * * @author Simon Thoresen Hult */ diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricProvider.java index bcd75c0b36b..57613113e73 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricProvider.java @@ -6,9 +6,9 @@ import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.application.MetricConsumer; /** - * <p>This class implements a {@link Provider} component of <tt>Metric</tt>. Because this class depends on {@link + * <p>This class implements a {@link Provider} component of <code>Metric</code>. Because this class depends on {@link * MetricConsumerProvider}, any change to the consumer configuration will trigger reconfiguration of this component, - * which in turn triggers reconfiguration of any component that depends on <tt>Metric</tt>.</p> + * which in turn triggers reconfiguration of any component that depends on <code>Metric</code>.</p> * * @author Simon Thoresen Hult */ diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java index c2ef789e8fc..98787eaeeeb 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java @@ -24,9 +24,6 @@ import java.util.TimerTask; */ public class MetricUpdater extends AbstractComponent { - @Deprecated private static final String DEPRECATED_FREE_MEMORY_BYTES = "freeMemoryBytes"; - @Deprecated private static final String DEPRECATED_USED_MEMORY_BYTES = "usedMemoryBytes"; - @Deprecated private static final String DEPRECATED_TOTAL_MEMORY_BYTES = "totalMemoryBytes"; private static final String FREE_MEMORY_BYTES = "mem.heap.free"; private static final String USED_MEMORY_BYTES = "mem.heap.used"; private static final String TOTAL_MEMORY_BYTES = "mem.heap.total"; @@ -104,9 +101,6 @@ public class MetricUpdater extends AbstractComponent { long freeMemory = runtime.freeMemory(); long totalMemory = runtime.totalMemory(); long usedMemory = totalMemory - freeMemory; - metric.set(DEPRECATED_FREE_MEMORY_BYTES, freeMemory, null); - metric.set(DEPRECATED_USED_MEMORY_BYTES, usedMemory, null); - metric.set(DEPRECATED_TOTAL_MEMORY_BYTES, totalMemory, null); metric.set(FREE_MEMORY_BYTES, freeMemory, null); metric.set(USED_MEMORY_BYTES, usedMemory, null); metric.set(TOTAL_MEMORY_BYTES, totalMemory, null); diff --git a/container-disc/src/main/resources/configdefinitions/container.jdisc.config.http-server.def b/container-disc/src/main/resources/configdefinitions/container.jdisc.config.http-server.def deleted file mode 100644 index 9275136c585..00000000000 --- a/container-disc/src/main/resources/configdefinitions/container.jdisc.config.http-server.def +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=container.jdisc.config - -# TODO Vespa 7: Remove this config and its adapter in config-model (ConnectorFactory) - -serverName string default="default" -tcpKeepAliveEnabled bool default=false -httpKeepAliveEnabled bool default=true -optimizedForPipeline bool default=false -removeRawPostBodyForWwwUrlEncodedPost bool default=false -responseChunkingEnabled bool default=true -tcpNoDelayEnabled bool default=true -listenPort int -maxChunkSize int default=65536 -maxHeaderSize int default=65536 -maxInitialLineLength int default=65536 -maxUnchunkedResponseSize int default=10362880 -numWorkerThreads int default=16 -receiveBufferSize int default=-1 -responseCompressionLevel int default=6 -sendBufferSize int default=-1 -tcpListenBacklogLength int default=0 -connectTimeout double default=10 -idleConnectionTimeout double default=180 -soLinger double default=-1 -perRequestMetricsEnabled bool default=false -metricUpdateInterval double default=1 -ssl.enabled bool default=false -ssl.keyStoreType string default="JKS" -ssl.keyStorePath string default="default" -ssl.trustStorePath string default="default" -ssl.keyDBKey string default="" -ssl.algorithm string default="SunX509" -ssl.protocol string default="TLS" - -filter[].id string -filter[].binding string - -#copy pasted from new http-server.def(namespace=jdisc.http.server) -accessLog.bufferSize int default = 10240 -accessLog.flushTimeout double default = 5 # seconds -accessLog.fileName string default = "" -accessLog.rotatable bool default = false -accessLog.directory string default = "logs/jdisc_core" -accessLog.prefix string default = "access_log." -accessLog.suffix string default = "" -accessLog.fileDateFormat string default = "yyyy-MM-dd" diff --git a/container-disc/src/main/resources/configdefinitions/metric-defaults.def b/container-disc/src/main/resources/configdefinitions/metric-defaults.def deleted file mode 100644 index 013c2a8c77e..00000000000 --- a/container-disc/src/main/resources/configdefinitions/metric-defaults.def +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=container.jdisc.config - -factory enum { STATE_MONITOR, YAMAS_SCOREBOARD } default = STATE_MONITOR diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh index 21c9dc28022..65826e650c4 100755 --- a/container-disc/src/main/sh/vespa-start-container-daemon.sh +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -90,6 +90,12 @@ configure_memory() { fi } +configure_cpu() { + if ((jvm_availableProcessors > 0)); then + cpu_options="-XX:ActiveProcessorCount=${jvm_availableProcessors}" + fi +} + configure_numactl() { log_message debug "starting ${VESPA_SERVICE_NAME} for ${VESPA_CONFIG_ID}" if numactl --interleave all true &> /dev/null; then @@ -115,7 +121,7 @@ configure_numactl() { } configure_gcopts() { - consider_fallback jvm_gcopts "-XX:+UseConcMarkSweepGC -XX:MaxTenuringThreshold=15 -XX:NewRatio=1" + consider_fallback jvm_gcopts "-XX:MaxTenuringThreshold=15 -XX:NewRatio=1" if [ "$jvm_verbosegc" = "true" ]; then jvm_gcopts="${jvm_gcopts} -verbose:gc" fi @@ -171,101 +177,31 @@ import_cfg_var () { fi } -exec_jsvc () { - if [ "$jsvc_classpath_pre" ]; then - CP="${jsvc_classpath_pre}:${CP}" - fi - for jf in $jsvc_extra_classpath_libjars ; do - CP="${CP}:${VESPA_HOME}/lib/jars/$jf.jar" - done - for jf in $jsvc_extra_classpath_files ; do - CP="${CP}:jf" - done - - PRELOAD="$PRELOAD $jsvc_extra_preload" - configure_preload - exec $numactlcmd $envcmd $jsvc_binary_name \ - -Dconfig.id="${VESPA_CONFIG_ID}" \ - -XX:+PreserveFramePointer \ - ${jsvc_opts} \ - ${memory_options} \ - ${jvm_gcopts} \ - -XX:MaxJavaStackTraceDepth=1000000 \ - -XX:+HeapDumpOnOutOfMemoryError \ - -XX:HeapDumpPath="${VESPA_HOME}/var/crash" \ - -XX:+ExitOnOutOfMemoryError \ - -Djava.library.path="${VESPA_HOME}/lib64" \ - -Djava.awt.headless=true \ - -Djavax.net.ssl.keyStoreType=JKS \ - -Dsun.rmi.dgc.client.gcInterval=3600000 \ - -Dsun.net.client.defaultConnectTimeout=5000 -Dsun.net.client.defaultReadTimeout=60000 \ - -Djdisc.config.file="$cfpfile" \ - -Djdisc.export.packages=${jdisc_export_packages} \ - -Djdisc.cache.path="$bundlecachedir" \ - -Djdisc.debug.resources=false \ - -Djdisc.bundle.path="${VESPA_HOME}/lib/jars" \ - -Djdisc.logger.enabled=true \ - -Djdisc.logger.level=ALL \ - -Djdisc.logger.tag="${VESPA_CONFIG_ID}" \ - -Dorg.apache.commons.logging.Log=org.apache.commons.logging.impl.Jdk14Logger \ - -Dvespa.log.control.dir="${VESPA_LOG_CONTROL_DIR}" \ - -Dzookeeperlogfile="${ZOOKEEPER_LOG_FILE}" \ - -Dfile.encoding=UTF-8 \ - -cp "$CP" \ - "$@" \ - com.yahoo.jdisc.core.BootstrapDaemon file:${VESPA_HOME}/lib/jars/container-disc-jar-with-dependencies.jar -} - -maybe_use_jsvc () { - import_cfg_var use_jsvc "false" - - # if configured, run JSVC aka commons.daemon - if [ "$use_jsvc" = "true" ]; then - import_cfg_var jsvc_binary_name "jsvc" - import_cfg_var jsvc_extra_preload - - import_cfg_var jsvc_use_pidfile "false" - - import_cfg_var jsvc_classpath_pre - import_cfg_var jsvc_extra_classpath_libjars - import_cfg_var jsvc_extra_classpath_files - import_cfg_var jsvc_ipv6opts - import_cfg_var jsvc_extra_opts - import_cfg_var jsvc_normal_opts - import_cfg_var jsvc_java_home_opt - if [ "$jsvc_use_pidfile" = "true" ]; then - import_cfg_var jsvc_pidfile_opt "-pidfile ${VESPA_HOME}/var/run/jsvc.${VESPA_SERVICE_NAME}.pid" - else - import_cfg_var jsvc_pidfile_opt "" - fi - import_cfg_var jsvc_user_opt - import_cfg_var jsvc_agent_opt - import_cfg_var jsvc_ynet_opt - import_cfg_var jsvc_unknown_opts - - jsvc_opts="$jsvc_ipv6opts $jsvc_extra_opts $jsvc_normal_opts $jsvc_java_home_opt $jsvc_pidfile_opt $jsvc_user_opt $jsvc_agent_opt $jsvc_ynet_opt $jsvc_unknown_opts" - exec_jsvc - fi -} - getconfig configure_memory configure_gcopts configure_env_vars configure_classpath configure_numactl +configure_cpu configure_preload -maybe_use_jsvc exec $numactlcmd $envcmd java \ -Dconfig.id="${VESPA_CONFIG_ID}" \ -XX:+PreserveFramePointer \ + ${VESPA_CONTAINER_JVMARGS} \ + ${cpu_options} \ ${memory_options} \ ${jvm_gcopts} \ -XX:MaxJavaStackTraceDepth=1000000 \ -XX:+HeapDumpOnOutOfMemoryError \ -XX:HeapDumpPath="${VESPA_HOME}/var/crash" \ -XX:+ExitOnOutOfMemoryError \ + --illegal-access=warn \ + --add-opens=java.base/java.io=ALL-UNNAMED \ + --add-opens=java.base/java.lang=ALL-UNNAMED \ + --add-opens=java.base/java.net=ALL-UNNAMED \ + --add-opens=java.base/jdk.internal.loader=ALL-UNNAMED \ -Djava.library.path="${VESPA_HOME}/lib64" \ -Djava.awt.headless=true \ -Djavax.net.ssl.keyStoreType=JKS \ diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java index e9e04eab3b4..25d5b7b9886 100644 --- a/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java +++ b/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricUpdaterTest.java @@ -27,7 +27,7 @@ public class MetricUpdaterTest { ContainerWatchdogMetrics containerWatchdogMetrics = mock(ContainerWatchdogMetrics.class); new MetricUpdater(new MockScheduler(), metric, containerWatchdogMetrics); verify(containerWatchdogMetrics, times(1)).emitMetrics(any()); - verify(metric, times(8 + 2 * gcCount)).set(anyString(), any(), any()); + verify(metric, times(5 + 2 * gcCount)).set(anyString(), any(), any()); } private static class MockScheduler implements MetricUpdater.Scheduler { diff --git a/container-integration-test/pom.xml b/container-integration-test/pom.xml index 4b4d5de21eb..b03e9e3131d 100644 --- a/container-integration-test/pom.xml +++ b/container-integration-test/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-integration-test</artifactId> <packaging>jar</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -41,4 +41,4 @@ </plugin> </plugins> </build> -</project>
\ No newline at end of file +</project> diff --git a/container-jersey2/pom.xml b/container-jersey2/pom.xml index 738d167a79b..78801126484 100644 --- a/container-jersey2/pom.xml +++ b/container-jersey2/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-jersey2</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> diff --git a/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/ResourceOrProviderClassVisitor.java b/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/ResourceOrProviderClassVisitor.java index 7cb47ac6118..bf53cec670e 100644 --- a/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/ResourceOrProviderClassVisitor.java +++ b/container-jersey2/src/main/java/com/yahoo/container/servlet/jersey/ResourceOrProviderClassVisitor.java @@ -29,7 +29,7 @@ public class ResourceOrProviderClassVisitor extends ClassVisitor { private boolean isAnnotated = false; public ResourceOrProviderClassVisitor() { - super(Opcodes.ASM6); + super(Opcodes.ASM7); } public Optional<String> getJerseyClassName() { diff --git a/container-messagebus/pom.xml b/container-messagebus/pom.xml index 1daf1df6fcb..fc7546d32c0 100644 --- a/container-messagebus/pom.xml +++ b/container-messagebus/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-messagebus</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>jar</packaging> <dependencies> <dependency> diff --git a/container-search-and-docproc/pom.xml b/container-search-and-docproc/pom.xml index 9b4419ae2b6..c66c5f9e1fe 100644 --- a/container-search-and-docproc/pom.xml +++ b/container-search-and-docproc/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-search-and-docproc</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> @@ -211,14 +211,6 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> </dependencies> <build> <plugins> diff --git a/container-search-and-docproc/src/main/java/org/apache/commons/codec/binary/package-info.java b/container-search-and-docproc/src/main/java/org/apache/commons/codec/binary/package-info.java deleted file mode 100644 index 80f76e6b263..00000000000 --- a/container-search-and-docproc/src/main/java/org/apache/commons/codec/binary/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 1, minor = 4, micro = 0)) -package org.apache.commons.codec.binary; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; - diff --git a/container-search-and-docproc/src/main/java/org/apache/commons/codec/digest/package-info.java b/container-search-and-docproc/src/main/java/org/apache/commons/codec/digest/package-info.java deleted file mode 100644 index 117120c1641..00000000000 --- a/container-search-and-docproc/src/main/java/org/apache/commons/codec/digest/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 1, minor = 4, micro = 0)) -package org.apache.commons.codec.digest; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; - diff --git a/container-search-and-docproc/src/main/java/org/apache/commons/codec/language/package-info.java b/container-search-and-docproc/src/main/java/org/apache/commons/codec/language/package-info.java deleted file mode 100644 index de288618ad9..00000000000 --- a/container-search-and-docproc/src/main/java/org/apache/commons/codec/language/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 1, minor = 4, micro = 0)) -package org.apache.commons.codec.language; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; - diff --git a/container-search-and-docproc/src/main/java/org/apache/commons/codec/net/package-info.java b/container-search-and-docproc/src/main/java/org/apache/commons/codec/net/package-info.java deleted file mode 100644 index 49510c6feac..00000000000 --- a/container-search-and-docproc/src/main/java/org/apache/commons/codec/net/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 1, minor = 4, micro = 0)) -package org.apache.commons.codec.net; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; - diff --git a/container-search-and-docproc/src/main/java/org/apache/commons/codec/package-info.java b/container-search-and-docproc/src/main/java/org/apache/commons/codec/package-info.java deleted file mode 100644 index c698ba0233a..00000000000 --- a/container-search-and-docproc/src/main/java/org/apache/commons/codec/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 1, minor = 4, micro = 0)) -package org.apache.commons.codec; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; - diff --git a/container-search-and-docproc/src/main/java/org/apache/http/auth/package-info.java b/container-search-and-docproc/src/main/java/org/apache/http/auth/package-info.java deleted file mode 100644 index dbf6d408a2f..00000000000 --- a/container-search-and-docproc/src/main/java/org/apache/http/auth/package-info.java +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* - * Exported due to usage in HTTPSearcher API - */ -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 6)) -package org.apache.http.auth; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; diff --git a/container-search-and-docproc/src/main/java/org/apache/http/client/config/package-info.java b/container-search-and-docproc/src/main/java/org/apache/http/client/config/package-info.java deleted file mode 100644 index bc480dd1838..00000000000 --- a/container-search-and-docproc/src/main/java/org/apache/http/client/config/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 6)) -package org.apache.http.client.config; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; - diff --git a/container-search-and-docproc/src/main/java/org/apache/http/client/entity/package-info.java b/container-search-and-docproc/src/main/java/org/apache/http/client/entity/package-info.java deleted file mode 100644 index c3bb1fbbf3e..00000000000 --- a/container-search-and-docproc/src/main/java/org/apache/http/client/entity/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 6)) -package org.apache.http.client.entity; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; - diff --git a/container-search-and-docproc/src/main/java/org/apache/http/client/methods/package-info.java b/container-search-and-docproc/src/main/java/org/apache/http/client/methods/package-info.java deleted file mode 100644 index c938418fa71..00000000000 --- a/container-search-and-docproc/src/main/java/org/apache/http/client/methods/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 6)) -package org.apache.http.client.methods; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; - diff --git a/container-search-and-docproc/src/main/java/org/apache/http/client/package-info.java b/container-search-and-docproc/src/main/java/org/apache/http/client/package-info.java deleted file mode 100644 index 85e68ea27b6..00000000000 --- a/container-search-and-docproc/src/main/java/org/apache/http/client/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 6)) -package org.apache.http.client; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; - diff --git a/container-search-and-docproc/src/main/java/org/apache/http/conn/package-info.java b/container-search-and-docproc/src/main/java/org/apache/http/conn/package-info.java deleted file mode 100644 index c8e606d25ee..00000000000 --- a/container-search-and-docproc/src/main/java/org/apache/http/conn/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 6)) -package org.apache.http.conn; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; - diff --git a/container-search-and-docproc/src/main/java/org/apache/http/conn/params/package-info.java b/container-search-and-docproc/src/main/java/org/apache/http/conn/params/package-info.java deleted file mode 100644 index 9469bfc7333..00000000000 --- a/container-search-and-docproc/src/main/java/org/apache/http/conn/params/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 6)) -package org.apache.http.conn.params; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; - diff --git a/container-search-and-docproc/src/main/java/org/apache/http/conn/scheme/package-info.java b/container-search-and-docproc/src/main/java/org/apache/http/conn/scheme/package-info.java deleted file mode 100644 index bd54cb060a9..00000000000 --- a/container-search-and-docproc/src/main/java/org/apache/http/conn/scheme/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 6)) -package org.apache.http.conn.scheme; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; - diff --git a/container-search-and-docproc/src/main/java/org/apache/http/conn/ssl/package-info.java b/container-search-and-docproc/src/main/java/org/apache/http/conn/ssl/package-info.java deleted file mode 100644 index 2dd9254cdb0..00000000000 --- a/container-search-and-docproc/src/main/java/org/apache/http/conn/ssl/package-info.java +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 6)) -package org.apache.http.conn.ssl; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; diff --git a/container-search-and-docproc/src/main/java/org/apache/http/entity/package-info.java b/container-search-and-docproc/src/main/java/org/apache/http/entity/package-info.java deleted file mode 100644 index a843aa9ceb2..00000000000 --- a/container-search-and-docproc/src/main/java/org/apache/http/entity/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 3)) -package org.apache.http.entity; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; - diff --git a/container-search-and-docproc/src/main/java/org/apache/http/impl/client/package-info.java b/container-search-and-docproc/src/main/java/org/apache/http/impl/client/package-info.java deleted file mode 100644 index 89973cb6cc5..00000000000 --- a/container-search-and-docproc/src/main/java/org/apache/http/impl/client/package-info.java +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* - * Exported due to usage in HTTPSearcher API - */ -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 6)) -package org.apache.http.impl.client; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; diff --git a/container-search-and-docproc/src/main/java/org/apache/http/impl/conn/package-info.java b/container-search-and-docproc/src/main/java/org/apache/http/impl/conn/package-info.java deleted file mode 100644 index 4d167d5e627..00000000000 --- a/container-search-and-docproc/src/main/java/org/apache/http/impl/conn/package-info.java +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* - * Exported due to usage in HTTPSearcher API - */ -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 6)) -package org.apache.http.impl.conn; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; diff --git a/container-search-and-docproc/src/main/java/org/apache/http/message/package-info.java b/container-search-and-docproc/src/main/java/org/apache/http/message/package-info.java deleted file mode 100644 index 7430e71d0c7..00000000000 --- a/container-search-and-docproc/src/main/java/org/apache/http/message/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 3)) -package org.apache.http.message; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; - diff --git a/container-search-and-docproc/src/main/java/org/apache/http/package-info.java b/container-search-and-docproc/src/main/java/org/apache/http/package-info.java deleted file mode 100644 index 9dcfc75e5a4..00000000000 --- a/container-search-and-docproc/src/main/java/org/apache/http/package-info.java +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/* - * Exported due to usage in HTTPSearcher API - */ -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 3)) -package org.apache.http; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; diff --git a/container-search-and-docproc/src/main/java/org/apache/http/params/package-info.java b/container-search-and-docproc/src/main/java/org/apache/http/params/package-info.java deleted file mode 100644 index 9bfd6e48404..00000000000 --- a/container-search-and-docproc/src/main/java/org/apache/http/params/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 3)) -package org.apache.http.params; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; - diff --git a/container-search-and-docproc/src/main/java/org/apache/http/protocol/package-info.java b/container-search-and-docproc/src/main/java/org/apache/http/protocol/package-info.java deleted file mode 100644 index 7bb784f7197..00000000000 --- a/container-search-and-docproc/src/main/java/org/apache/http/protocol/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 3)) -package org.apache.http.protocol; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; - diff --git a/container-search-and-docproc/src/main/java/org/apache/http/util/package-info.java b/container-search-and-docproc/src/main/java/org/apache/http/util/package-info.java deleted file mode 100644 index 89f73ab1a12..00000000000 --- a/container-search-and-docproc/src/main/java/org/apache/http/util/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage(version = @Version(major = 4, minor = 3, micro = 3)) -package org.apache.http.util; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; - diff --git a/container-search-gui/pom.xml b/container-search-gui/pom.xml index 1804e029039..c51ce8595af 100644 --- a/container-search-gui/pom.xml +++ b/container-search-gui/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-search-gui</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>com.google.inject</groupId> diff --git a/container-search/CMakeLists.txt b/container-search/CMakeLists.txt index dcf6c3461d3..a6099fea63b 100644 --- a/container-search/CMakeLists.txt +++ b/container-search/CMakeLists.txt @@ -6,13 +6,10 @@ install_config_definition(src/main/resources/configdefinitions/federation.def se install_config_definition(src/main/resources/configdefinitions/fs4.def container.search.fs4.def) install_config_definition(src/main/resources/configdefinitions/index-info.def search.config.index-info.def) install_config_definition(src/main/resources/configdefinitions/keyvalue.def prelude.searcher.keyvalue.def) -install_config_definition(src/main/resources/configdefinitions/legacy-emulation.def container.search.legacy-emulation.def) install_config_definition(src/main/resources/configdefinitions/lowercasing.def search.querytransform.lowercasing.def) install_config_definition(src/main/resources/configdefinitions/measure-qps.def search.statistics.measure-qps.def) install_config_definition(src/main/resources/configdefinitions/page-templates.def search.pagetemplates.page-templates.def) install_config_definition(src/main/resources/configdefinitions/provider.def search.federation.provider.def) -install_config_definition(src/main/resources/configdefinitions/qr-binary-cache-region.def search.cache.qr-binary-cache-region.def) -install_config_definition(src/main/resources/configdefinitions/qr-binary-cache.def search.cache.qr-binary-cache.def) install_config_definition(src/main/resources/configdefinitions/qr-monitor.def prelude.cluster.qr-monitor.def) install_config_definition(src/main/resources/configdefinitions/qr-quotetable.def prelude.searcher.qr-quotetable.def) install_config_definition(src/main/resources/configdefinitions/qr-start.def search.config.qr-start.def) diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 829c86711ef..e37836a5f06 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -1653,347 +1653,6 @@ ], "fields": [] }, - "com.yahoo.prelude.templates.Context": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "abstract" - ], - "methods": [ - "public void setXmlEscape(boolean)", - "public boolean getXmlEscape()", - "public abstract java.lang.Object put(java.lang.String, java.lang.Object)", - "public abstract java.lang.Object get(java.lang.String)", - "public abstract java.lang.Object remove(java.lang.Object)", - "public void setBoldOpenTag(java.lang.String)", - "public void setBoldCloseTag(java.lang.String)", - "public void setSeparatorTag(java.lang.String)", - "protected java.lang.Object normalizeValue(java.lang.Object)", - "public java.lang.String getBoldOpenTag()", - "public java.lang.String getBoldCloseTag()", - "public java.lang.String getSeparatorTag()", - "public abstract java.util.Collection getKeys()", - "public boolean isUtf8Output()", - "public void setUtf8Output(boolean)" - ], - "fields": [] - }, - "com.yahoo.prelude.templates.DefaultTemplateSet": { - "superClass": "com.yahoo.prelude.templates.UserTemplate", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "protected void <init>(java.lang.String)", - "public void <init>()", - "public com.yahoo.text.XMLWriter wrapWriter(java.io.Writer)", - "public void header(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "public void footer(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "public void hit(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "public void hitFooter(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "public void error(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "public void noHits(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "protected static void renderCoverageAttributes(com.yahoo.search.result.Coverage, com.yahoo.text.XMLWriter)", - "protected void renderHitAttributes(com.yahoo.search.result.Hit, com.yahoo.text.XMLWriter)", - "protected void renderHitGroup(com.yahoo.search.result.HitGroup, com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "protected void renderGroup(com.yahoo.search.result.HitGroup, com.yahoo.text.XMLWriter)", - "protected void renderHitGroupOfTypeGroupHit(com.yahoo.prelude.templates.Context, com.yahoo.search.result.HitGroup, com.yahoo.text.XMLWriter)", - "protected void renderId(com.yahoo.net.URI, com.yahoo.text.XMLWriter)", - "protected void renderHitFields(com.yahoo.prelude.templates.Context, com.yahoo.search.result.Hit, com.yahoo.text.XMLWriter)", - "protected void renderField(com.yahoo.prelude.templates.Context, com.yahoo.search.result.Hit, java.util.Map$Entry, com.yahoo.text.XMLWriter)", - "protected void renderFieldContent(com.yahoo.prelude.templates.Context, com.yahoo.search.result.Hit, java.lang.String, com.yahoo.text.XMLWriter)", - "protected boolean shouldRenderField(com.yahoo.search.result.Hit, java.lang.String)", - "public bridge synthetic void noHits(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic void error(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic void hitFooter(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic void hit(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic void footer(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic void header(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic java.io.Writer wrapWriter(java.io.Writer)" - ], - "fields": [] - }, - "com.yahoo.prelude.templates.FormattingOptions": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void formatFieldAsAttribute(java.lang.String, java.lang.String)", - "public java.util.Set fieldsAsAttributes()", - "public java.lang.String getAttributeName(java.lang.String)", - "public void formatFieldWithSubtype(java.lang.String, java.lang.String, java.lang.String, java.lang.String)", - "public com.yahoo.prelude.templates.FormattingOptions$SubtypeField getSubtype(java.lang.String)", - "public void formatFieldWithSubtype(java.lang.String, java.lang.String, java.lang.String)", - "public com.yahoo.prelude.templates.FormattingOptions$SubtypeFieldWithPrefix getSubtypeWithPrefix(java.lang.String)", - "public boolean shouldRenderField(java.lang.String)", - "public void setFieldToRender(java.lang.String)", - "public void setFieldNotToRender(java.lang.String)" - ], - "fields": [ - "public static final java.lang.String DEFAULT_TYPE_ATTRIBUTE_NAME" - ] - }, - "com.yahoo.prelude.templates.GenericTemplateSet": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String, java.lang.String, java.lang.String)", - "public java.lang.String getName()", - "public java.lang.String getMimeType()", - "public java.lang.String getEncoding()", - "public final java.lang.String getRequestedEncoding(com.yahoo.search.Query)", - "public com.yahoo.prelude.templates.Template getTemplate(java.lang.String)", - "public void setTemplate(java.lang.String, com.yahoo.prelude.templates.Template)", - "public void setTemplateNotNull(java.lang.String, com.yahoo.prelude.templates.Template)", - "public void setHighlightTags(java.lang.String, java.lang.String, java.lang.String)", - "public java.lang.String getBoldOpenTag()", - "public java.lang.String getBoldCloseTag()", - "public java.lang.String getSeparatorTag()", - "public void setSummaryClass(java.lang.String)", - "public java.lang.String getSummaryClass()" - ], - "fields": [ - "public static final java.lang.String DEFAULT_MIMETYPE", - "public static final java.lang.String DEFAULT_ENCODING" - ] - }, - "com.yahoo.prelude.templates.HitContext": { - "superClass": "com.yahoo.prelude.templates.Context", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.search.result.Hit, com.yahoo.prelude.templates.Context)", - "public java.lang.Object put(java.lang.String, java.lang.Object)", - "public java.lang.Object get(java.lang.String)", - "public java.lang.Object remove(java.lang.Object)", - "public java.util.Collection getKeys()", - "public void setBoldOpenTag(java.lang.String)", - "public void setBoldCloseTag(java.lang.String)", - "public void setSeparatorTag(java.lang.String)", - "public java.lang.String getBoldOpenTag()", - "public java.lang.String getBoldCloseTag()", - "public java.lang.String getSeparatorTag()", - "public boolean isUtf8Output()", - "public void setUtf8Output(boolean)", - "public void setXmlEscape(boolean)", - "public boolean getXmlEscape()", - "protected java.lang.Object normalizeValue(java.lang.Object)" - ], - "fields": [] - }, - "com.yahoo.prelude.templates.LogExceptionUserTemplateDelegator": { - "superClass": "com.yahoo.prelude.templates.UserTemplate", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.prelude.templates.UserTemplate)", - "public com.yahoo.prelude.templates.Context createContext()", - "public java.io.Writer wrapWriter(java.io.Writer)", - "public boolean isDefaultTemplateSet()", - "public java.lang.String getSummaryClass()", - "public java.lang.String getBoldOpenTag()", - "public java.lang.String getBoldCloseTag()", - "public java.lang.String getSeparatorTag()", - "public void setSummaryClass(java.lang.String)", - "public void setHighlightTags(java.lang.String, java.lang.String, java.lang.String)", - "public java.lang.String getName()", - "public java.lang.String getMimeType()", - "public java.lang.String getEncoding()", - "public com.yahoo.prelude.templates.Template getTemplate(java.lang.String)", - "public void setTemplate(java.lang.String, com.yahoo.prelude.templates.Template)", - "public void setTemplateNotNull(java.lang.String, com.yahoo.prelude.templates.Template)", - "public void header(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void footer(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void hit(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void hitFooter(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void error(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void noHits(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void queryContext(com.yahoo.prelude.templates.Context, java.io.Writer)" - ], - "fields": [] - }, - "com.yahoo.prelude.templates.MapContext": { - "superClass": "com.yahoo.prelude.templates.Context", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public java.lang.Object get(java.lang.String)", - "public java.lang.Object put(java.lang.String, java.lang.Object)", - "public java.lang.Object remove(java.lang.Object)", - "public java.util.Collection getKeys()" - ], - "fields": [] - }, - "com.yahoo.prelude.templates.PageTemplateSet": { - "superClass": "com.yahoo.prelude.templates.TiledTemplateSet", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public com.yahoo.text.XMLWriter wrapWriter(java.io.Writer)", - "public void header(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "public void footer(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "protected void renderSection(com.yahoo.search.result.HitGroup, com.yahoo.text.XMLWriter)", - "public void hit(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "public void hitFooter(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "public java.lang.String toString()", - "public bridge synthetic void hitFooter(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic void hit(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic void footer(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic void header(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic java.io.Writer wrapWriter(java.io.Writer)" - ], - "fields": [] - }, - "com.yahoo.prelude.templates.SearchRendererAdaptor": { - "superClass": "com.yahoo.search.rendering.Renderer", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(com.yahoo.prelude.templates.UserTemplate)", - "public void init()", - "public static void callRender(java.io.OutputStream, com.yahoo.search.Result)", - "public java.lang.String getEncoding()", - "public java.lang.String getMimeType()", - "public java.lang.String getDefaultSummaryClass()", - "public void render(java.io.Writer, com.yahoo.search.Result)", - "public java.io.Writer wrapWriter(java.io.Writer)", - "public void beginResult(java.io.Writer, com.yahoo.search.Result)", - "public void endResult(java.io.Writer, com.yahoo.search.Result)", - "public void error(java.io.Writer, java.util.Collection)", - "public void emptyResult(java.io.Writer, com.yahoo.search.Result)", - "public void queryContext(java.io.Writer, com.yahoo.search.query.context.QueryContext)", - "public void renderHit(java.io.Writer, com.yahoo.search.result.Hit, int)", - "public static boolean simpleRenderDefaultErrorHit(java.io.Writer, com.yahoo.search.result.ErrorHit)", - "public static void renderMessageDefaultErrorHit(com.yahoo.text.XMLWriter, com.yahoo.processing.request.ErrorMessage)", - "public boolean simpleRenderErrorHit(java.io.Writer, com.yahoo.search.result.ErrorHit)", - "public static void rendererErrorHitMessageMessage(com.yahoo.text.XMLWriter, com.yahoo.search.result.ErrorHit, com.yahoo.processing.request.ErrorMessage)", - "public com.yahoo.prelude.templates.UserTemplate getAdaptee()" - ], - "fields": [] - }, - "com.yahoo.prelude.templates.Template": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "abstract" - ], - "methods": [ - "public void <init>()", - "public abstract void render(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public abstract java.lang.String getName()" - ], - "fields": [] - }, - "com.yahoo.prelude.templates.TemplateSet": { - "superClass": "com.yahoo.prelude.templates.UserTemplate", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String, java.lang.String, java.lang.String)", - "public static com.yahoo.prelude.templates.UserTemplate getDefault()", - "public com.yahoo.prelude.templates.Template getHeader(com.yahoo.search.Result)", - "public void setHeader(com.yahoo.prelude.templates.Template)", - "public com.yahoo.prelude.templates.Template getFooter(com.yahoo.search.Result)", - "public void setFooter(com.yahoo.prelude.templates.Template)", - "public com.yahoo.prelude.templates.Template getNohits(com.yahoo.search.Result)", - "public com.yahoo.prelude.templates.Template getQueryContext(com.yahoo.search.Result)", - "public void setQueryContext(com.yahoo.prelude.templates.Template)", - "public void setNohits(com.yahoo.prelude.templates.Template)", - "public com.yahoo.prelude.templates.Template getError(com.yahoo.search.Result)", - "public void setError(com.yahoo.prelude.templates.Template)", - "public com.yahoo.prelude.templates.Template getHit(com.yahoo.search.result.Hit)", - "public void setHit(com.yahoo.prelude.templates.Template)", - "public com.yahoo.prelude.templates.Template getHitFooter(com.yahoo.search.result.Hit)", - "public java.lang.String toString()", - "public void header(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void footer(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void hit(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void error(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void hitFooter(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void noHits(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void queryContext(com.yahoo.prelude.templates.Context, java.io.Writer)" - ], - "fields": [] - }, - "com.yahoo.prelude.templates.TiledTemplateSet": { - "superClass": "com.yahoo.prelude.templates.DefaultTemplateSet", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(java.lang.String)", - "public com.yahoo.text.XMLWriter wrapWriter(java.io.Writer)", - "public void header(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "protected void renderHitAttributes(com.yahoo.search.result.Hit, com.yahoo.text.XMLWriter)", - "protected void renderField(com.yahoo.prelude.templates.Context, com.yahoo.search.result.Hit, java.util.Map$Entry, com.yahoo.text.XMLWriter)", - "protected void renderHitFields(com.yahoo.prelude.templates.Context, com.yahoo.search.result.Hit, com.yahoo.text.XMLWriter)", - "protected boolean shouldRenderField(com.yahoo.search.result.Hit, java.lang.String)", - "public void hit(com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "protected void renderHitGroup(com.yahoo.search.result.HitGroup, com.yahoo.prelude.templates.Context, com.yahoo.text.XMLWriter)", - "protected void renderSection(com.yahoo.search.result.HitGroup, com.yahoo.text.XMLWriter)", - "protected void renderSectionContent(com.yahoo.search.result.HitGroup, com.yahoo.text.XMLWriter)", - "protected void renderProvider(com.yahoo.prelude.templates.Context, com.yahoo.search.result.Hit, com.yahoo.text.XMLWriter)", - "protected void renderProviderHitAttributes(com.yahoo.search.result.Hit, com.yahoo.text.XMLWriter)", - "protected void renderProviderHitFields(com.yahoo.prelude.templates.Context, com.yahoo.search.result.Hit, com.yahoo.text.XMLWriter)", - "protected void renderProviderField(com.yahoo.prelude.templates.Context, com.yahoo.search.result.Hit, java.util.Map$Entry, com.yahoo.text.XMLWriter)", - "public java.lang.String toString()", - "public bridge synthetic void hit(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic void header(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public bridge synthetic java.io.Writer wrapWriter(java.io.Writer)" - ], - "fields": [] - }, - "com.yahoo.prelude.templates.UserTemplate": { - "superClass": "com.yahoo.prelude.templates.GenericTemplateSet", - "interfaces": [], - "attributes": [ - "public", - "abstract" - ], - "methods": [ - "public void <init>(java.lang.String, java.lang.String, java.lang.String)", - "public void <init>(java.lang.String)", - "public java.io.Writer wrapWriter(java.io.Writer)", - "public com.yahoo.prelude.templates.Context createContext()", - "public boolean isDefaultTemplateSet()", - "public abstract void header(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public abstract void footer(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public abstract void hit(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public abstract void hitFooter(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public abstract void error(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public abstract void noHits(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public void queryContext(com.yahoo.prelude.templates.Context, java.io.Writer)", - "public static void dumpAndXMLQuoteUTF8(com.yahoo.io.ByteWriter, byte[])", - "public static boolean dumpBytes(com.yahoo.io.ByteWriter, com.yahoo.prelude.fastsearch.FastHit, java.lang.String)" - ], - "fields": [] - }, "com.yahoo.search.Query$Type": { "superClass": "java.lang.Enum", "interfaces": [], @@ -2044,7 +1703,6 @@ "public long getStartTime()", "public long getDurationTime()", "public long getTimeLeft()", - "public boolean requestHasProperty(java.lang.String)", "public long getTimeout()", "public void setTimeout(long)", "public void setTimeout(java.lang.String)", @@ -2061,8 +1719,6 @@ "public void setHits(int)", "public void setOffset(int)", "public void setWindow(int, int)", - "public void setCompress(boolean)", - "public boolean getCompress()", "public java.lang.String toString()", "public java.lang.String toDetailString()", "public int encode(java.nio.ByteBuffer)", @@ -2081,7 +1737,6 @@ "public com.yahoo.search.query.Ranking getRanking()", "public com.yahoo.search.query.Model getModel()", "public com.yahoo.container.jdisc.HttpRequest getHttpRequest()", - "public com.yahoo.search.query.SessionId getSessionId(boolean)", "public com.yahoo.search.query.SessionId getSessionId()", "public com.yahoo.search.query.SessionId getSessionId(java.lang.String)", "public boolean hasEncodableProperties()", @@ -2146,7 +1801,6 @@ "public void setCoverage(com.yahoo.search.result.Coverage)", "public com.yahoo.search.result.Coverage getCoverage(boolean)", "public com.yahoo.collections.ListMap getHeaders(boolean)", - "public com.yahoo.search.result.Templating getTemplating()", "public bridge synthetic com.yahoo.component.provider.ListenableFreezableClass clone()", "public bridge synthetic com.yahoo.component.provider.FreezableClass clone()", "public bridge synthetic java.lang.Object clone()" @@ -2209,7 +1863,6 @@ "public" ], "methods": [ - "public void <init>(com.yahoo.search.cluster.NodeManager, java.lang.String)", "public void <init>(com.yahoo.search.cluster.NodeManager)", "public com.yahoo.search.cluster.MonitorConfiguration getConfiguration()", "public void add(java.lang.Object, boolean)", @@ -2445,10 +2098,8 @@ "public java.util.TimeZone getTimeZone()", "public com.yahoo.search.grouping.GroupingRequest setTimeZone(java.util.TimeZone)", "public com.yahoo.search.grouping.result.RootGroup getResultGroup(com.yahoo.search.Result)", - "public com.yahoo.search.grouping.GroupingRequest setResultGroup(com.yahoo.search.grouping.result.RootGroup)", "public java.util.List continuations()", "public static com.yahoo.search.grouping.GroupingRequest newInstance(com.yahoo.search.Query)", - "public static java.util.List getRequests(com.yahoo.search.Query)", "public java.lang.String toString()" ], "fields": [] @@ -4138,19 +3789,6 @@ ], "fields": [] }, - "com.yahoo.search.grouping.request.YmumValue": { - "superClass": "com.yahoo.search.grouping.request.DocumentValue", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public com.yahoo.search.grouping.request.YmumValue copy()", - "public bridge synthetic com.yahoo.search.grouping.request.GroupingExpression copy()" - ], - "fields": [] - }, "com.yahoo.search.grouping.request.ZCurveXFunction": { "superClass": "com.yahoo.search.grouping.request.FunctionNode", "interfaces": [], @@ -4449,8 +4087,6 @@ ], "methods": [ "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.statistics.Statistics, com.yahoo.language.Linguistics, com.yahoo.jdisc.Metric, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.config.QueryProfilesConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.container.core.ContainerHttpConfig)", - "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.statistics.Statistics, com.yahoo.language.Linguistics, com.yahoo.jdisc.Metric, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.config.QueryProfilesConfig, com.yahoo.component.provider.ComponentRegistry)", - "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.container.core.QrTemplatesConfig, com.yahoo.container.protect.FreezeDetector, com.yahoo.statistics.Statistics, com.yahoo.language.Linguistics, com.yahoo.jdisc.Metric, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.config.QueryProfilesConfig, com.yahoo.component.provider.ComponentRegistry)", "protected void destroy()", "public final com.yahoo.container.jdisc.HttpResponse handle(com.yahoo.container.jdisc.HttpRequest)", "public com.yahoo.search.Result searchAndFill(com.yahoo.search.Query, com.yahoo.component.chain.Chain, com.yahoo.search.searchchain.SearchChainRegistry)", @@ -5249,7 +4885,6 @@ "methods": [ "public static com.yahoo.search.query.profile.types.QueryProfileType getArgumentType()", "public void <init>(com.yahoo.search.Query)", - "public void traceLanguage()", "public com.yahoo.language.Language getParsingLanguage()", "public com.yahoo.language.Language getParsingLanguage(java.lang.String)", "public com.yahoo.language.Language getLanguage()", @@ -5289,9 +4924,7 @@ "public com.yahoo.search.searchchain.Execution getExecution()", "public static com.yahoo.search.query.Model getFrom(com.yahoo.search.Query)", "public java.lang.String toString()", - "public void prepare(com.yahoo.search.query.Ranking)", - "public void setYTraceHeaderToNext(java.lang.String)", - "public java.lang.String getYTraceHeaderToNext()" + "public void prepare(com.yahoo.search.query.Ranking)" ], "fields": [ "public static final java.lang.String MODEL", @@ -5337,8 +4970,6 @@ "public void setSummary(java.lang.String)", "public boolean getBolding()", "public void setBolding(boolean)", - "public boolean getReportCoverage()", - "public void setReportCoverage(boolean)", "public com.yahoo.component.ComponentSpecification getRenderer()", "public void setRenderer(com.yahoo.component.ComponentSpecification)", "public java.lang.String getFormat()", @@ -5357,7 +4988,6 @@ "public static final java.lang.String BOLDING", "public static final java.lang.String TIMING", "public static final java.lang.String SUMMARY", - "public static final java.lang.String REPORT_COVERAGE", "public static final java.lang.String SUMMARY_FIELDS", "public static final java.lang.String FORMAT" ] @@ -5694,7 +5324,6 @@ ], "methods": [ "public java.lang.String toString()", - "public static com.yahoo.search.query.UniqueRequestId next()", "public static com.yahoo.search.query.UniqueRequestId next(java.lang.String)" ], "fields": [] @@ -6587,9 +6216,7 @@ "public java.util.Map listProperties(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)", "public final com.yahoo.search.Query getParentQuery()" ], - "fields": [ - "public static final com.yahoo.processing.request.CompoundName[] PER_SOURCE_QUERY_PROPERTIES" - ] + "fields": [] }, "com.yahoo.search.query.properties.QueryPropertyAliases": { "superClass": "com.yahoo.search.query.properties.PropertyAliases", @@ -6830,7 +6457,7 @@ "public static final java.lang.String ENABLE", "public static final java.lang.String FACTOR", "public static final java.lang.String TAILCOST", - "public java.lang.Boolean enable" + "public static final com.yahoo.processing.request.CompoundName enableProperty" ] }, "com.yahoo.search.query.rewrite.QueryRewriteSearcher": { @@ -7110,50 +6737,6 @@ ], "fields": [] }, - "com.yahoo.search.rendering.DefaultRenderer$RenderingVisitor": { - "superClass": "com.yahoo.yolean.trace.TraceVisitor", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(com.yahoo.text.XMLWriter, long)", - "public void entering(com.yahoo.yolean.trace.TraceNode)", - "public void leaving(com.yahoo.yolean.trace.TraceNode)", - "public void visit(com.yahoo.yolean.trace.TraceNode)" - ], - "fields": [] - }, - "com.yahoo.search.rendering.DefaultRenderer": { - "superClass": "com.yahoo.processing.rendering.AsynchronousSectionedRenderer", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>()", - "public void <init>(java.util.concurrent.Executor)", - "public void init()", - "public java.lang.String getEncoding()", - "public java.lang.String getMimeType()", - "public void error(com.yahoo.text.XMLWriter, com.yahoo.search.Result)", - "public void queryContext(com.yahoo.text.XMLWriter, com.yahoo.search.query.context.QueryContext, com.yahoo.search.Query)", - "public static boolean simpleRenderDefaultErrorHit(com.yahoo.text.XMLWriter, com.yahoo.search.result.ErrorHit)", - "public static void renderMessageDefaultErrorHit(com.yahoo.text.XMLWriter, com.yahoo.search.result.ErrorMessage)", - "public void beginResponse(java.io.OutputStream)", - "public final java.lang.String getRequestedEncoding(com.yahoo.search.Query)", - "public void beginList(com.yahoo.processing.response.DataList)", - "public void data(com.yahoo.processing.response.Data)", - "public void endList(com.yahoo.processing.response.DataList)", - "public void endResponse()" - ], - "fields": [ - "public static final java.lang.String DEFAULT_MIMETYPE", - "public static final java.lang.String DEFAULT_ENCODING" - ] - }, "com.yahoo.search.rendering.JsonRenderer": { "superClass": "com.yahoo.processing.rendering.AsynchronousSectionedRenderer", "interfaces": [], @@ -7216,6 +6799,7 @@ ], "fields": [ "public static final com.yahoo.component.ComponentId xmlRendererId", + "public static final com.yahoo.component.ComponentId pageRendererId", "public static final com.yahoo.component.ComponentId jsonRendererId", "public static final com.yahoo.component.ComponentId defaultRendererId" ] @@ -7361,8 +6945,6 @@ "public void <init>(long, long)", "public void <init>(long, long, int)", "public void <init>(long, long, int, int)", - "public void <init>(long, int, boolean)", - "public void <init>(long, int, boolean, int)", "public com.yahoo.search.result.Coverage setSoonActive(long)", "public com.yahoo.search.result.Coverage setDegradedReason(int)", "public com.yahoo.search.result.Coverage setNodesTried(int)", @@ -7400,7 +6982,6 @@ "public void <init>(java.lang.String, com.yahoo.search.result.ErrorMessage)", "public void <init>(java.lang.String, java.util.List)", "public void setSource(java.lang.String)", - "public com.yahoo.search.result.ErrorMessage getMainError()", "public void addError(com.yahoo.search.result.ErrorMessage)", "public void addErrors(com.yahoo.search.result.ErrorHit)", "public java.util.Iterator errorIterator()", @@ -7428,7 +7009,6 @@ ], "methods": [ "public abstract void setSource(java.lang.String)", - "public abstract com.yahoo.search.result.ErrorMessage getMainError()", "public abstract void addError(com.yahoo.search.result.ErrorMessage)", "public abstract void addErrors(com.yahoo.search.result.ErrorHit)", "public abstract java.util.Iterator errorIterator()", @@ -7587,30 +7167,17 @@ "public java.lang.Object removeField(java.lang.String)", "protected boolean hasField(java.lang.String)", "protected boolean hasFields()", - "public void changeFieldKey(java.lang.String, java.lang.String)", - "public com.yahoo.prelude.hitfield.HitField buildHitField(java.lang.String)", "public com.yahoo.prelude.hitfield.HitField buildHitField(java.lang.String, boolean)", - "public com.yahoo.prelude.hitfield.HitField buildHitField(java.lang.String, boolean, boolean)", "public java.util.Set types()", - "public java.lang.String getTypeString()", - "public int getAddNumber()", - "public void setAddNumber(int)", "public boolean isMeta()", "public void setMeta(boolean)", "public boolean isAuxiliary()", "public void setAuxiliary(boolean)", - "public int getSourceNumber()", - "public void setSourceNumber(int)", "public com.yahoo.search.Query getQuery()", "public com.yahoo.processing.Request request()", "public final void setQuery(com.yahoo.search.Query)", - "public java.lang.String getFieldXML(java.lang.String)", - "public java.lang.String getUnboldedField(java.lang.String, boolean)", "public void setSearcherSpecificMetaData(com.yahoo.search.Searcher, java.lang.Object)", "public java.lang.Object getSearcherSpecificMetaData(com.yahoo.search.Searcher)", - "protected final void setFilledInternal(java.util.Set)", - "protected final java.util.Set getFilledInternal()", - "public static java.lang.String stripCharacter(char, java.lang.String)", "protected void close()", "public boolean equals(java.lang.Object)", "public int hashCode()", @@ -7662,7 +7229,6 @@ "public com.yahoo.search.result.Hit remove(java.lang.String)", "public com.yahoo.search.result.Hit remove(com.yahoo.net.URI)", "public com.yahoo.search.result.Hit remove(int)", - "public void setError(com.yahoo.search.result.ErrorMessage)", "public void addError(com.yahoo.search.result.ErrorMessage)", "public com.yahoo.search.result.ErrorHit getErrorHit()", "public com.yahoo.search.result.DefaultErrorHit removeErrorHit()", @@ -7853,31 +7419,6 @@ ], "fields": [] }, - "com.yahoo.search.result.Templating": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.search.Result)", - "public int getFirstHitNo()", - "public long getNextFirstHitNo()", - "public long getNextLastHitNo()", - "public int getLastHitNo()", - "public int getPrevFirstHitNo()", - "public int getPrevLastHitNo()", - "public java.lang.String getNextResultURL()", - "public java.lang.String getPreviousResultURL()", - "public java.lang.String getCurrentResultURL()", - "public com.yahoo.prelude.templates.UserTemplate getTemplates()", - "public void setTemplates(com.yahoo.prelude.templates.UserTemplate)", - "public com.yahoo.processing.rendering.Renderer getRenderer()", - "public void setRenderer(com.yahoo.processing.rendering.Renderer)", - "public boolean usesDefaultTemplate()" - ], - "fields": [] - }, "com.yahoo.search.searchchain.AsyncExecution": { "superClass": "java.lang.Object", "interfaces": [], diff --git a/container-search/pom.xml b/container-search/pom.xml index 13e05bb1e81..35b1477f478 100644 --- a/container-search/pom.xml +++ b/container-search/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-search</artifactId> <packaging>jar</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>junit</groupId> @@ -21,6 +21,11 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.hamcrest</groupId> <artifactId>hamcrest-library</artifactId> <scope>test</scope> @@ -107,14 +112,6 @@ </exclusions> </dependency> <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> <groupId>commons-codec</groupId> <artifactId>commons-codec</artifactId> </dependency> diff --git a/container-search/src/main/java/com/yahoo/fs4/GetDocSumsPacket.java b/container-search/src/main/java/com/yahoo/fs4/GetDocSumsPacket.java index a3b8e4f29a2..7dabb52a233 100644 --- a/container-search/src/main/java/com/yahoo/fs4/GetDocSumsPacket.java +++ b/container-search/src/main/java/com/yahoo/fs4/GetDocSumsPacket.java @@ -76,7 +76,7 @@ public class GetDocSumsPacket extends Packet { boolean useQueryCache = query.getRanking().getQueryCache(); // If feature cache is used we need to include the sessionId as key. if (useQueryCache) { // TODO: Move this decision (and the key) to ranking - query.getRanking().getProperties().put(sessionIdKey, query.getSessionId(false).toString()); + query.getRanking().getProperties().put(sessionIdKey, query.getSessionId().toString()); } // set the default features diff --git a/container-search/src/main/java/com/yahoo/prelude/Index.java b/container-search/src/main/java/com/yahoo/prelude/Index.java index 109ecfd29f8..8433939090b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Index.java +++ b/container-search/src/main/java/com/yahoo/prelude/Index.java @@ -53,7 +53,6 @@ public class Index { private boolean uriIndex = false; private boolean hostIndex = false; private StemMode stemMode = StemMode.NONE; - private Attribute[] matchGroup = null; private boolean isAttribute = false; private boolean isDefaultPosition = false; private boolean dynamicSummary=false; @@ -165,8 +164,6 @@ public class Index { setAttribute(true); } else if (commandString.equals("default-position")) { setDefaultPosition(true); - } else if (commandString.startsWith("match-group ")) { - setMatchGroup(commandString.substring(12).split(" ")); } else if (commandString.equals("plain-tokens")) { setPlainTokens(true); } else if (commandString.equals("multivalue")) { @@ -269,19 +266,6 @@ public class Index { return "(null)".equals(name); } - public Attribute[] getMatchGroup() { // TODO: Not in use on Vespa 6 - return matchGroup; - } - - public void setMatchGroup(String[] attributes) { - Attribute[] a = new Attribute[attributes.length]; - - for (int i = 0; i < attributes.length; i++) { - a[i] = new Attribute(attributes[i].trim()); - } - this.matchGroup = a; - } - public boolean isAttribute() { return isAttribute; } diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java index 448d8e7855f..2fcd2466dd8 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java @@ -44,12 +44,8 @@ public class IndexFacts { /** * The name of the default search definition, which is the union of all * known document types. - * - * @deprecated do not use */ - // TODO: Make this package private in Vespa 7 - @Deprecated // OK - public static final String unionName = "unionOfAllKnown"; + static final String unionName = "unionOfAllKnown"; /** A search definition which contains the union of all settings. */ @SuppressWarnings("deprecation") @@ -106,31 +102,6 @@ public class IndexFacts { return clusters != null ? clusters : Collections.<String>emptyList(); } - /** - * Public only for testing. - * - * @deprecated set at creation time - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void setClusters(Map<String, List<String>> clusters) { - ensureNotFrozen(); - this.clusters = clusters; - clusterByDocument = invert(clusters); - } - - /** - * @deprecated set indexes at creation time instead - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void setSearchDefinitions(Map<String, SearchDefinition> searchDefinitions, - SearchDefinition unionSearchDefinition) { - ensureNotFrozen(); - this.searchDefinitions = searchDefinitions; - this.unionSearchDefinition = unionSearchDefinition; - } - private boolean isInitialized() { return searchDefinitions.size() > 0; } @@ -312,58 +283,6 @@ public class IndexFacts { } } - - /** - * Add a string to be accepted as an index name when parsing a - * query. - * - * For testing only. - * - * @param sdName name of search definition containing index, if null, modify default set - * @param indexName name of index, actual or otherwise - * @deprecated set indexes at creation time instead - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void addIndex(String sdName, String indexName) { - ensureNotFrozen(); - - SearchDefinition sd; - if (sdName == null) { - sd = unionSearchDefinition; - } else if (searchDefinitions.containsKey(sdName)) { - sd = searchDefinitions.get(sdName); - } else { - sd = new SearchDefinition(sdName); - searchDefinitions.put(sdName, sd); - } - sd.getOrCreateIndex(indexName); - unionSearchDefinition.getOrCreateIndex(indexName); - } - - /** - * Adds an index to the specified index, and the default index settings, - * overriding any current settings for this index - * @deprecated set indexes at creation time instead - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void addIndex(String sdName, Index index) { - ensureNotFrozen(); - - SearchDefinition sd; - if (sdName == null) { - sd = unionSearchDefinition; - } else if (searchDefinitions.containsKey(sdName)) { - sd = searchDefinitions.get(sdName); - } else { - sd = new SearchDefinition(sdName); - searchDefinitions.put(sdName, sd); - } - sd.addIndex(index); - unionSearchDefinition.addIndex(index); - } - public String getDefaultPosition(String sdName) { SearchDefinition sd; if (sdName == null) { diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java index d2950078868..dd2bd1ee2f1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java @@ -39,8 +39,10 @@ public final class IndexModel { /** * Use IndexModel as a pure wrapper for the parameters given. + * + * @deprecated use the constructor without the third parameter */ - // TODO: Deprecate on Vespa 7 and remove on Vespa 8 + @Deprecated // TODO: Remove Vespa 8 public IndexModel(Map<String, List<String>> masterClusters, Map<String, SearchDefinition> searchDefinitions, SearchDefinition unionSearchDefinition) { @@ -90,7 +92,6 @@ public final class IndexModel { IndexInfoConfig.Indexinfo.Command command = j.next(); sd.addCommand(command.indexname(),command.command()); } - sd.fillMatchGroups(); searchDefinitions.put(info.name(), sd); } @@ -127,7 +128,6 @@ public final class IndexModel { } } - union.fillMatchGroups(); return union; } @@ -135,7 +135,8 @@ public final class IndexModel { public Map<String, SearchDefinition> getSearchDefinitions() { return searchDefinitions; } - // TODO: Deprecate on Vespa 7 and make package scope on Vespa 8 + /** @deprecated do not use */ + @Deprecated // TODO: Remove on Vespa 8 public SearchDefinition getUnionSearchDefinition() { return unionSearchDefinition; } } diff --git a/container-search/src/main/java/com/yahoo/prelude/Pong.java b/container-search/src/main/java/com/yahoo/prelude/Pong.java index ba3ff2eda00..cef64c293af 100644 --- a/container-search/src/main/java/com/yahoo/prelude/Pong.java +++ b/container-search/src/main/java/com/yahoo/prelude/Pong.java @@ -46,7 +46,7 @@ public class Pong { public int getErrorSize() { return errors.size(); } - + /** Returns the number of active documents in the backend responding in this Pong, if available */ public Optional<Long> activeDocuments() { if ( ! pongPacket.isPresent()) return Optional.empty(); diff --git a/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java b/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java index aa76bd5912a..7859a9698d9 100644 --- a/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java +++ b/container-search/src/main/java/com/yahoo/prelude/SearchDefinition.java @@ -96,19 +96,4 @@ public class SearchDefinition { } } - public void fillMatchGroups() { - for (Index i : indices.values()) { - Attribute[] matchGroup = i.getMatchGroup(); - if (matchGroup == null) { - continue; - } - for (Attribute a : matchGroup) { - Index m = getIndex(a.name); - if (m != null) { - a.setTokenizedContent(!m.isAttribute()); - } - } - } - } - } diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index fd150f975c3..92b6eef906e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -11,7 +11,6 @@ import com.yahoo.concurrent.Receiver.MessageState; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.handler.VipStatus; import com.yahoo.fs4.mplex.Backend; -import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.net.HostName; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.prelude.fastsearch.FS4ResourcePool; @@ -100,7 +99,6 @@ public class ClusterSearcher extends Searcher { QrSearchersConfig qrsConfig, ClusterConfig clusterConfig, DocumentdbInfoConfig documentDbConfig, - LegacyEmulationConfig emulationConfig, QrMonitorConfig monitorConfig, DispatchConfig dispatchConfig, ClusterInfoConfig clusterInfoConfig, @@ -133,7 +131,7 @@ public class ClusterSearcher extends Searcher { SummaryParameters docSumParams = new SummaryParameters(qrsConfig .com().yahoo().prelude().fastsearch().FastSearcher().docsum() .defaultclass()); - + for (DocumentdbInfoConfig.Documentdb docDb : documentDbConfig.documentdb()) { String docTypeName = docDb.name(); documentTypes.add(docTypeName); @@ -146,7 +144,7 @@ public class ClusterSearcher extends Searcher { if (searchClusterConfig.indexingmode() == STREAMING) { VdsStreamingSearcher searcher = vdsCluster(fs4ResourcePool.getServerId(), searchClusterIndex, - searchClusterConfig, cacheParams, emulationConfig, docSumParams, + searchClusterConfig, cacheParams, docSumParams, documentDbConfig); addBackendSearcher(searcher); } else { @@ -156,7 +154,7 @@ public class ClusterSearcher extends Searcher { if ( ! isRemote(searchClusterConfig.dispatcher(dispatcherIndex).host())) { Backend dispatchBackend = createBackend(searchClusterConfig.dispatcher(dispatcherIndex)); FastSearcher searcher = searchDispatch(searchClusterIndex, fs4ResourcePool, - cacheParams, emulationConfig, docSumParams, + cacheParams, docSumParams, documentDbConfig, dispatchBackend, dispatcher, dispatcherIndex); addBackendSearcher(searcher); } @@ -192,22 +190,21 @@ public class ClusterSearcher extends Searcher { } private static ClusterParams makeClusterParams(int searchclusterIndex, - LegacyEmulationConfig emulConfig, int dispatchIndex) { - return new ClusterParams("sc" + searchclusterIndex + ".num" + dispatchIndex, emulConfig); + return new ClusterParams("sc" + searchclusterIndex + ".num" + dispatchIndex); } private static FastSearcher searchDispatch(int searchclusterIndex, FS4ResourcePool fs4ResourcePool, CacheParams cacheParams, - LegacyEmulationConfig emulConfig, SummaryParameters docSumParams, DocumentdbInfoConfig documentdbInfoConfig, Backend backend, Dispatcher dispatcher, int dispatcherIndex) { - ClusterParams clusterParams = makeClusterParams(searchclusterIndex, emulConfig, dispatcherIndex); - return new FastSearcher(backend, fs4ResourcePool, dispatcher, docSumParams, clusterParams, cacheParams, + ClusterParams clusterParams = makeClusterParams(searchclusterIndex, + dispatcherIndex); + return new FastSearcher(backend, fs4ResourcePool, dispatcher, docSumParams, clusterParams, cacheParams, documentdbInfoConfig); } @@ -215,14 +212,13 @@ public class ClusterSearcher extends Searcher { int searchclusterIndex, QrSearchersConfig.Searchcluster searchClusterConfig, CacheParams cacheParams, - LegacyEmulationConfig emulConfig, SummaryParameters docSumParams, DocumentdbInfoConfig documentdbInfoConfig) { if (searchClusterConfig.searchdef().size() != 1) { throw new IllegalArgumentException("Search clusters in streaming search shall only contain a single searchdefinition : " + searchClusterConfig.searchdef()); } - ClusterParams clusterParams = makeClusterParams(searchclusterIndex, emulConfig, 0); + ClusterParams clusterParams = makeClusterParams(searchclusterIndex, 0); VdsStreamingSearcher searcher = (VdsStreamingSearcher) VespaBackEndSearcher .getSearcher("com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher"); searcher.setSearchClusterConfigId(searchClusterConfig.rankprofiles().configid()); @@ -319,7 +315,7 @@ public class ClusterSearcher extends Searcher { if (invalidInDocTypes != null && !invalidInDocTypes.isEmpty()) { String plural = invalidInDocTypes.size() > 1 ? "s" : ""; - return new Result(query, + return new Result(query, ErrorMessage.createInvalidQueryParameter("Requested rank profile '" + rankProfile + "' is undefined for document type" + plural + " '" + StringUtils.join(invalidInDocTypes.iterator(), ", ") + "'")); @@ -384,7 +380,7 @@ public class ClusterSearcher extends Searcher { if (query.getTimeout() <= maxQueryTimeout) return; if (query.isTraceable(2)) { - query.trace("Query timeout (" + query.getTimeout() + " ms) > max query timeout (" + + query.trace("Query timeout (" + query.getTimeout() + " ms) > max query timeout (" + maxQueryTimeout + " ms). Setting timeout to " + maxQueryTimeout + " ms.", 2); } query.setTimeout(maxQueryTimeout); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ClusterParams.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ClusterParams.java index 3dfa506a967..2787447e791 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ClusterParams.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ClusterParams.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; -import com.yahoo.container.search.LegacyEmulationConfig; - /** * Helper class for carrying around cluster-related * config parameters to the FastSearcher class. @@ -12,21 +10,12 @@ import com.yahoo.container.search.LegacyEmulationConfig; public class ClusterParams { public final String searcherName; - public final LegacyEmulationConfig emulation; - - /** - * For testcases only - */ - public ClusterParams(String name) { - this(name, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder())); - } /** * Make up full ClusterParams */ - public ClusterParams(String name, LegacyEmulationConfig cfg) { + public ClusterParams(String name) { this.searcherName = name; - this.emulation = cfg; } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java index cdf696ebe1e..20615a099b6 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinition.java @@ -4,7 +4,6 @@ package com.yahoo.prelude.fastsearch; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.yahoo.data.access.Inspector; -import com.yahoo.container.search.LegacyEmulationConfig; import java.util.ArrayList; import java.util.HashMap; @@ -41,8 +40,7 @@ public class DocsumDefinition { this.fieldNameToIndex = fieldNameToIndexBuilder.build(); } - // TODO: Remove LegacyEmulationConfig (the config, not just the usage) on Vespa 7 - DocsumDefinition(DocumentdbInfoConfig.Documentdb.Summaryclass config, LegacyEmulationConfig emulConfig) { + DocsumDefinition(DocumentdbInfoConfig.Documentdb.Summaryclass config) { this.name = config.name(); List<DocsumField> fieldsBuilder = new ArrayList<>(); @@ -51,7 +49,7 @@ public class DocsumDefinition { for (DocumentdbInfoConfig.Documentdb.Summaryclass.Fields field : config.fields()) { // no, don't switch the order of the two next lines :) fieldNameToIndexBuilder.put(field.name(), fieldsBuilder.size()); - fieldsBuilder.add(DocsumField.create(field.name(), field.type(), emulConfig)); + fieldsBuilder.add(DocsumField.create(field.name(), field.type())); if (field.dynamic()) dynamic = true; } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java index 07997f0c8f6..7745a71b24f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java @@ -7,7 +7,6 @@ import com.yahoo.data.access.Inspector; import com.yahoo.slime.Slime; import com.yahoo.data.access.slime.SlimeAdapter; import com.yahoo.prelude.ConfigurationException; -import com.yahoo.container.search.LegacyEmulationConfig; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -31,27 +30,15 @@ public final class DocsumDefinitionSet { private final static Logger log = Logger.getLogger(DocsumDefinitionSet.class.getName()); private final Map<String, DocsumDefinition> definitionsByName; - private final LegacyEmulationConfig emulationConfig; public DocsumDefinitionSet(DocumentdbInfoConfig.Documentdb config) { - this(config, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder())); - } - - public DocsumDefinitionSet(DocumentdbInfoConfig.Documentdb config, LegacyEmulationConfig emulConfig) { - this(toDocsums(config, emulConfig), emulConfig); + this(toDocsums(config)); } public DocsumDefinitionSet(Collection<DocsumDefinition> docsumDefinitions) { - this(docsumDefinitions, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder())); - } - - public DocsumDefinitionSet(Collection<DocsumDefinition> docsumDefinitions, LegacyEmulationConfig emulConfig) { this.definitionsByName = ImmutableMap.copyOf(docsumDefinitions.stream().collect(Collectors.toMap(DocsumDefinition::getName, p -> p))); - this.emulationConfig = emulConfig; } - public LegacyEmulationConfig legacyEmulationConfig() { return emulationConfig; } - /** * Returns the summary definition of the given name, or the default if not found. * @@ -111,10 +98,10 @@ public final class DocsumDefinitionSet { return definitionsByName.size(); } - private static Collection<DocsumDefinition> toDocsums(DocumentdbInfoConfig.Documentdb config, LegacyEmulationConfig emulConfig) { + private static Collection<DocsumDefinition> toDocsums(DocumentdbInfoConfig.Documentdb config) { Collection<DocsumDefinition> docsums = new ArrayList<>(); for (int i = 0; i < config.summaryclass().size(); ++i) - docsums.add(new DocsumDefinition(config.summaryclass(i), emulConfig)); + docsums.add(new DocsumDefinition(config.summaryclass(i))); if (docsums.isEmpty()) log.warning("No summary classes found in DocumentdbInfoConfig.Documentdb"); return docsums; diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java index 777919286dd..4f52ef91725 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; -import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.data.access.Inspector; import com.yahoo.log.LogLevel; @@ -31,19 +30,14 @@ public abstract class DocsumField { constructors.put(typename, constructor); } - DocsumField create(String typename, String name, LegacyEmulationConfig emulConfig) + DocsumField create(String typename, String name) throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { DocsumField f = constructors.get(typename).newInstance(name); - f.emulConfig = emulConfig; return f; } } - private LegacyEmulationConfig emulConfig; - - final LegacyEmulationConfig getEmulConfig() { return emulConfig; } - static { fieldFactory = new FieldFactory(); @@ -75,14 +69,9 @@ public abstract class DocsumField { this.name = name; } - /* For unit test only */ public static DocsumField create(String name, String typename) { - return create(name, typename, new LegacyEmulationConfig(new LegacyEmulationConfig.Builder())); - } - - public static DocsumField create(String name, String typename, LegacyEmulationConfig emulConfig) { try { - return fieldFactory.create(typename, name, emulConfig); + return fieldFactory.create(typename, name); } catch (Exception e) { throw new RuntimeException("Unknown field type '" + typename + "'", e); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java index 0ae0983a1ae..aa492524c83 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java @@ -2,7 +2,6 @@ package com.yahoo.prelude.fastsearch; import com.google.common.collect.ImmutableMap; -import com.yahoo.container.search.LegacyEmulationConfig; import java.util.ArrayList; import java.util.Collection; @@ -29,8 +28,8 @@ public class DocumentDatabase { private final ImmutableMap<String, RankProfile> rankProfiles; - public DocumentDatabase(DocumentdbInfoConfig.Documentdb documentDb, LegacyEmulationConfig emulConfig) { - this(documentDb.name(), new DocsumDefinitionSet(documentDb, emulConfig), toRankProfiles(documentDb.rankprofile())); + public DocumentDatabase(DocumentdbInfoConfig.Documentdb documentDb) { + this(documentDb.name(), new DocsumDefinitionSet(documentDb), toRankProfiles(documentDb.rankprofile())); } public DocumentDatabase(String name, DocsumDefinitionSet docsumDefinitionSet, Collection<RankProfile> rankProfiles) { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index 28d426cec06..8d73f6795d4 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -77,7 +77,6 @@ public class FastHit extends Hit { // Note: This constructor is only used for tests, production use is always of the empty constructor public FastHit(String uri, double relevance, String source) { setId(uri); - super.setField("uri", uri); // TODO: Remove on Vespa 7 setRelevance(new Relevance(relevance)); setSource(source); types().add("summary"); @@ -97,13 +96,6 @@ public class FastHit extends Hit { URI uri = super.getId(); if (uri != null) return uri; - // TODO: Remove on Vespa 7, this should be one of the last vestiges of URL field magic - Object uriField = getField("uri"); - if (uriField != null) { - setId(uriField.toString()); - return super.getId(); - } - // Fallback to index:[source]/[partid]/[id] if (indexUri != null) return indexUri; StringBuilder sb = new StringBuilder(64); @@ -315,21 +307,6 @@ public class FastHit extends Hit { return super.hasFields(); } - /** - * Changes the key under which a value is found. This is useful because it allows keys to be changed - * without accessing the value (which may be lazily created). - * - * @deprecated do not use - */ - @Deprecated // OK - @Override - @SuppressWarnings("deprecation") - public void changeFieldKey(String oldKey, String newKey) { - Object value = removeField(oldKey); - if (value != null) - setField(newKey, value); - } - private Object getSummaryValue(String name) { if (removedFields != null && removedFields.contains(name)) return null; @@ -357,13 +334,6 @@ public class FastHit extends Hit { } } - /** @deprecated do not use */ - // TODO: Make private on Vespa 7 - @Deprecated // OK - public static String asHexString(GlobalId gid) { - return asHexString(new StringBuilder(), gid).toString(); - } - private static StringBuilder asHexString(StringBuilder sb, GlobalId gid) { byte[] rawGid = gid.getRawId(); for (byte b : rawGid) { @@ -539,7 +509,7 @@ public class FastHit extends Hit { DocsumField fieldType = type.getField(name); if (fieldType == null) return null; Inspector fieldValue = data.field(name); - if ( ! fieldValue.valid() && ! fieldType.getEmulConfig().forceFillEmptyFields()) return null; + if ( ! fieldValue.valid()) return null; return fieldType.convert(fieldValue); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FeatureDataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FeatureDataField.java index 28c63aa2d3a..1f60dd3d1cf 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FeatureDataField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FeatureDataField.java @@ -3,7 +3,6 @@ package com.yahoo.prelude.fastsearch; import com.yahoo.data.access.Inspector; import com.yahoo.data.access.Type; -import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.search.result.FeatureData; /** @@ -13,7 +12,7 @@ import com.yahoo.search.result.FeatureData; */ public class FeatureDataField extends LongstringField { - public FeatureDataField (String name) { + public FeatureDataField(String name) { super(name); } @@ -25,23 +24,12 @@ public class FeatureDataField extends LongstringField { @Override public Object convert(Inspector value) { if (! value.valid()) { - if (getEmulConfig().stringBackedFeatureData()) { - return ""; - } else if (getEmulConfig().forceFillEmptyFields()) { - return new FeatureData(com.yahoo.data.access.simple.Value.empty()); - } else { - return null; - } + return null; } if (value.type() == Type.STRING) { return value.asString(); } - FeatureData obj = new FeatureData(value); - if (getEmulConfig().stringBackedFeatureData()) { - return obj.toJson(); - } else { - return obj; - } + return new FeatureData(value); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StructDataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StructDataField.java index 9521854477a..18986667a6c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StructDataField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StructDataField.java @@ -4,7 +4,6 @@ package com.yahoo.prelude.fastsearch; import com.yahoo.search.result.StructuredData; import com.yahoo.data.access.Inspector; import com.yahoo.data.access.Type; -import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.prelude.hitfield.JSONString; /** @@ -23,7 +22,7 @@ public class StructDataField extends JSONField { @Override public Object convert(Inspector value) { - if (getEmulConfig().stringBackedStructuredData() || value.type() == Type.STRING) { + if (value.type() == Type.STRING) { return super.convert(value); } return new StructuredData(value); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index e8f5d7110f7..d6e87e58fd8 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -177,7 +177,7 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { if (documentdbInfoConfig != null) { for (DocumentdbInfoConfig.Documentdb docDb : documentdbInfoConfig.documentdb()) { - DocumentDatabase db = new DocumentDatabase(docDb, clusterParams.emulation); + DocumentDatabase db = new DocumentDatabase(docDb); if (documentDbs.isEmpty()) { defaultDocumentDb = db; } @@ -372,7 +372,7 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { s.append(" ranking.queryCache=true"); } if (query.getGroupingSessionCache() || query.getRanking().getQueryCache()) { - s.append(" sessionId=").append(query.getSessionId(false)); + s.append(" sessionId=").append(query.getSessionId()); } List<Grouping> grouping = GroupingExecutor.getGroupingList(query); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/XMLField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/XMLField.java index cb115502468..d768dda2657 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/XMLField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/XMLField.java @@ -31,10 +31,10 @@ public class XMLField extends DocsumField { @Override public Object convert(Inspector value) { /* In Vespa 6 the backend will send an XML-formatted string to represent - * positions data. This will change in next version to sending an array - * of objects instead, suitable for the PositionsData class. + * positions data. This will change in next version to sending an object + * or an array of objects instead, suitable for the PositionsData class. */ - if (value.type() == Type.ARRAY) { + if (value.type() == Type.OBJECT || value.type() == Type.ARRAY) { return new PositionsData(value); } return convert(value.asString("")); diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/FieldIterator.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/FieldIterator.java index c8dc87aeb52..3c61678bbde 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/FieldIterator.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/FieldIterator.java @@ -7,7 +7,7 @@ import java.util.ListIterator; /** * A specialized list iterator to manipulate FieldParts in HitField objects. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class FieldIterator implements ListIterator<FieldPart> { diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java index f6619a32a2b..2702da099be 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java @@ -24,7 +24,7 @@ public class HitField { private boolean xmlProperty; private List<FieldPart> tokenizedContent = null; - private String content = null; + private String content; private Object original; @@ -48,7 +48,7 @@ public class HitField { /** * @param f The field name * @param c The field content - * @param cjk true if this is a cjk-document + * @param cjk true if the content is CJK text */ public HitField(String f, String c, boolean cjk) { this(f, c, cjk, false); @@ -57,7 +57,7 @@ public class HitField { /** * @param f The field name * @param c The field content - * @param cjk true if this is a cjk-document + * @param cjk true if the content is CJK text */ public HitField(String f, XMLString c, boolean cjk) { this(f, c.toString(), cjk, true); @@ -66,7 +66,7 @@ public class HitField { /** * @param f The field name * @param c The field content - * @param cjk true if this is a cjk-document + * @param cjk true if the content is CJK text * @param xmlProperty true if this should not quote XML syntax */ public HitField(String f, String c, boolean cjk, boolean xmlProperty) { @@ -279,9 +279,8 @@ public class HitField { // Must null content reference _before_ calling getContent() content = null; } - /** - * @return the content of this field - */ + + /** Returns the content of this field */ public String getContent() { if (content == null) { StringBuilder buf = new StringBuilder(); @@ -294,13 +293,8 @@ public class HitField { return content; } - /** - * @return the content of this field, using the arguments as bolding - * tags - */ - public String getContent(String boldOpenTag, - String boldCloseTag, - String separatorTag) { + /** Returns the content of this field, using the arguments as bolding tags */ + public String getContent(String boldOpenTag, String boldCloseTag, String separatorTag) { StringBuilder buf = new StringBuilder(); Iterator<FieldPart> iter = ensureTokenized().iterator(); while(iter.hasNext()) { @@ -372,8 +366,9 @@ public class HitField { } return xml.toString(); } + /** - * @return the content of the field, stripped of markup + * Returns the content of the field, stripped of markup */ public String bareContent(boolean XMLQuote, boolean inAttribute) { StringBuilder bareContent = new StringBuilder(); diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java index 06db012309e..eee7b310d13 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java @@ -86,6 +86,7 @@ public class JSONString implements Inspectable { didInitContent = true; } + @Override public String toString() { if (value != null) { return renderFromInspector(); diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/TokenFieldIterator.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/TokenFieldIterator.java index 95100dd4d39..beda005152c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/TokenFieldIterator.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/TokenFieldIterator.java @@ -8,7 +8,7 @@ import java.util.NoSuchElementException; /** * A specialized list iterator to manipulate tokens in HitField objects. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class TokenFieldIterator implements ListIterator<FieldPart> { diff --git a/container-search/src/main/java/com/yahoo/prelude/logging/AccessLogEntry.java b/container-search/src/main/java/com/yahoo/prelude/logging/AccessLogEntry.java deleted file mode 100644 index f9faf242350..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/logging/AccessLogEntry.java +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.logging; - -/** - * Hollow compatibility class for com.yahoo.container.logging.AccessLogEntry. - * - * @author Steinar Knutsen - * @deprecated do not use - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class AccessLogEntry extends com.yahoo.container.logging.AccessLogEntry { - - public AccessLogEntry() { - super(); - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/logging/package-info.java b/container-search/src/main/java/com/yahoo/prelude/logging/package-info.java deleted file mode 100644 index 01126ec3484..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/logging/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.prelude.logging; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java index f6f96ae215c..244d895f357 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java @@ -13,7 +13,7 @@ import static com.yahoo.prelude.query.parser.Token.Kind.NUMBER; * @author Steinar Knutsen * @deprecated since 5.11, YQL+ should be used for formal queries */ -@Deprecated // OK DO NOT REMOVE (we'll keep this around longer) +@Deprecated // DO NOT REMOVE (we'll keep this around longer) public class AdvancedParser extends StructuredParser { public AdvancedParser(ParserEnvironment environment) { diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/IndexCombinatorSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/IndexCombinatorSearcher.java deleted file mode 100644 index ff603a64725..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/IndexCombinatorSearcher.java +++ /dev/null @@ -1,361 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.querytransform; - -import static com.yahoo.prelude.querytransform.PhrasingSearcher.PHRASE_REPLACEMENT; - -import com.yahoo.component.chain.dependencies.After; -import com.yahoo.component.chain.dependencies.Before; -import com.yahoo.component.chain.dependencies.Provides; -import com.yahoo.log.LogLevel; -import com.yahoo.prelude.Index; -import com.yahoo.prelude.Index.Attribute; -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.*; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.search.searchchain.PhaseNames; - -import java.util.*; - -/** - * Searcher to rewrite queries to achieve mixed recall between indices and - * memory attributes. - * - * @author Steinar Knutsen - * @deprecated do not use - */ -// TODO: Remove on Vespa 7 (not necessary any more) -@After({PhaseNames.RAW_QUERY, PHRASE_REPLACEMENT}) -@Before(PhaseNames.TRANSFORMED_QUERY) -@Provides(IndexCombinatorSearcher.MIXED_RECALL_REWRITE) -@Deprecated // OK -public class IndexCombinatorSearcher extends Searcher { - - public static final String MIXED_RECALL_REWRITE = "MixedRecallRewrite"; - - private static class ArrayComparator implements Comparator<Attribute[]> { - /** - * Note, this ignores if there is a difference in whether to - * attributes have tokenized content. (If this is the case, - * we are having worse problems anyway.) - */ - public int compare(Attribute[] o1, Attribute[] o2 ) { - if (o1.length < o2.length) { - return -1; - } else if (o1.length > o2.length) { - return 1; - } - int limit = o1.length; - for (int i = 0; i < limit; ++i) { - int r = o1[i].name.compareTo(o2[i].name); - if (r != 0) { - return r; - } - } - return 0; - } - } - - private final ArrayComparator comparator = new ArrayComparator(); - - private enum RewriteStrategies { - NONE, CHEAP_AND, EXPENSIVE_AND, FLAT - } - - @Override - public Result search(Query query, Execution execution) { - Item root = query.getModel().getQueryTree().getRoot(); - IndexFacts.Session session = execution.context().getIndexFacts().newSession(query); - String oldQuery = (query.getTraceLevel() >= 2) ? root.toString() : ""; - - if (root instanceof BlockItem || root instanceof PhraseItem) { - root = convertSinglePhraseOrBlock(root, session); - } else if (root instanceof CompositeItem) { - root = rewrite((CompositeItem) root, session); - } - query.getModel().getQueryTree().setRoot(root); - - if ((query.getTraceLevel() >= 2) && !(oldQuery.equals(root.toString()))) { - query.trace("Rewriting for mixed recall between indices and attributes", true, 2); - } - return execution.search(query); - } - - private RewriteStrategies chooseRewriteStrategy(CompositeItem c, IndexFacts.Session session) { - if (c instanceof OrItem) { - return RewriteStrategies.FLAT; - } else if (!(c instanceof AndItem)) { - return RewriteStrategies.NONE; - } - Map<Attribute[], Integer> m = new TreeMap<>(comparator); - for (Iterator<Item> i = c.getItemIterator(); i.hasNext();) { - Item j = i.next(); - if (j instanceof BlockItem || j instanceof PhraseItem) { - Attribute[] attributes= getIndices((HasIndexItem) j, session); - if (attributes == null) { - continue; - } - Integer count = m.get(attributes); - if (count == null) { - count = 1; - } else { - count = count.intValue() + 1; - } - m.put(attributes, count); - } - } - - if (m.size() == 0) { - return RewriteStrategies.NONE; - } - - int singles = 0; - int pairs = 0; - int higher = 0; - // count the number of sets being associated with 1, 2 or more terms - for (Integer i : m.values()) { - switch (i.intValue()) { - case 1: - ++singles; - break; - case 2: - pairs += 2; - break; - default: - ++higher; - break; - } - } - if (higher == 0 && pairs + singles <= 2) { - return RewriteStrategies.EXPENSIVE_AND; - } else { - return RewriteStrategies.CHEAP_AND; - } - } - - private CompositeItem rewriteNot(NotItem not, IndexFacts.Session session) { - Item positive = not.getItem(0); - if (positive instanceof BlockItem || positive instanceof PhraseItem) { - positive = convertSinglePhraseOrBlock(positive, session); - not.setItem(0, positive); - } else if (positive instanceof CompositeItem) { - CompositeItem c = (CompositeItem) positive; - positive = rewrite(c, session); - not.setItem(0, positive); - } - - int length = not.getItemCount(); - // no need for keeping proximity in the negative branches, so we - // convert them one by one, _and_ always uses cheap transform - for (int i = 1; i < length; ++i) { - Item exclusion = not.getItem(i); - if (exclusion instanceof BlockItem || exclusion instanceof PhraseItem) { - exclusion = convertSinglePhraseOrBlock(exclusion, session); - not.setItem(i, exclusion); - } else if (exclusion instanceof CompositeItem) { - CompositeItem c = (CompositeItem) exclusion; - switch (chooseRewriteStrategy(c, session)) { - case NONE: - c = traverse(c, session); - break; - case CHEAP_AND: - case EXPENSIVE_AND: - c = cheapTransform(c, session); - break; - default: - c = flatTransform(c, session); - break; - } - not.setItem(i, c); - } - } - return not; - } - - private Item rewrite(CompositeItem c, IndexFacts.Session session) { - if (c instanceof NotItem) { - c = rewriteNot((NotItem) c, session); - return c; - } else { - switch (chooseRewriteStrategy(c, session)) { - case NONE: - c = traverse(c, session); - break; - case CHEAP_AND: - c = cheapTransform(c, session); - break; - case EXPENSIVE_AND: - c = expensiveTransform((AndItem) c, session); - break; - case FLAT: - c = flatTransform(c, session); - break; - default: - break; - } - } - return c; - } - - private CompositeItem traverse(CompositeItem c, IndexFacts.Session session) { - int length = c.getItemCount(); - for (int i = 0; i < length; ++i) { - Item word = c.getItem(i); - if (word instanceof CompositeItem && !(word instanceof PhraseItem) && !(word instanceof BlockItem)) { - c.setItem(i, rewrite((CompositeItem) word, session)); - } - } - return c; - } - - private CompositeItem expensiveTransform(AndItem c, IndexFacts.Session session) { - int[] indices = new int[2]; - int items = 0; - int length = c.getItemCount(); - Attribute[][] names = new Attribute[2][]; - CompositeItem result = null; - for (int i = 0; i < length; ++i) { - Item word = c.getItem(i); - if (word instanceof BlockItem || word instanceof PhraseItem) { - Attribute[] attributes = getIndices((HasIndexItem) word, session); - if (attributes == null) { - continue; - } - // this throwing an out of bounds if more than two candidates is intentional - names[items] = attributes; - indices[items++] = i; - } else if (word instanceof CompositeItem) { - c.setItem(i, rewrite((CompositeItem) word, session)); - } - } - switch (items) { - case 1: - result = linearAnd(c, names[0], indices[0]); - break; - case 2: - result = quadraticAnd(c, names[0], names[1], indices[0], indices[1]); - break; - default: - // should never happen - getLogger().log( - LogLevel.WARNING, - "Unexpected number of items for mixed recall, got " + items - + ", expected 1 or 2."); - break; - } - return result; - } - - private Attribute[] getIndices(HasIndexItem block, IndexFacts.Session session) { - return session.getIndex(block.getIndexName()).getMatchGroup(); - } - - private OrItem linearAnd(AndItem c, Attribute[] names, int brancherIndex) { - OrItem or = new OrItem(); - for (int i = 0; i < names.length; ++i) { - AndItem duck = (AndItem) c.clone(); - Item b = retarget(duck.getItem(brancherIndex), names[i]); - duck.setItem(brancherIndex, b); - or.addItem(duck); - } - return or; - } - - private OrItem quadraticAnd(AndItem c, Attribute[] firstNames, Attribute[] secondNames, int firstBrancher, int secondBrancher) { - OrItem or = new OrItem(); - for (int i = 0; i < firstNames.length; ++i) { - for (int j = 0; j < secondNames.length; ++j) { - AndItem duck = (AndItem) c.clone(); - Item b = retarget(duck.getItem(firstBrancher), firstNames[i]); - duck.setItem(firstBrancher, b); - b = retarget(duck.getItem(secondBrancher), secondNames[j]); - duck.setItem(secondBrancher, b); - or.addItem(duck); - } - } - return or; - } - - private CompositeItem flatTransform(CompositeItem c, IndexFacts.Session session) { - int maxIndex = c.getItemCount() - 1; - for (int i = maxIndex; i >= 0; --i) { - Item word = c.getItem(i); - if (word instanceof BlockItem || word instanceof PhraseItem) { - Attribute[] attributes = getIndices((HasIndexItem) word, session); - if (attributes == null) { - continue; - } - c.removeItem(i); - for (Attribute name : attributes) { - Item term = word.clone(); - Item forNewIndex = retarget(term, name); - c.addItem(forNewIndex); - } - } else if (word instanceof CompositeItem) { - c.setItem(i, rewrite((CompositeItem) word, session)); - } - } - return c; - } - - private CompositeItem cheapTransform(CompositeItem c, IndexFacts.Session session) { - if (c instanceof OrItem) { - return flatTransform(c, session); - } - int length = c.getItemCount(); - for (int i = 0; i < length; ++i) { - Item j = c.getItem(i); - if (j instanceof BlockItem || j instanceof PhraseItem) { - Attribute[] attributes = getIndices((HasIndexItem) j, session); - if (attributes == null) { - continue; - } - CompositeItem or = searchAllForItem(j, attributes); - c.setItem(i, or); - } else if (j instanceof CompositeItem) { - c.setItem(i, rewrite((CompositeItem) j, session)); - } - } - return c; - } - - private OrItem searchAllForItem(Item word, Attribute[] attributes) { - OrItem or = new OrItem(); - for (Attribute name : attributes) { - Item term = word.clone(); - term = retarget(term, name); - or.addItem(term); - } - return or; - } - - private Item retarget(Item word, Attribute newIndex) { - if (word instanceof PhraseItem && !newIndex.isTokenizedContent()) { - PhraseItem asPhrase = (PhraseItem) word; - WordItem newWord = new WordItem(asPhrase.getIndexedString(), newIndex.name, false); - return newWord; - } else if (word instanceof IndexedItem) { - word.setIndexName(newIndex.name); - } else if (word instanceof CompositeItem) { - CompositeItem asComposite = (CompositeItem) word; - for (Iterator<Item> i = asComposite.getItemIterator(); i.hasNext();) { - Item segment = i.next(); - segment.setIndexName(newIndex.name); - } - } - return word; - } - - private Item convertSinglePhraseOrBlock(Item item, IndexFacts.Session session) { - Item newItem; - Attribute[] attributes = getIndices((HasIndexItem) item, session); - if (attributes == null) { - return item; - } - newItem = searchAllForItem(item, attributes); - return newItem; - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java index 28d384b0f63..61ce9d98e69 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java @@ -38,21 +38,20 @@ public class BlendingSearcher extends Searcher { public static final String BLENDING = "Blending"; - private final String documentId; + private final String blendingField; @Inject public BlendingSearcher(ComponentId id, QrSearchersConfig cfg) { super(id); QrSearchersConfig.Com.Yahoo.Prelude.Searcher.BlendingSearcher s = cfg.com().yahoo().prelude().searcher().BlendingSearcher(); - documentId = s.docid().length() > 0 ? s.docid() : null; - + blendingField = s.docid().length() > 0 ? s.docid() : null; } /** * Only for legacy tests. */ - public BlendingSearcher(String blendingDocumentId) { - this.documentId = blendingDocumentId; + public BlendingSearcher(String blendingField) { + this.blendingField = blendingField; } @Override @@ -107,7 +106,7 @@ public class BlendingSearcher extends Searcher { result.hits().setOrderer(groups.get(0).getOrderer()); return result; } else { - if (documentId != null) { + if (blendingField != null) { return blendResultsUniquely(result, q, offset, hits, groups, execution); } else { return blendResultsDirectly(result, q, offset, hits, groups, execution); @@ -125,6 +124,7 @@ public class BlendingSearcher extends Searcher { } private abstract class DocumentMerger { + protected Set<String> documentsToStrip; protected Result result; protected HitGroup group; @@ -137,22 +137,22 @@ public class BlendingSearcher extends Searcher { return result; } - //Since we cannot use prelude.hit#getProperty, we'll have to improvise private String getProperty(Hit hit, String field) { + if ("[id]".equals(field)) return hit.getId().toString(); Object o = hit.getField(field); return o == null ? null : o.toString(); } protected void storeID(Hit hit, Execution execution) { - String id = getProperty(hit, documentId); + String id = getProperty(hit, blendingField); if (id != null) { documentsToStrip.add(id); } else { if (!result.isFilled(result.getQuery().getPresentation().getSummary())) { fill(result, result.getQuery().getPresentation().getSummary(), execution); - id = getProperty(hit, documentId); + id = getProperty(hit, blendingField); if (id != null) { documentsToStrip.add(id); } @@ -161,14 +161,14 @@ public class BlendingSearcher extends Searcher { } protected boolean known(HitGroup source, Hit hit, Execution execution) { - String stripID = getProperty(hit, documentId); + String stripID = getProperty(hit, blendingField); if (stripID == null) { if (!source.isFilled(result.getQuery().getPresentation().getSummary())) { Result nResult = new Result(result.getQuery()); nResult.hits().add(source); fill(nResult, nResult.getQuery().getPresentation().getSummary(), execution); - stripID = getProperty(hit, documentId); + stripID = getProperty(hit, blendingField); if (stripID == null) { return false; } diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/DocumentSourceSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/DocumentSourceSearcher.java deleted file mode 100644 index bc3c6665ae2..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/DocumentSourceSearcher.java +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.searcher; - -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; - -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; - - -/** - * <p>Implements a document source. You pass in a query and a Result - * set. When this Searcher is called with that query it will return - * that result set.</p> - * - * <p>This supports multi-phase search.</p> - * - * <p>To avoid having to add type information for the fields, a quck hack is used to - * support testing of attribute prefetching. - * Any field in the configured hits which has a name starting by attribute - * will be returned when attribute prefetch filling is requested.</p> - * - * @author bratseth - * @deprecated use {@link com.yahoo.search.searchchain.testutil.DocumentSourceSearcher} - */ -@SuppressWarnings({"rawtypes"}) -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class DocumentSourceSearcher extends Searcher { - - // as for the SuppressWarnings annotation above, we are inside - // com.yahoo.prelude, this is old stuff, really no point firing off those - // warnings here... - - private Result defaultFilledResult; - private Map<Query, Result> completelyFilledResults = new HashMap<>(); - private Map<Query, Result> attributeFilledResults = new HashMap<>(); - private Map<Query, Result> unFilledResults = new HashMap<>(); - - /** Time (in ms) at which the index of this searcher was last modified */ - long editionTimeStamp=0; - - private int queryCount; - - public DocumentSourceSearcher() { - addDefaultResults(); - } - - /** - * Adds a result which can be returned either as empty, - * filled or attribute only filled later. - * Summary fields starting by "a" are attributes, others are not. - * - * @return true when replacing an existing <query, result> pair. - */ - public boolean addResultSet(Query query, Result fullResult) { - Result emptyResult = new Result(query.clone()); - Result attributeResult = new Result(query.clone()); - emptyResult.setTotalHitCount(fullResult.getTotalHitCount()); - attributeResult.setTotalHitCount(fullResult.getTotalHitCount()); - int counter=0; - for (Iterator i = fullResult.hits().deepIterator();i.hasNext();) { - Hit fullHit = (Hit)i.next(); - - Hit emptyHit = fullHit.clone(); - emptyHit.clearFields(); - emptyHit.setFillable(); - emptyHit.setRelevance(fullHit.getRelevance()); - - Hit attributeHit = fullHit.clone(); - removePropertiesNotStartingByA(attributeHit); - attributeHit.setFillable(); - attributeHit.setRelevance(fullHit.getRelevance()); - for (Object propertyKeyObject : fullHit.fields().keySet()) { - String propertyKey=propertyKeyObject.toString(); - if (propertyKey.startsWith("attribute")) - attributeHit.setField(propertyKey, fullHit.getField(propertyKey)); - } - if (fullHit.getField(Hit.SDDOCNAME_FIELD)!=null) - attributeHit.setField(Hit.SDDOCNAME_FIELD, fullHit.getField(Hit.SDDOCNAME_FIELD)); - - // A simple summary lookup mechanism, similar to FastSearch's - emptyHit.setField("summaryid", String.valueOf(counter)); - attributeHit.setField("summaryid", String.valueOf(counter)); - fullHit.setField("summaryid", String.valueOf(counter)); - - counter++; - emptyResult.hits().add(emptyHit); - attributeResult.hits().add(attributeHit); - } - unFilledResults.put(getQueryKeyClone(query), emptyResult); - attributeFilledResults.put(getQueryKeyClone(query), attributeResult); - if (completelyFilledResults.put(getQueryKeyClone(query), fullResult.clone()) != null) { - setEditionTimeStamp(System.currentTimeMillis()); - return true; - } - return false; - } - - /** - * Returns a query clone which has source, offset and hits set to null. This is used by access to - * the maps using the query as key to achieve lookup independent of offset/hits value - */ - private Query getQueryKeyClone(Query query) { - Query key = query.clone(); - key.setWindow(0,0); - key.getModel().setSources(""); - return key; - } - - private void removePropertiesNotStartingByA(Hit hit) { - List<String> toRemove=new java.util.ArrayList<>(); - for (Iterator i= ((Set) hit.fields().keySet()).iterator(); i.hasNext(); ) { - String key=(String)i.next(); - if (!key.startsWith("a")) - toRemove.add(key); - } - for (Iterator<String> i=toRemove.iterator(); i.hasNext(); ) { - String propertyName=i.next(); - hit.removeField(propertyName); - } - } - - private void addDefaultResults() { - Query q = new Query("?query=default"); - Result r = new Result(q); - r.hits().add(new Hit("http://default-1.html")); - r.hits().add(new Hit("http://default-2.html")); - r.hits().add(new Hit("http://default-3.html")); - r.hits().add(new Hit("http://default-4.html")); - defaultFilledResult = r; - addResultSet(q, r); - } - - public long getEditionTimeStamp(){ - long myEditionTime; - synchronized(this){ - myEditionTime=this.editionTimeStamp; - } - return myEditionTime; - } - - public void setEditionTimeStamp(long editionTime) { - synchronized(this){ - this.editionTimeStamp=editionTime; - } - } - - public Result search(com.yahoo.search.Query query, Execution execution) { - queryCount++; - Result r; - r = unFilledResults.get(getQueryKeyClone(query)); - if (r == null) { - r = defaultFilledResult.clone(); - } else { - r = r.clone(); - } - r.setQuery(query); - r.hits().trim(query.getOffset(), query.getHits()); - return r; - } - - @Override - public void fill(com.yahoo.search.Result result, String summaryClass, Execution execution) { - Result filledResult; - if ("attributeprefetch".equals(summaryClass)) - filledResult=attributeFilledResults.get(getQueryKeyClone(result.getQuery())); - else - filledResult = completelyFilledResults.get(getQueryKeyClone(result.getQuery())); - - if (filledResult == null) { - filledResult = defaultFilledResult; - } - fillHits(filledResult,result,summaryClass); - } - - private void fillHits(Result source,Result target,String summaryClass) { - for (Iterator hitsToFill= target.hits().deepIterator() ; hitsToFill.hasNext();) { - Hit hitToFill = (Hit) hitsToFill.next(); - String summaryId= (String) hitToFill.getField("summaryid"); - if (summaryId==null) continue; // Can not fill this - Hit filledHit = lookupBySummaryId(source,summaryId); - if (filledHit==null) - throw new RuntimeException("Can't fill hit with summaryid '" + summaryId + "', not present"); - - for (Iterator props= filledHit.fieldIterator();props.hasNext();) { - Map.Entry propertyEntry = (Map.Entry)props.next(); - hitToFill.setField(propertyEntry.getKey().toString(), - propertyEntry.getValue()); - } - hitToFill.setFilled(summaryClass); - } - target.analyzeHits(); - } - - private Hit lookupBySummaryId(Result result,String summaryId) { - for (Iterator i= result.hits().deepIterator(); i.hasNext(); ) { - Hit hit=(Hit)i.next(); - if (summaryId.equals(hit.getField("summaryid"))) { - return hit; - } - } - return null; - } - - /** - * Returns the number of queries made to this searcher since the last - * reset. For testing - not reliable if multiple threads makes - * queries simultaneously - */ - public int getQueryCount() { - return queryCount; - } - - public void resetQueryCount() { - queryCount=0; - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java index 5c56379efc0..d11b8d86d29 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java @@ -77,10 +77,9 @@ public class JuniperSearcher extends Searcher { @Override public void fill(Result result, String summaryClass, Execution execution) { - Result workResult = result; - int worstCase = workResult.getHitCount(); + int worstCase = result.getHitCount(); List<Hit> hits = new ArrayList<>(worstCase); - for (Iterator<Hit> i = workResult.hits().deepIterator(); i.hasNext();) { + for (Iterator<Hit> i = result.hits().deepIterator(); i.hasNext();) { Hit sniffHit = i.next(); if ( ! (sniffHit instanceof FastHit)) continue; @@ -89,26 +88,26 @@ public class JuniperSearcher extends Searcher { hits.add(hit); } - execution.fill(workResult, summaryClass); - highlight(workResult.getQuery().getPresentation().getBolding(), hits.iterator(), summaryClass, + execution.fill(result, summaryClass); + highlight(result.getQuery().getPresentation().getBolding(), hits.iterator(), summaryClass, execution.context().getIndexFacts().newSession(result.getQuery())); } private void highlight(boolean bolding, Iterator<Hit> hitsToHighlight, String summaryClass, IndexFacts.Session indexFacts) { while (hitsToHighlight.hasNext()) { - Hit sniffHit = hitsToHighlight.next(); - if ( ! (sniffHit instanceof FastHit)) continue; + Hit hit = hitsToHighlight.next(); + if ( ! (hit instanceof FastHit)) continue; - FastHit hit = (FastHit) sniffHit; - if (summaryClass != null && ! hit.isFilled(summaryClass)) continue; + FastHit fastHit = (FastHit) hit; + if (summaryClass != null && ! fastHit.isFilled(summaryClass)) continue; - Object searchDefinitionField = hit.getField(MAGIC_FIELD); + Object searchDefinitionField = fastHit.getField(MAGIC_FIELD); if (searchDefinitionField == null) continue; for (Index index : indexFacts.getIndexes(searchDefinitionField.toString())) { if (index.getDynamicSummary() || index.getHighlightSummary()) { - HitField fieldValue = hit.buildHitField(index.getName(), true, true); + HitField fieldValue = fastHit.buildHitField(index.getName(), true); if (fieldValue != null) insertTags(fieldValue, bolding, index.getDynamicSummary()); } @@ -116,9 +115,9 @@ public class JuniperSearcher extends Searcher { } } - private void insertTags(HitField oldProperty, boolean bolding, boolean dynteaser) { + private void insertTags(HitField field, boolean bolding, boolean dynteaser) { boolean insideHighlight = false; - for (ListIterator<FieldPart> i = oldProperty.listIterator(); i.hasNext();) { + for (ListIterator<FieldPart> i = field.listIterator(); i.hasNext();) { FieldPart f = i.next(); if (f instanceof SeparatorFieldPart) setSeparatorString(bolding, (SeparatorFieldPart) f); @@ -138,8 +137,7 @@ public class JuniperSearcher extends Searcher { break; case RAW_SEPARATOR_CHAR: newFieldParts = initFieldParts(newFieldParts); - addSeparator(bolding, dynteaser, f, toQuote, newFieldParts, - previous, j); + addSeparator(bolding, dynteaser, f, toQuote, newFieldParts, previous, j); previous = j + 1; break; default: diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/QuerySnapshotSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/QuerySnapshotSearcher.java deleted file mode 100644 index 4edb907d337..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/QuerySnapshotSearcher.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.searcher; - -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.Relevance; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.searchchain.Execution; - -/** - * Save the query in the incoming state to a meta hit in the result. - * - * @author Steinar Knutsen - * @deprecated do not use - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class QuerySnapshotSearcher extends Searcher { - - public Result search(Query query, Execution execution) { - Query q = query.clone(); - Result r = execution.search(query); - Hit h = new Hit("meta:querysnapshot", new Relevance(Double.POSITIVE_INFINITY)); - h.setMeta(true); - h.setField("query", q); - r.hits().add(h); - return r; - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/QueryValidatingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/QueryValidatingSearcher.java deleted file mode 100644 index fd155354d1c..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/QueryValidatingSearcher.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.searcher; - -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.searchchain.Execution; - -/** - * Ensures hits is 1000 or less and offset is 1000 or less. - * - * @author Steinar Knutsen - * @deprecated do not use - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class QueryValidatingSearcher extends Searcher { - - public Result search(Query query, Execution execution) { - if (query.getHits() > 1000) { - Result result = new Result(query); - ErrorMessage error - = ErrorMessage.createInvalidQueryParameter("Too many hits (more than 1000) requested."); - result.hits().addError(error); - return result; - } - if (query.getOffset() > 1000) { - Result result = new Result(query); - ErrorMessage error - = ErrorMessage.createInvalidQueryParameter("Offset too high (above 1000)."); - result.hits().addError(error); - return result; - } - return execution.search(query); - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java b/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java index 89849f98461..4d46107726e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java @@ -15,7 +15,6 @@ import com.yahoo.search.searchchain.PhaseNames; import java.util.*; -import static com.yahoo.prelude.querytransform.IndexCombinatorSearcher.MIXED_RECALL_REWRITE; import static com.yahoo.prelude.querytransform.StemmingSearcher.STEMMING; /** @@ -24,7 +23,7 @@ import static com.yahoo.prelude.querytransform.StemmingSearcher.STEMMING; * @author bratseth */ @After(PhaseNames.RAW_QUERY) -@Before({PhaseNames.TRANSFORMED_QUERY, STEMMING, MIXED_RECALL_REWRITE}) +@Before({PhaseNames.TRANSFORMED_QUERY, STEMMING}) public class SemanticSearcher extends Searcher { private static final CompoundName rulesRulebase=new CompoundName("rules.rulebase"); diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index beb79acb893..3606e01ffe5 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -3,7 +3,6 @@ package com.yahoo.prelude.statistics; import com.yahoo.component.chain.dependencies.Before; import com.yahoo.concurrent.CopyOnWriteHashMap; -import com.yahoo.container.Server; import com.yahoo.container.protect.Error; import com.yahoo.jdisc.Metric; import com.yahoo.log.LogLevel; @@ -18,9 +17,7 @@ import com.yahoo.search.result.ErrorHit; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.PhaseNames; -import com.yahoo.statistics.Callback; import com.yahoo.statistics.Counter; -import com.yahoo.statistics.Handle; import com.yahoo.statistics.Value; import java.util.HashMap; diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/Context.java b/container-search/src/main/java/com/yahoo/prelude/templates/Context.java deleted file mode 100644 index 7662ca832f9..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/Context.java +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import java.util.Collection; - -import com.yahoo.text.XML; - -/** - * A set of variable bindings for template rendering - * - * @author bratseth - * @deprecated use a Renderer instead - */ -@SuppressWarnings("deprecation") -// TODO: Remove on Vespa 7 -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -public abstract class Context { - - private boolean xmlEscape = true; - - // These may be wrapped in an object if it gets unruly like this... - private String boldOpenTag; - private String boldCloseTag; - private String separatorTag; - - private boolean utf8Output = false; - - //prevent sub-classing outside of this package. - Context() {} - - // set|getXmlEscape no longer final on cause of HitContext subclassing _and_ wrapping Context - /** Sets whether this context should xml-escape returned values */ - public void setXmlEscape(boolean xmlEscape) { this.xmlEscape=xmlEscape; } - - /** Returns whether this context xml-escapes returned values. Default is true */ - public boolean getXmlEscape() { return xmlEscape; } - - /** - * Makes a <b>secondary</b> binding - * - * @return the old value bound to this key, or null it the key was previously unbound - */ - public abstract Object put(String key,Object value); - - /** - * <p>Returns a value by looking it up in the primary, - * and thereafter in secondary sources.</p> - * - * <p>If xml escaping is on and this is a string, xml attribute escaping is done - * </p> - */ - abstract public Object get(String key); - - /** - * Removes a <b>secondary</b> binding - * - * @return the removed value, or null if it wasn't bound - */ - public abstract Object remove(Object key); - - - // These three may be collapsed to one method - public void setBoldOpenTag(String boldOpenTag) { - this.boldOpenTag = boldOpenTag; - } - public void setBoldCloseTag(String boldCloseTag) { - this.boldCloseTag = boldCloseTag; - } - public void setSeparatorTag(String separatorTag) { - this.separatorTag = separatorTag; - } - - - protected Object normalizeValue(Object value) { - if (value == null) { - return ""; - } else if (xmlEscape && value instanceof String) { - return XML.xmlEscape((String) value, true, null); - } else { - return value; - } - } - - public String getBoldOpenTag() { - return boldOpenTag; - } - - public String getBoldCloseTag() { - return boldCloseTag; - } - - public String getSeparatorTag() { - return separatorTag; - } - - public abstract Collection<? extends Object> getKeys(); - - /** - * Used by the template to decide whether to use UTF-8 optimizations. - * - * @return whether the result encoding is UTF-8 - */ - public boolean isUtf8Output() { - return utf8Output; - } - - /** - * Used by the template to decide whether to use UTF-8 optimizations. - * TODO: TVT: Make this package private again - * @param utf8Output whether the output encoding is UTF-8 - */ - public void setUtf8Output(boolean utf8Output) { - this.utf8Output = utf8Output; - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/DefaultTemplateSet.java b/container-search/src/main/java/com/yahoo/prelude/templates/DefaultTemplateSet.java deleted file mode 100644 index 4d6406fe00d..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/DefaultTemplateSet.java +++ /dev/null @@ -1,299 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import com.yahoo.concurrent.CopyOnWriteHashMap; -import com.yahoo.io.ByteWriter; -import com.yahoo.net.URI; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.hitfield.HitField; -import com.yahoo.prelude.hitfield.JSONString; -import com.yahoo.prelude.hitfield.XMLString; -import com.yahoo.search.Result; -import com.yahoo.search.grouping.result.HitRenderer; -import com.yahoo.search.result.*; -import com.yahoo.text.Utf8String; -import com.yahoo.text.XML; -import com.yahoo.text.XMLWriter; - -import java.io.IOException; -import java.io.Writer; -import java.util.Iterator; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * <p>A template set which provides XML rendering of results and hits.</p> - * - * <p>This can be extended to create custom programmatic templates. - * Create a subclass which has static inner classes extending DefaultTemplate for the templates - * you wish to override and call the set method for those templates in the subclass template set - * constructor. Some of the default templates contained utility functions, and can be overridden - * in place of DefaultTemplate to gain access to these. See TiledTemplateSet for an example.</p> - * - * @author bratseth - * @deprecated use JsonRenderer instead - */ -@SuppressWarnings("deprecation") -// TODO: Remove on Vespa 7 -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -public class DefaultTemplateSet extends UserTemplate<XMLWriter> { - - private static final Utf8String RESULT = new Utf8String("result"); - private static final Utf8String GROUP = new Utf8String("group"); - private static final Utf8String ID = new Utf8String("id"); - private static final Utf8String FIELD = new Utf8String("field"); - private static final Utf8String HIT = new Utf8String("hit"); - private static final Utf8String ERROR = new Utf8String("error"); - private static final Utf8String TOTAL_HIT_COUNT = new Utf8String("total-hit-count"); - private static final Utf8String QUERY_TIME = new Utf8String("querytime"); - private static final Utf8String SUMMARY_FETCH_TIME = new Utf8String("summaryfetchtime"); - private static final Utf8String SEARCH_TIME = new Utf8String("searchtime"); - private static final Utf8String NAME = new Utf8String("name"); - private static final Utf8String CODE = new Utf8String("code"); - private static final Utf8String COVERAGE_DOCS = new Utf8String("coverage-docs"); - private static final Utf8String COVERAGE_NODES = new Utf8String("coverage-nodes"); - private static final Utf8String COVERAGE_FULL = new Utf8String("coverage-full"); - private static final Utf8String COVERAGE = new Utf8String("coverage"); - private static final Utf8String RESULTS_FULL = new Utf8String("results-full"); - private static final Utf8String RESULTS = new Utf8String("results"); - private static final Utf8String TYPE = new Utf8String("type"); - private static final Utf8String RELEVANCY = new Utf8String("relevancy"); - private static final Utf8String SOURCE = new Utf8String("source"); - - private final CopyOnWriteHashMap<String, Utf8String> fieldNameMap = new CopyOnWriteHashMap<>(); - - - /** - * Create a template set with a name. This will be initialized with the default templates - - * use the set methods from the subclass constructor to override any of these with other template classes. - */ - protected DefaultTemplateSet(String name) { - super(name, - DEFAULT_MIMETYPE, - DEFAULT_ENCODING - ); - } - - public DefaultTemplateSet() { - this("default"); - } - - /** Uses an XML writer in this template */ - @Override - public XMLWriter wrapWriter(Writer writer) { - return XMLWriter.from(writer, 10, -1); - } - - @Override - public void header(Context context, XMLWriter writer) throws IOException { - Result result=(Result)context.get("result"); - // TODO: move setting this to Result - context.setUtf8Output("utf-8".equalsIgnoreCase(getRequestedEncoding(result.getQuery()))); - writer.xmlHeader(getRequestedEncoding(result.getQuery())); - writer.openTag(RESULT).attribute(TOTAL_HIT_COUNT,String.valueOf(result.getTotalHitCount())); - renderCoverageAttributes(result.getCoverage(false), writer); - renderTime(writer, result); - writer.closeStartTag(); - } - - private void renderTime(final XMLWriter writer, final Result result) { - if (!result.getQuery().getPresentation().getTiming()) { - return; - } - - final String threeDecimals = "%.3f"; - final double milli = .001d; - final long now = System.currentTimeMillis(); - final long searchTime = now - result.getElapsedTime().first(); - final double searchSeconds = ((double) searchTime) * milli; - - if (result.getElapsedTime().firstFill() != 0L) { - final long queryTime = result.getElapsedTime().weightedSearchTime(); - final long summaryFetchTime = result.getElapsedTime().weightedFillTime(); - final double querySeconds = ((double) queryTime) * milli; - final double summarySeconds = ((double) summaryFetchTime) * milli; - writer.attribute(QUERY_TIME, String.format(threeDecimals, querySeconds)); - writer.attribute(SUMMARY_FETCH_TIME, String.format(threeDecimals, summarySeconds)); - } - writer.attribute(SEARCH_TIME, String.format(threeDecimals, searchSeconds)); - } - - @Override - public void footer(Context context, XMLWriter writer) throws IOException { - writer.closeTag(); - } - - @Override - /** - * Renders the header of a hit.<br/> - * Post-condition: The hit tag is open in this XML writer - */ - public void hit(Context context, XMLWriter writer) throws IOException { - Hit hit=(Hit)context.get("hit"); - - if (hit instanceof HitGroup) { - renderHitGroup((HitGroup) hit, context, writer); - } else { - writer.openTag(HIT); - renderHitAttributes(hit,writer); - writer.closeStartTag(); - renderHitFields(context, hit, writer); - } - } - - - @Override - /** - * Renders the footer of a hit. - * - * Pre-condition: The hit tag is open in this XML writer.<br/> - * Post-condition: The hit tag is closed - */ - public void hitFooter(Context context, XMLWriter writer) throws IOException { - writer.closeTag(); - } - - @Override - public void error(Context context, XMLWriter writer) { - ErrorMessage error=((Result)context.get("result")).hits().getError(); - writer.openTag(ERROR).attribute(CODE,error.getCode()).content(error.getMessage(),false).closeTag(); - } - - @Override - public void noHits(Context context, XMLWriter writer) { - // no hits, do nothing :) - } - - protected static void renderCoverageAttributes(Coverage coverage, XMLWriter writer) { - if (coverage == null) return; - writer.attribute(COVERAGE_DOCS,coverage.getDocs()); - writer.attribute(COVERAGE_NODES,coverage.getNodes()); - writer.attribute(COVERAGE_FULL,coverage.getFull()); - writer.attribute(COVERAGE,coverage.getResultPercentage()); - writer.attribute(RESULTS_FULL,coverage.getFullResultSets()); - writer.attribute(RESULTS,coverage.getResultSets()); - } - - /** - * Writes a hit's default attributes like 'type', 'source', 'relevancy'. - */ - protected void renderHitAttributes(Hit hit, XMLWriter writer) throws IOException { - writer.attribute(TYPE, hit.types().stream().collect(Collectors.joining(" "))); - if (hit.getRelevance() != null) { - writer.attribute(RELEVANCY, hit.getRelevance().toString()); - } - writer.attribute(SOURCE, hit.getSource()); - } - - /** Opens (but does not close) the group hit tag */ - protected void renderHitGroup(HitGroup hit, Context context, XMLWriter writer) throws IOException { - if (HitRenderer.renderHeader(hit, writer)) { - // empty - } else if (hit.types().contains("grouphit")) { - // TODO Keep this? - renderHitGroupOfTypeGroupHit(context, hit, writer); - } else { - renderGroup(hit, writer); - } - } - - /** - * Renders a hit group. - */ - protected void renderGroup(HitGroup hit, XMLWriter writer) throws IOException { - writer.openTag(GROUP); - renderHitAttributes(hit, writer); - writer.closeStartTag(); - } - - // Can't name this renderGroupHit as GroupHit is a class having nothing to do with HitGroup. - // Confused yet? Good! - protected void renderHitGroupOfTypeGroupHit(Context context, HitGroup hit, XMLWriter writer) throws IOException { - writer.openTag(HIT); - renderHitAttributes(hit, writer); - renderId(hit.getId(), writer); - writer.closeStartTag(); - } - - - protected void renderId(URI uri, XMLWriter writer) { - if (uri != null) { - writer.openTag(ID).content(uri.stringValue(),false).closeTag(); - } - } - - /** - * Renders all fields of a hit. - * Simply calls {@link #renderField(Context, Hit, java.util.Map.Entry, XMLWriter)} for every field. - */ - protected void renderHitFields(Context context, Hit hit, XMLWriter writer) throws IOException { - renderSyntheticRelevancyField(hit, writer); - for (Iterator<Map.Entry<String, Object>> it = hit.fieldIterator(); it.hasNext(); ) { - renderField(context, hit, it.next(), writer); - } - } - - private void renderSyntheticRelevancyField(Hit hit, XMLWriter writer) { - final String relevancyFieldName = "relevancy"; - final Relevance relevance = hit.getRelevance(); - - if (shouldRenderField(hit, relevancyFieldName) && relevance != null) { - renderSimpleField(relevancyFieldName, relevance, writer); - } - } - - protected void renderField(Context context, Hit hit, Map.Entry<String, Object> entry, XMLWriter writer) throws IOException { - String fieldName = entry.getKey(); - - if (!shouldRenderField(hit, fieldName)) return; - if (fieldName.startsWith("$")) return; // Don't render fields that start with $ // TODO: Move to should render - - writeOpenFieldElement(fieldName, writer); - renderFieldContent(context, hit, fieldName, writer); - writeCloseFieldElement(writer); - } - - private void writeOpenFieldElement(String fieldName, XMLWriter writer) { - Utf8String utf8 = fieldNameMap.get(fieldName); - if (utf8 == null) { - utf8 = new Utf8String(fieldName); - fieldNameMap.put(fieldName, utf8); - } - writer.openTag(FIELD).attribute(NAME, utf8); - writer.closeStartTag(); - } - - private void writeCloseFieldElement(XMLWriter writer) { - writer.closeTag(); - } - - protected void renderFieldContent(Context context, Hit hit, String name, XMLWriter writer) { - writer.escapedContent(asXML(hit.getField(name)), false); - } - - private String asXML(Object value) { - if (value == null) - return "(null)"; - else if (value instanceof HitField) - return ((HitField)value).quotedContent(false); - else if (value instanceof StructuredData || value instanceof XMLString || value instanceof JSONString) - return value.toString(); - else - return XML.xmlEscape(value.toString(), false, '\u001f'); - } - - private void renderSimpleField(String fieldName, Object fieldValue, XMLWriter writer) { - writeOpenFieldElement(fieldName, writer); - writer.content(fieldValue.toString(),false); - writeCloseFieldElement(writer); - } - - /** Returns whether a field should be rendered. This default implementation always returns true */ - protected boolean shouldRenderField(Hit hit, String fieldName) { - // skip depending on hit type - return true; - } - - - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/FormattingOptions.java b/container-search/src/main/java/com/yahoo/prelude/templates/FormattingOptions.java deleted file mode 100644 index 1f724aba112..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/FormattingOptions.java +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Set; - -/** - * Defines formatting options used with special kinds of hits. - * - * @author laboisse - * @deprecated use a Renderer instead - */ -@SuppressWarnings("deprecation") -// TODO: Remove on Vespa 7 -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -public class FormattingOptions { - - public static final String DEFAULT_TYPE_ATTRIBUTE_NAME = "type"; - - /** - * A structure that defines the tag name and attribute name for a field - * that sould be formatted as a field with a subtype. - * @author laboisse - * - */ - static class SubtypeField { - String tagName; - String attributeName; - String attributeValue; - } - - static class SubtypeFieldWithPrefix extends SubtypeField { - - /* Note: attributeValue should always be null for instances of this class */ - - int prefixLength; - } - - private Map<String, String> fieldsAsAttributes = new LinkedHashMap<>(); - - private Map<String, SubtypeField> fieldsWithSubtypes = new LinkedHashMap<>(); - private Map<String, SubtypeFieldWithPrefix> prefixedFieldsWithSubtypes = new LinkedHashMap<>(); - - private Set<String> fieldsNotRendered = new LinkedHashSet<>(); - private Set<String> fieldsRendered = new LinkedHashSet<>(); - - /** - * Tells to format a field as an attribute of the hit's tag. - * - * For instance, field 'query-latency' could be rendered as an attribute 'latency' by - * invoking {@code formatFieldAsAttribute("query-latency", "latency")}. - * - * Output would be: - * <pre> - * <hit latency="100"></hit> - * </pre> - * instead of: - * <pre> - * <hit><latency>100</latency></hit> - * </pre> - */ - public void formatFieldAsAttribute(String fieldName, String attributeName) { - fieldsAsAttributes.put(fieldName, attributeName); - } - - public Set<Map.Entry<String, String>> fieldsAsAttributes() { - return Collections.unmodifiableSet(this.fieldsAsAttributes.entrySet()); - } - - public String getAttributeName(String fieldName) { - return this.fieldsAsAttributes.get(fieldName); - } - - /** - * Tells to format a field using a subtype. A subtype is used when there is kind of a grouping - * for a set of fields. - * - * For instance, fields 'latency-connect', 'latency-finish' all belong to the same 'latency' logical group. - * So invoking {@code formatFieldWithSubtype("latency-connect", "latency", "type", "connect"}, - * {@code formatFieldWithSubtype("latency-finish", "latency", "type", "connect"} and so on, - * allows to have a common 'latency' tag name for all fields of the same kind. - * Note that it does no collapsing on tags. - * - * Output would be: - * <pre> - * <latency type="connect">50</latency> - * <latency type="finish">250</latency> - * </pre> - * Instead of: - * <pre> - * <hit> - * <latency-connect>50</latency-connect> - * <latency-finish>50</latency-finish> - * </pre> - */ - public void formatFieldWithSubtype(String fieldName, String tagName, String typeAttributeName, String typeAttributeValue) { - SubtypeField names = new SubtypeField(); - names.attributeName = typeAttributeName; - names.attributeValue = typeAttributeValue; - names.tagName = tagName; - fieldsWithSubtypes.put(fieldName, names); - } - - public SubtypeField getSubtype(String fieldName) { - return this.fieldsWithSubtypes.get(fieldName); - } - - /** - * Same as {@link #formatFieldWithSubtype(String, String, String, String)} except that fields - * are selected based on the beginning of their name and the type attribute value is deduced - * from the rest of their name. So this may select many fields instead of only one. - * Invoking {@code formatFieldWithSubtype("latency-", "latency", "type")} only once allows to have a common 'latency' - * tag name for all fields that start with 'latency-'. Type attribute value will be 'start' for field 'latency-start'. - * Note that it does no collapsing on tags. - * - * This is mostly used when you don't know all field names ahead. - * - * Output would be: - * <pre> - * <latency type="connect">50</latency> - * <latency type="finish">250</latency> - * </pre> - * Instead of: - * <pre> - * <hit> - * <latency-connect>50</latency-connect> - * <latency-finish>50</latency-finish> - * </pre> - * - * Note: don't use this with prefixes that start with a common substring (e.g. 'http', 'http_proxy'), I can tell you it just won't work. - */ - public void formatFieldWithSubtype(String fieldNamePrefix, String tagName, String typeAttributeName) { - SubtypeFieldWithPrefix names = new SubtypeFieldWithPrefix(); - names.attributeName = typeAttributeName; - names.tagName = tagName; - names.prefixLength = fieldNamePrefix.length(); - prefixedFieldsWithSubtypes.put(fieldNamePrefix, names); - } - - public SubtypeFieldWithPrefix getSubtypeWithPrefix(String fieldName) { - for(Map.Entry<String, SubtypeFieldWithPrefix> e : this.prefixedFieldsWithSubtypes.entrySet()) { - if(fieldName.startsWith(e.getKey())) - return e.getValue(); - } - return null; - } - - /** - * Tells whether a field should be rendered. - * - * @see #setFieldNotToRender(String) - * @see #setFieldToRender(String) - */ - public boolean shouldRenderField(String fieldName) { - if(fieldName == null) - return false; - if (fieldName.startsWith("$")) { - return false; - } - if(!this.fieldsRendered.isEmpty()) - return this.fieldsRendered.contains(fieldName); - return !this.fieldsNotRendered.contains(fieldName); - } - - /** - * Tells a field should be rendered. - * - * <p> - * Note: if at least one field is set to render, then only - * these fields should be rendered. Use {@link #setFieldNotToRender(String)} - * to only exclude specific fields. - */ - public void setFieldToRender(String fieldName) { - this.fieldsRendered.add(fieldName); - } - - /** - * Tells a field should not be rendered. - * - * <p> - * Note: all other fields should be rendered. Use {@link #setFieldToRender(String)} - * to only include specific fields. - */ - public void setFieldNotToRender(String fieldName) { - this.fieldsNotRendered.add(fieldName); - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/GenericTemplateSet.java b/container-search/src/main/java/com/yahoo/prelude/templates/GenericTemplateSet.java deleted file mode 100644 index 95875d0dd1f..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/GenericTemplateSet.java +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import com.yahoo.protect.Validator; -import com.yahoo.search.Query; - -import java.io.Writer; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Properties; - -/** - * Superclass of a set of templates for rendering (serializing) results - * - * @deprecated use a renderer instead - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -public class GenericTemplateSet { - - public static final String DEFAULT_MIMETYPE = "text/xml"; - public static final String DEFAULT_ENCODING = "utf-8"; - - /** Templates */ - private HashMap<String, Template<? extends Writer>> templates; - - /** The text MIME subtype this template returns, xml, plain or html */ - private String mimeType; - - /** The charset encoding this template should have */ - private String encoding; - - private String boldOpenTag = null; - private String boldCloseTag = null; - private String separatorTag = null; - - /** - * Document summary class for this template - */ - private String summaryClass = null; - - /** - * The unique name of this template set - */ - private final String name; - - /** - * Creates a template set containing no templates - */ - public GenericTemplateSet(String name, String mimeType, String encoding) { - this.mimeType = mimeType; - this.encoding = encoding; - this.name = name; - - templates = new LinkedHashMap<>(); - } - - - public String getName() { - return name; - } - - /** - * Returns the text MIME - */ - public String getMimeType() { return mimeType; } - - /** - * Returns the text encoding - */ - public String getEncoding() { return encoding; } - - /** Returns the encoding of the query, or the encoding given by the template if none is set */ - public final String getRequestedEncoding(Query query) { - String encoding = query.getModel().getEncoding(); - if (encoding != null) return encoding; - return getEncoding(); - } - - /** - * Returns the selected template - * - * @return the template to use, never null - */ - public Template<? extends Writer> getTemplate(String templateName) { - return templates.get(templateName); - } - - /** - * Sets the selected template - * - * @throws NullPointerException if the given template is null - */ - public void setTemplate(String templateName, Template<? extends Writer> template) { - templates.put(templateName,template); - } - - /** - * Sets the selected template - * - * @throws NullPointerException if the given template is null - */ - public void setTemplateNotNull(String templateName, Template<? extends Writer> template) { - Validator.ensureNotNull("Template "+templateName,template); - templates.put(templateName,template); - } - - - /** - * Sets the highligting marks for this template - * - * @param start the highlingting start mark - * @param end the highlingting end mark - * @param sep the highlingting separator mark - */ - public void setHighlightTags(String start, String end, String sep) { - boldOpenTag = start; - boldCloseTag = end; - separatorTag = sep; - } - - // may return null - public String getBoldOpenTag() { - return boldOpenTag; - } - - // may return null - public String getBoldCloseTag() { - return boldCloseTag; - } - - // may return null - public String getSeparatorTag() { - return separatorTag; - } - - - /** - * Set the default summary class to use with this template. - */ - public void setSummaryClass(String summaryClass) { - this.summaryClass = summaryClass; - } - - /** - * Type safe accessor to get the default document summary class for this - * template set. This is also here to insulate the rest of the code - * against changes in the naming of the properties in the property file. - */ - public String getSummaryClass() { - if (summaryClass != null && ! summaryClass.isEmpty()) { - return summaryClass; - } else { - return null; - } - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/HitContext.java b/container-search/src/main/java/com/yahoo/prelude/templates/HitContext.java deleted file mode 100644 index 5ef28d6f6c6..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/HitContext.java +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import com.yahoo.prelude.hitfield.HitField; -import com.yahoo.prelude.hitfield.JSONString; -import com.yahoo.prelude.hitfield.XMLString; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.StructuredData; -import com.yahoo.text.XML; - -import java.util.Collection; -import java.util.HashSet; -import java.util.Set; - -/** - * A context providing all the fields of a hit, and falls back to MapContext behavior for all other keys. - * - * @author Tony Vaagenes - * @deprecated use a Renderer instead - */ -@SuppressWarnings("deprecation") -// TODO: Remove on Vespa 7 -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -public class HitContext extends Context { - - private final Hit hit; - private final Context fallbackContext; - - public HitContext(Hit hit, Context fallbackContext) { - this.hit = hit; - this.fallbackContext = fallbackContext; - } - - @Override - public Object put(String key, Object value) { - return fallbackContext.put(key, value); - } - - @Override - public Object get(String key) { - Object value = normalizedHitProperty(key); - return value != null ? - value : - fallbackContext.get(key); - } - - @Override - public Object remove(Object key) { - return fallbackContext.remove(key); - } - - @Override - public Collection<? extends Object> getKeys() { - Set<Object> keys = new HashSet<>(fallbackContext.getKeys()); - keys.addAll(hit.fieldKeys()); - return keys; - } - - @Override - public void setBoldOpenTag(String boldOpenTag) { - fallbackContext.setBoldOpenTag(boldOpenTag); - } - - @Override - public void setBoldCloseTag(String boldCloseTag) { - fallbackContext.setBoldCloseTag(boldCloseTag); - } - - @Override - public void setSeparatorTag(String separatorTag) { - fallbackContext.setSeparatorTag(separatorTag); - } - - @Override - public String getBoldOpenTag() { - return fallbackContext.getBoldOpenTag(); - } - - @Override - public String getBoldCloseTag() { - return fallbackContext.getBoldCloseTag(); - } - - @Override - public String getSeparatorTag() { - return fallbackContext.getSeparatorTag(); - } - - @Override - //TVT: TODO: Make this package private again. - public boolean isUtf8Output() { - return fallbackContext.isUtf8Output(); - } - - @Override - //TODO: TVT: make this package private again - public void setUtf8Output(boolean utf8Output) { - fallbackContext.setUtf8Output(utf8Output); - } - - @Override - public void setXmlEscape(boolean xmlEscape) { - fallbackContext.setXmlEscape(xmlEscape); - } - - @Override - public boolean getXmlEscape() { - return fallbackContext.getXmlEscape(); - } - - @Override - protected Object normalizeValue(Object value) { - return fallbackContext.normalizeValue(value); - } - - private Object normalizedHitProperty(String key) { - Object value = hit.getField(key); - return value == null ? - null : - normalizeHitFieldValue(value); - } - - private Object normalizeHitFieldValue(Object value) { - if (value instanceof HitField) { - HitField hf = (HitField) value; - if (getXmlEscape()) { - return hf.quotedContent(getBoldOpenTag(), - getBoldCloseTag(), - getSeparatorTag(), - true); - } else { - return hf.getContent(getBoldOpenTag(), - getBoldCloseTag(), - getSeparatorTag()); - } - } else if (value instanceof StructuredData) { - return value.toString(); - } else if (value instanceof XMLString || value instanceof JSONString) { - return value.toString(); - } else if (getXmlEscape()) { - return XML.xmlEscape(value.toString(), true, null); - } else { - return value.toString(); - } - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/LogExceptionUserTemplateDelegator.java b/container-search/src/main/java/com/yahoo/prelude/templates/LogExceptionUserTemplateDelegator.java deleted file mode 100644 index 7696790897e..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/LogExceptionUserTemplateDelegator.java +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import com.yahoo.log.LogLevel; -import com.yahoo.yolean.Exceptions; - -import java.io.IOException; -import java.io.Writer; -import java.util.Properties; -import java.util.logging.Logger; - -/** - * Delegates to another UserTemplate, but handles any exceptions(except IOException) by logging them. - * - * @author Tony Vaagenes - * @deprecated use a renderer instead - */ -@SuppressWarnings("deprecation") -// TODO: Remove on Vespa 7 -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -public class LogExceptionUserTemplateDelegator<T extends Writer> extends UserTemplate<T> { - - private static Logger log = Logger.getLogger(LogExceptionUserTemplateDelegator.class.getName()); - private final UserTemplate<T> delegate; - - public LogExceptionUserTemplateDelegator(UserTemplate<T> delegate) { - super(LogExceptionUserTemplateDelegator.class.getSimpleName()); - this.delegate = delegate; - } - - @Override - public Context createContext() { - return delegate.createContext(); - } - - @Override - public T wrapWriter(Writer writer) { - return delegate.wrapWriter(writer); - } - - @Override - public boolean isDefaultTemplateSet() { - return delegate.isDefaultTemplateSet(); - } - - @Override - public String getSummaryClass() { - return delegate.getSummaryClass(); - } - - @Override - public String getBoldOpenTag() { - return delegate.getBoldOpenTag(); - } - - @Override - public String getBoldCloseTag() { - return delegate.getBoldCloseTag(); - } - - @Override - public String getSeparatorTag() { - return delegate.getSeparatorTag(); - } - - @Override - public void setSummaryClass(String summaryClass) { - delegate.setSummaryClass(summaryClass); - } - - @Override - public void setHighlightTags(String start, String end, String sep) { - delegate.setHighlightTags(start, end, sep); - } - - @Override - public String getName() { - return delegate.getName(); - } - - @Override - public String getMimeType() { - return delegate.getMimeType(); - } - - @Override - public String getEncoding() { - return delegate.getEncoding(); - } - - @Override - public Template<T> getTemplate(String templateName) { - throw new UnsupportedOperationException(); - } - - @Override - public void setTemplate(String templateName, Template<? extends Writer> template) { - throw new UnsupportedOperationException(); - } - - @Override - public void setTemplateNotNull(String templateName, Template<? extends Writer> template) { - throw new UnsupportedOperationException(); - } - - /*** Template - - @Override - public void <methodName>(Context context, T writer) throws IOException { - try { - delegate.<methodName>(context, writer); - } catch (Exception e) { - handleException(e); - } - } - - ***/ - - /*** Begin expanded template for - header, footer, hit, hitFooter, error, noHits, queryContext, - Thanks java, for giving me the opportunely to use copy-paste ***/ - - - @Override - public void header(Context context, T writer) throws IOException { - try { - delegate.header(context, writer); - } catch (Exception e) { - handleException(e); - } - } - - @Override - public void footer(Context context, T writer) throws IOException { - try { - delegate.footer(context, writer); - } catch (Exception e) { - handleException(e); - } - } - - @Override - public void hit(Context context, T writer) throws IOException { - try { - delegate.hit(context, writer); - } catch (Exception e) { - handleException(e); - } - } - - @Override - public void hitFooter(Context context, T writer) throws IOException { - try { - delegate.hitFooter(context, writer); - } catch (Exception e) { - handleException(e); - } - } - - @Override - public void error(Context context, T writer) throws IOException { - try { - delegate.error(context, writer); - } catch (Exception e) { - handleException(e); - } - } - - @Override - public void noHits(Context context, T writer) throws IOException { - try { - delegate.noHits(context, writer); - } catch (Exception e) { - handleException(e); - } - } - - @Override - public void queryContext(Context context, T writer) throws IOException { - try { - delegate.queryContext(context, writer); - } catch (Exception e) { - handleException(e); - } - } - - /*** End expanded template. ***/ - - private void handleException(Exception e) throws IOException { - if (e instanceof IOException) { - throw (IOException) e; - } else { - log.log(LogLevel.WARNING, "Exception thrown in " + getName() - + ": " + Exceptions.toMessageString(e), e); - } - } - - UserTemplate<T> getDelegate() { - return delegate; - } -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/MapContext.java b/container-search/src/main/java/com/yahoo/prelude/templates/MapContext.java deleted file mode 100644 index c08c9701741..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/MapContext.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; - -/** - * A context having a map as secondary storage - * @deprecated use a Renderer instead - */ -@SuppressWarnings("deprecation") -// TODO: Remove on Vespa 7 -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -public class MapContext extends Context { - - private Map<String, Object> map = new LinkedHashMap<>(); - - @Override - public Object get(String key) { - return normalizeValue(map.get(key)); - } - - public Object put(String name, Object value) { - return map.put(name, value); - } - - public Object remove(Object name) { - return map.remove(name); - } - - @Override - public Collection<? extends Object> getKeys() { - return map.keySet(); - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/PageTemplateSet.java b/container-search/src/main/java/com/yahoo/prelude/templates/PageTemplateSet.java deleted file mode 100644 index cac06011679..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/PageTemplateSet.java +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import com.yahoo.search.Result; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import com.yahoo.text.XMLWriter; - -import java.io.IOException; -import java.io.Writer; - -/** - * A template implementing the 'page' format. - * This is a variant of the tiled template set - see that class for details. - * - * @author bratseth - * @deprecated use a Renderer instead - */ -@SuppressWarnings("deprecation") -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class PageTemplateSet extends TiledTemplateSet { - - public PageTemplateSet() { - super("page"); - } - - @Override - /** Uses an XML writer in this */ - public XMLWriter wrapWriter(Writer writer) { return new XMLWriter(super.wrapWriter(writer)); } - - @Override - public void header(Context context,XMLWriter writer) throws IOException { - Result result=(Result)context.get("result"); - writer.xmlHeader(getRequestedEncoding(result.getQuery())); - writer.openTag("page").attribute("version","1.0").attribute("layout",result.hits().getField("layout")); - renderCoverageAttributes(result.getCoverage(false), writer); - writer.closeStartTag(); - renderSectionContent(result.hits(),writer); - } - - @Override - public void footer(Context context,XMLWriter writer) throws IOException { - if (writer.isIn("content")) - writer.closeTag(); - super.footer(context,writer); - } - - @Override - protected void renderSection(HitGroup hit, XMLWriter writer) throws IOException { - writer.openTag("section"); - writer.attribute("id",hit.getDisplayId()); - writer.attribute("layout",hit.getField("layout")); - writer.attribute("region",hit.getField("region")); - writer.closeStartTag(); - renderSectionContent(hit,writer); - } - - @Override - public void hit(Context context, XMLWriter writer) throws IOException { - Hit hit = (Hit) context.get("hit"); - if (!hit.isMeta() && !writer.isIn("content")) - writer.openTag("content"); - super.hit(context,writer); - } - - @Override - public void hitFooter(Context context, XMLWriter writer) throws IOException { - if (writer.isIn("content")) - writer.closeTag(); - super.hitFooter(context, writer); - } - - public String toString() { return "page template"; } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/SearchRendererAdaptor.java b/container-search/src/main/java/com/yahoo/prelude/templates/SearchRendererAdaptor.java deleted file mode 100644 index 467cd615ebd..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/SearchRendererAdaptor.java +++ /dev/null @@ -1,259 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import com.yahoo.prelude.fastsearch.GroupingListHit; -import com.yahoo.search.Result; -import com.yahoo.search.query.context.QueryContext; -import com.yahoo.search.rendering.Renderer; -import com.yahoo.search.result.*; -import com.yahoo.search.result.ErrorHit; -import com.yahoo.processing.request.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.text.XMLWriter; - -import java.io.*; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; - -/** - * Renders a search result using the old templates API. - * - * @author Tony Vaagenes - * @deprecated do not use - */ -@SuppressWarnings({ "rawtypes", "deprecation", "unchecked" }) -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -// TODO: Remove on Vespa 7 -public final class SearchRendererAdaptor extends Renderer { - - private final LogExceptionUserTemplateDelegator templates; - - //Per instance members, must be created at rendering time, not construction time due to cloning. - private Context context; - - public SearchRendererAdaptor(UserTemplate userTemplate) { - templates = new LogExceptionUserTemplateDelegator(userTemplate); - } - - @Override - public void init() { - super.init(); - context = templates.createContext(); - } - - /** A legacy test utility - do not use. */ - public static void callRender(OutputStream stream, Result result) throws IOException { - Renderer rendererAdaptor = new SearchRendererAdaptor(result.getTemplating().getTemplates()); - rendererAdaptor.init(); - result.getTemplating().setRenderer(rendererAdaptor); - rendererAdaptor.render(stream, result, result.getQuery().getModel().getExecution(), result.getQuery()); - } - - @Override - public String getEncoding() { - return templates.getEncoding(); - } - - @Override - public String getMimeType() { - return templates.getMimeType(); - } - - @Override - public String getDefaultSummaryClass() { - return templates.getSummaryClass(); - } - - /** - * Renders this result - */ - public void render(Writer writer, Result result) throws java.io.IOException { - Writer wrappedWriter = wrapWriter(writer); - - beginResult(wrappedWriter, result); - - if (result.hits().getError() != null || result.hits().getQuery().errors().size() > 0) { - error(wrappedWriter, Collections.unmodifiableCollection( - all(result.hits().getQuery().errors(), result.hits().getError()))); - } - - if (result.getConcreteHitCount() == 0) { - emptyResult(wrappedWriter, result); - } - - if (result.getContext(false) != null) { - queryContext(wrappedWriter, result.getContext(false)); - } - - renderHitGroup(wrappedWriter, result.hits(), result.hits().getQuery().getOffset() + 1); - - endResult(wrappedWriter, result); - } - - - private <T> Collection<T> all(Collection<T> collection, T extra) { - Collection<T> result = new ArrayList<>(collection); - result.add(extra); - return result; - } - - - public Writer wrapWriter(Writer writer) { - return templates.wrapWriter(writer); - } - - - public void beginResult(Writer writer, Result result) throws IOException { - context.put("context", context); - context.put("result", result); - context.setBoldOpenTag(templates.getBoldOpenTag()); - context.setBoldCloseTag(templates.getBoldCloseTag()); - context.setSeparatorTag(templates.getSeparatorTag()); - - templates.header(context, writer); - } - - public void endResult(Writer writer, Result result) throws IOException { - templates.footer(context, writer); - } - - public void error(Writer writer, Collection<ErrorMessage> errorMessages) throws IOException { - templates.error(context, writer); - } - - - public void emptyResult(Writer writer, Result result) throws IOException { - templates.noHits(context, writer); - } - - public void queryContext(Writer writer, QueryContext queryContext) throws IOException { - templates.queryContext(context, writer); - } - - private void renderHitGroup(Writer writer, HitGroup hitGroup, int hitnumber) - throws IOException { - boolean defaultTemplate = templates.isDefaultTemplateSet(); - for (Hit hit : hitGroup.asList()) { - if (!defaultTemplate && hit instanceof ErrorHit) continue; // TODO: Stop doing this - - renderHit(writer, hit, hitnumber); - if (!hit.isAuxiliary()) - hitnumber++; - } - } - - - /** - * Renders this hit as xml. The default implementation will call the simpleRender() - * hook. If it returns true, nothing more is done, otherwise the - * given template set will be used for rendering. - * - * - * @param writer the writer to append this hit to - * @throws java.io.IOException if rendering fails - */ - public void renderHit(Writer writer, Hit hit, int hitno) throws IOException { - renderRegularHit(writer, hit, hitno); - } - - private void renderRegularHit(Writer writer, Hit hit, int hitno) throws IOException { - boolean renderedSimple = simpleRenderHit(writer, hit); - - if (renderedSimple) { - return; - } - - HitContext hitContext = new HitContext(hit, context); - hitContext.put("hit", hit); - hitContext.put("hitno", Integer.valueOf(hitno)); - hitContext.put("relevancy",hit.getRelevance()); - templates.hit(hitContext, writer); - - if (hit instanceof HitGroup) - renderHitGroup(writer, (HitGroup) hit, hitno); - - // Put these back - may have been changed by nested rendering - hitContext.put("hit", hit); - hitContext.put("hitno", Integer.valueOf(hitno)); - templates.hitFooter(hitContext, writer); - - - hitContext.remove("hit"); - hitContext.remove("hitno"); - } - - private boolean simpleRenderHit(Writer writer, Hit hit) throws IOException { - if (hit instanceof DefaultErrorHit) { - return simpleRenderDefaultErrorHit(writer, (DefaultErrorHit) hit); - } else if (hit instanceof GroupingListHit) { - return true; - } else { - return false; - } - } - - public static boolean simpleRenderDefaultErrorHit(Writer writer, ErrorHit defaultErrorHit) throws IOException { - XMLWriter xmlWriter=(writer instanceof XMLWriter) ? (XMLWriter)writer : new XMLWriter(writer,10,-1); - xmlWriter.openTag("errordetails"); - for (Iterator i = defaultErrorHit.errorIterator(); i.hasNext();) { - ErrorMessage error = (ErrorMessage) i.next(); - renderMessageDefaultErrorHit(xmlWriter, error); - } - xmlWriter.closeTag(); - return true; - } - - public static void renderMessageDefaultErrorHit(XMLWriter writer, ErrorMessage error) throws IOException { - writer.openTag("error"); - if (error instanceof com.yahoo.search.result.ErrorMessage) - writer.attribute("source",((com.yahoo.search.result.ErrorMessage)error).getSource()); - writer.attribute("error",error.getMessage()); - writer.attribute("code",Integer.toString(error.getCode())); - writer.content(error.getDetailedMessage(),false); - if (error.getCause()!=null) { - writer.openTag("cause"); - writer.content("\n",true); - StringWriter stackTrace=new StringWriter(); - error.getCause().printStackTrace(new PrintWriter(stackTrace)); - writer.content(stackTrace.toString(),true); - writer.closeTag(); - } - writer.closeTag(); - } - - /** - * Renders this hit as XML, disregarding the given template. - * The main error will be rendered first, the all the following errors. - */ - public boolean simpleRenderErrorHit(Writer writer, com.yahoo.search.result.ErrorHit errorHit) throws IOException { - XMLWriter xmlWriter=(writer instanceof XMLWriter) ? (XMLWriter)writer : new XMLWriter(writer,10,-1); - xmlWriter.openTag("errordetails"); - for (Iterator i = errorHit.errorIterator(); i.hasNext();) { - ErrorMessage error = (ErrorMessage) i.next(); - rendererErrorHitMessageMessage(xmlWriter, errorHit, error); - } - xmlWriter.closeTag(); - return true; - } - - public static void rendererErrorHitMessageMessage(XMLWriter writer, com.yahoo.search.result.ErrorHit errorHit, ErrorMessage error) throws IOException { - writer.openTag("error"); - if (errorHit instanceof Hit) { - writer.attribute("source", ((Hit) errorHit).getSource()); - } - writer.attribute("error",error.getMessage()); - writer.attribute("code",Integer.toString(error.getCode())); - writer.content(error.getDetailedMessage(),false); - writer.closeTag(); - } - - /** - * For internal use only - */ - public UserTemplate getAdaptee() { - return templates.getDelegate(); - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/Template.java b/container-search/src/main/java/com/yahoo/prelude/templates/Template.java deleted file mode 100644 index 761eb1db562..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/Template.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import java.io.Writer; - - -/** - * A template turns a template string and some state into - * an instantiated string. Add support for a particular - * template mechanism by subclassing this. - * - * @author bratseth - * @deprecated use a Renderer instead - */ -@SuppressWarnings("deprecation") -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -// TODO: Remove on Vespa 7 -public abstract class Template<T extends Writer> { - - /** - * Renders this template - * - * @param context the context to evaluate in - * @param writer the writer to render to - */ - public abstract void render(Context context,T writer) throws java.io.IOException; - - - /** - * Get template name - * - * @return template name - */ - public abstract String getName(); - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/TemplateSet.java b/container-search/src/main/java/com/yahoo/prelude/templates/TemplateSet.java deleted file mode 100644 index 65a5b4cacfd..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/TemplateSet.java +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import com.yahoo.search.Result; -import com.yahoo.search.result.Hit; -import com.yahoo.text.GenericWriter; -import com.yahoo.text.XMLWriter; - -import java.io.IOException; -import java.io.Writer; - -/** - * <p>A template set contains instances of the various templates - * required to render a result.</p> - * - * <p>Normal usage is to create an instance and populate it with templates, - * but this class also supports subclassing to refine the behaviour, - * like returning different templates for different hit types.</p> - * - * @author bratseth - * @deprecated use a renderer instead - */ -@SuppressWarnings("deprecation") -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -// TODO: Remove on Vespa 7 -public class TemplateSet<T extends Writer> extends UserTemplate<T> { - - private static final String queryContextTemplateName = "queryContext"; - - private static final DefaultTemplateSet defaultTemplateSet=new DefaultTemplateSet(); - - /** - * Creates a template set containing no templates - * - * @param name the unique name of this template set, used for - * refering to it by clients - */ - public TemplateSet(String name, - String mimeType, - String encoding) { - super(name, mimeType,encoding); - } - - /** - * Returns the default template set. This is a template set which renders in - * the default xml format - */ - public static UserTemplate<XMLWriter> getDefault() { - return defaultTemplateSet; - } - - /** - * Returns the result header template - * - * @param result the result which will use the template - * @return the template to use, never null - */ - @SuppressWarnings("unchecked") - public Template<T> getHeader(Result result) { return (Template<T>) getTemplate("header"); } - - /** - * Sets the header template - * - * @param header the template to use for rendering getHeaders - * @throws NullPointerException if the given template is null - */ - public void setHeader(Template<T> header) { - setTemplateNotNull("header",header); - } - - /** - * Returns the result footer template - * - * @param result the result which will use the template - * @return the template to use, never null - */ - @SuppressWarnings("unchecked") - public Template<T> getFooter(Result result) { return (Template<T>) getTemplate("footer"); } - - /** - * Sets the footer template - * - * @param footer the template to use for rendering footers - * @throws NullPointerException if the given template is null - */ - public void setFooter(Template<T> footer) { - setTemplateNotNull("footer",footer); - } - - /** - * Returns the empty body template - * - * @param result the result which will use the template - * @return the template to use, never null - */ - @SuppressWarnings("unchecked") - public Template<T> getNohits(Result result) { return (Template<T>) getTemplate("nohits"); } - - - /** - * @return the template for rendering the query context, never null - */ - @SuppressWarnings("unchecked") - public Template<T> getQueryContext(Result result) { - return (Template<T>) getTemplate(queryContextTemplateName); - } - - /** - * @param template The template to be used for rendering query contexts, never null. - */ - public void setQueryContext(Template<T> template) { - setTemplateNotNull(queryContextTemplateName, template); - } - - /** - * Sets the nohits template - * - * @param nohits the template to use for rendering empty results - * @throws NullPointerException if the given template is null - */ - public void setNohits(Template<T> nohits) { - setTemplateNotNull("nohits",nohits); - } - - /** - * Returns the error body template - * - * @param result the result which will use the template - * @return the template to use, never null - */ - @SuppressWarnings("unchecked") - public Template<T> getError(Result result) { return (Template<T>) getTemplate("error"); } - - /** - * Sets the error template - * - * @param error the template to use for rendering errors - * @throws NullPointerException if the given template is null - */ - public void setError(Template<T> error) { - setTemplateNotNull("error",error); - } - - /** - * Returns the hit template - * - * @param resultHit the hit which will use the template - * @return the template to use, never null - */ - @SuppressWarnings("unchecked") - public Template<T> getHit(Hit resultHit) { return (Template<T>) getTemplate("hit"); } - - /** - * Sets the hit template - * - * @param hit the template to use for rendering hits - * @throws NullPointerException if the given template is null - */ - public void setHit(Template<T> hit) { - setTemplateNotNull("hit",hit); - } - - /** - * Returns the hit footer template - * - * @param hit the hit which will use the template - * @return the template to use, or null if no hit footer is used - */ - @SuppressWarnings("unchecked") - public Template<T> getHitFooter(Hit hit) { return (Template<T>) getTemplate("hitfooter"); } - - public String toString() { - return "template set " + getName() + " of type " + getMimeType() + - " [header=" + getTemplate("header") + - ",footer=" + getTemplate("footer") + - ",nohits=" + getTemplate("nohits") + - ",error=" + getTemplate("error") + - ",hit=" + getTemplate("hit") + "]"; - } - - @Override - public void header(Context context, T writer) throws IOException { - getHeader(null).render(context, writer); - } - - @Override - public void footer(Context context, T writer) throws IOException { - getFooter(null).render(context, writer); - } - - @Override - public void hit(Context context, T writer) throws IOException { - getHit(null).render(context, writer); - } - - @Override - public void error(Context context, T writer) throws IOException { - getError(null).render(context, writer); - } - - @Override - public void hitFooter(Context context, T writer) throws IOException { - Template<T> hitFooter = getHitFooter(null); - if (hitFooter != null) - hitFooter.render(context, writer); - } - - @Override - public void noHits(Context context, T writer) throws IOException { - getNohits(null).render(context, writer); - } - - @Override - public void queryContext(Context context, T writer) throws IOException { - getQueryContext(null).render(context, writer); - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/TiledTemplateSet.java b/container-search/src/main/java/com/yahoo/prelude/templates/TiledTemplateSet.java deleted file mode 100644 index 2fa6e14019e..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/TiledTemplateSet.java +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import com.yahoo.prelude.hitfield.HitField; -import com.yahoo.prelude.hitfield.JSONString; -import com.yahoo.prelude.hitfield.XMLString; -import com.yahoo.prelude.templates.FormattingOptions.SubtypeFieldWithPrefix; -import com.yahoo.search.Result; -import com.yahoo.search.pagetemplates.model.Renderer; -import com.yahoo.search.pagetemplates.model.Source; -import com.yahoo.search.pagetemplates.result.SectionHitGroup; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import com.yahoo.search.result.StructuredData; -import com.yahoo.text.XML; -import com.yahoo.text.XMLWriter; - -import java.io.IOException; -import java.io.Writer; -import java.util.Iterator; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * A template set which implements the 'tiled' format. - * - * This template implementation requires a few rules to be observed for it to work properly: - * <ul> - * <li>As hit fields are rendered as XML tag names, their name must be compatible with XML tag names.</li> - * <li>Results sections, meta section, provider tags are rendered based on hits having specific types (as in {@link Hit#types()}, - * see table below for a list of hit types that are needed in order for hits to render properly.</li> - * <li>Some fields inside hits corresponding to provider tags (/result/meta/provider) are formatted in a specific way, see provider fields formatting options - * below. Other fields are rendered the usual way.</li> - * </ul> - * - * <p>Hit types required for proper rendering</p> - * <table summary="Hit types required for proper rendering"> - * <tr><td>XML tag path</td><td>Required hit type</td></tr> - * <tr><td>/result/section</td><td>A hit group and have a "section" type</td></tr> - * <tr><td>/result/meta</td><td>A hit group and have a "meta" type</td></tr> - * <tr><td>/result/meta/provider</td><td>A hit that has a "logging" type</td></tr> - * </table> - * - * <p>Provider fields formatting options</p> - * <table summary="Provider fields formatting options"> - * <tr><td>Field</td><td>Formatting</td><td>Field type</td></tr> - * <tr><td>provider</td><td>name attribute of <provider> tag</td><td>Provided by container</td></tr> - * <tr><td>scheme</td><td>scheme attribute of <provider> tag</td><td>Provided by container</td></tr> - * <tr><td>host</td><td>host attribute of <provider> tag</td><td>Provided by container</td></tr> - * <tr><td>port</td><td>port attribute of <provider> tag</td><td>Provided by container</td></tr> - * <tr><td>path</td><td>path attribute of <provider> tag</td><td>Provided by container</td></tr> - * <tr><td>status</td><td>result attribute of <provider> tag</td><td>Provided by container</td></tr> - * <tr><td>latency_connect</td><td><latency type="connect"> tag</td><td>Provided by container</td></tr> - * <tr><td>latency_start</td><td><latency type="start"> tag</td><td>Provided by container</td></tr> - * <tr><td>latency_finish</td><td><latency type="finish"> tag</td><td>Provided by container</td></tr> - * <tr><td>query_param_*</td><td><parameter name="..."> tag</td><td>Provided by container</td></tr> - * <tr><td>header_*</td><td><header name="..."> tag</td><td>Provided by container</td></tr> - * <tr><td>response_header_*</td><td><response-header name="..."> tag</td><td>Provided by container</td></tr> - * <tr><td>count_first</td><td><count type="first"> tag</td><td>Provided by container</td></tr> - * <tr><td>count_last</td><td><count type="last"> tag</td><td>Provided by container</td></tr> - * <tr><td>count_total</td><td><count type="total"> tag</td><td>Provided by container</td></tr> - * <tr><td>count_deep</td><td><count type="deep"> tag</td><td>Provided by container</td></tr> - * <tr><td>queryattrs_xorronum</td><td><queryattrs name="xorronum"> tag</td><td>Provided by YST searcher</td></tr> - * <tr><td>queryattrs_RankFeaturesRewriterAttr</td><td><queryattrs name="RankFeaturesRewriterAttr"> tag</td><td>Provided by YST searcher</td></tr> - * <tr><td>queryattrs_intlannotator</td><td><queryattrs name="intlannotator"> tag</td><td>Provided by YST searcher</td></tr> - * <tr><td>queryattrs_category</td><td><queryattrs name="category"> tag</td><td>Provided by YST searcher</td></tr> - * <tr><td>wordcounts_*</td><td><wordcounts word="..."> tag</td><td>Provided by YST searcher</td></tr> - * </table> - * - * @author bratseth - * @author laboisse - * @deprecated use a Renderer instead - */ -@SuppressWarnings("deprecation") -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class TiledTemplateSet extends DefaultTemplateSet { - - private FormattingOptions hitOptionsForProvider; - private FormattingOptions hitOptions; - - public TiledTemplateSet() { - this("tiled"); - } - - public TiledTemplateSet(String templateName) { - super(templateName); - - // Define formatting options that will be used by various rendering methods - hitOptions = new FormattingOptions(); - // Render provider field as an attribute, not as a regular field - hitOptions.formatFieldAsAttribute("provider", "provider"); - hitOptions.setFieldNotToRender("provider"); - - - // Define formatting options that will be used by various rendering methods, for /result/meta/provider tags - hitOptionsForProvider = new FormattingOptions(); - hitOptionsForProvider.formatFieldAsAttribute("provider", "name"); // Provider name is rendered a provider/@name - // hitOptionsForProvider.formatFieldAsAttribute("uri", "query"); // FIXME Issue with attribute formatting, keeping as regular field for now - hitOptionsForProvider.formatFieldAsAttribute("scheme", "scheme"); - hitOptionsForProvider.formatFieldAsAttribute("host", "host"); - hitOptionsForProvider.formatFieldAsAttribute("port", "port"); - hitOptionsForProvider.formatFieldAsAttribute("path", "path"); - hitOptionsForProvider.formatFieldAsAttribute("status", "result"); - // Latency fields are not defined using prefixes as we know all the field names and prefixes are expensive - hitOptionsForProvider.formatFieldWithSubtype("latency_connect", "latency", "type", "connect"); - hitOptionsForProvider.formatFieldWithSubtype("latency_start", "latency", "type", "start"); - hitOptionsForProvider.formatFieldWithSubtype("latency_finish", "latency", "type", "finish"); - // Must use prefix for query parameters - hitOptionsForProvider.formatFieldWithSubtype("query_param_", "parameter", "name"); - // Must use prefix for getHeaders - hitOptionsForProvider.formatFieldWithSubtype("header_", "header", "name"); - // Must use prefix for response getHeaders - hitOptionsForProvider.formatFieldWithSubtype("response_header_", "response-header", "name"); - // Count fields are not defined using prefixes as we know all the field names and prefixes are expensive - hitOptionsForProvider.formatFieldWithSubtype("count_first", "count", "type", "first"); - hitOptionsForProvider.formatFieldWithSubtype("count_last", "count", "type", "last"); - hitOptionsForProvider.formatFieldWithSubtype("count_total", "count", "type", "total"); - hitOptionsForProvider.formatFieldWithSubtype("count_deep", "count", "type", "deep"); - - hitOptionsForProvider.formatFieldWithSubtype("queryattrs_xorronum", "queryattrs", "name", "xorronum"); - hitOptionsForProvider.formatFieldWithSubtype("queryattrs_RankFeaturesRewriterAttr", "queryattrs", "name", "RankFeaturesRewriterAttr"); - hitOptionsForProvider.formatFieldWithSubtype("queryattrs_intlannotator", "queryattrs", "name", "intlannotator"); - hitOptionsForProvider.formatFieldWithSubtype("queryattrs_category", "queryattrs", "name", "category"); - - hitOptionsForProvider.formatFieldWithSubtype("wordcounts_", "wordcounts", "word"); - // Provider field should not be rendered in logging hits as we already have <provider name="..."> - hitOptionsForProvider.setFieldNotToRender("provider"); - } - - @Override - /** Uses an XML writer in this template */ - public XMLWriter wrapWriter(Writer writer) { return new XMLWriter(super.wrapWriter(writer)); } - - @Override - public void header(Context context,XMLWriter writer) throws IOException { - Result result=(Result)context.get("result"); - writer.xmlHeader(getRequestedEncoding(result.getQuery())); - writer.openTag("result").attribute("version","1.0"); - writer.attribute("layout", result.hits().getField("layout")); - renderCoverageAttributes(result.getCoverage(false), writer); - writer.closeStartTag(); - renderSectionContent(result.hits(),writer); - } - - /** - * Augments default hit attributes rendering with formatting options. - * There's also a hacky part: if hit is actually a hit group, tries to use - * the 'type' field in place of the hit's type, to avoid having the 'group' hit type. - */ - @Override - protected void renderHitAttributes(Hit hit, XMLWriter writer) throws IOException { - if (hit instanceof HitGroup) { - String type = hit.types().stream().collect(Collectors.joining(" ")); - if ("group".equals(type)) - type = String.valueOf(hit.getField("type")); - writer.attribute("type", type); - } - else { - writer.attribute("type", hit.types().stream().collect(Collectors.joining(" "))); - } - - if (hit.getRelevance() != null) - writer.attribute("relevance", hit.getRelevance()); - writer.attribute("source", hit.getSource()); - - for (Map.Entry<String, String> attr : hitOptions.fieldsAsAttributes()) { - Object val = hit.getField(attr.getKey()); - if (val != null) - writer.attribute(attr.getValue(), String.valueOf(val)); - } - } - - @Override - protected void renderField(Context context, Hit hit, Map.Entry<String, Object> entry, XMLWriter writer) throws IOException { - String fieldName = entry.getKey(); - - if ( !shouldRenderField(hit, fieldName)) return; - - writer.openTag(fieldName); - renderFieldContent(context, hit, fieldName, writer); - writer.closeTag(); - } - - /** Renders all fields of the hit */ - @Override - protected void renderHitFields(Context context, Hit hit, XMLWriter writer) throws IOException { - renderId(hit.getId(), writer); - for (Iterator<Map.Entry<String, Object>> it = hit.fieldIterator(); it.hasNext(); ) { - Map.Entry<String, Object> entry = it.next(); - // Exclude fields that should not be rendered - if (hitOptions.shouldRenderField(entry.getKey())) - renderField(context, hit, entry, writer); - } - } - - @Override - protected boolean shouldRenderField(Hit hit, String fieldName) { - if (fieldName.equals("relevancy")) return false; - if (fieldName.equals("collapseId")) return false; - return true; - } - - /** - * Overrides {@link DefaultTemplateSet#hit(Context, Writer)} - * to print 'logging' type meta hits as /result/meta/provider tags. - * Fails back to {@code super.hit(context, writer)} in other cases. - */ - @Override - public void hit(Context context, XMLWriter writer) throws IOException { - Hit hit = (Hit) context.get("hit"); - if (hit.isMeta() && hit.types().contains("logging")) - renderProvider(context, hit, writer); - else - super.hit(context, writer); - } - - /** - * Overrides {@link DefaultTemplateSet#renderHitGroup(HitGroup, Context, XMLWriter)} - * for /result/section and /result/meta hit groups. - * Fails back to {@code super.renderHitGroup(hit, context, writer)} otherwise. - */ - @Override - protected void renderHitGroup(HitGroup hit, Context context, XMLWriter writer) throws IOException { - if (hit.types().contains("section")) { - renderSection(hit, writer); // Renders /result/section - } - else if (hit.types().contains("meta")) { - writer.openTag("meta"); // renders /result/meta - writer.closeStartTag(); - } - else { - super.renderHitGroup(hit, context, writer); - } - } - - /** - * Renders /result/section. - * Doesn't use {@link #renderHitAttributes(Hit, XMLWriter)}. - */ - protected void renderSection(HitGroup hit, XMLWriter writer) throws IOException { - writer.openTag("section"); - writer.attribute("id",hit.getDisplayId()); - writer.attribute("layout",hit.getField("layout")); - writer.attribute("region",hit.getField("region")); - writer.attribute("placement",hit.getField("placement")); // deprecated in 5.0 - writer.closeStartTag(); - renderSectionContent(hit,writer); - } - - protected void renderSectionContent(HitGroup hit,XMLWriter writer) throws IOException { - if (hit instanceof SectionHitGroup) { // render additional information - SectionHitGroup sectionGroup=(SectionHitGroup)hit; - for (Source source : sectionGroup.sources()) { - writer.openTag("source").attribute("url",source.getUrl()); - renderParameters(source.parameters(),writer); - writer.closeTag(); - } - for (Renderer renderer : sectionGroup.renderers()) { - writer.openTag("renderer").attribute("for",renderer.getRendererFor()).attribute("name",renderer.getName()); - renderParameters(renderer.parameters(),writer); - writer.closeTag(); - } - } - } - - private void renderParameters(Map<String,String> parameters,XMLWriter writer) throws IOException { - // Render content - for (Map.Entry<String, String> parameter : parameters.entrySet()) - writer.openTag("parameter").attribute("name",parameter.getKey()).content(parameter.getValue(),false).closeTag(); - } - - /** - * Renders /result/meta/provider. - * Uses {@link #renderProviderHitAttributes(Hit, XMLWriter)} instead of the default {@link #renderHitAttributes(Hit, XMLWriter)}. - * @see #renderProviderHitAttributes(Hit, XMLWriter) - * @see #renderProviderHitFields(Context, Hit, XMLWriter) - */ - protected void renderProvider(Context context, Hit hit, XMLWriter writer) - throws IOException { - writer.openTag("provider"); - renderProviderHitAttributes(hit, writer); - writer.closeStartTag(); - renderProviderHitFields(context, hit, writer); - } - - /** - * Specific hit attributes rendering for 'provider' meta hits under /result/meta. - */ - protected void renderProviderHitAttributes(Hit hit, XMLWriter writer) throws IOException { - // Browse through fields that should be rendered as attributes - for (Map.Entry<String, String> attr : hitOptionsForProvider.fieldsAsAttributes()) - writer.attribute(attr.getValue(),hit.getField(attr.getKey())); - } - - - /** - * Renders fields under /result/meta/provider. - * - * @see #renderProviderField(Context, Hit, java.util.Map.Entry, XMLWriter) - */ - protected void renderProviderHitFields(Context context, Hit hit, XMLWriter writer) - throws IOException { - renderId(hit.getId(), writer); - for (Iterator<Map.Entry<String, Object>> it = hit.fieldIterator(); it.hasNext(); ) { - Map.Entry<String, Object> entry = it.next(); - // Exclude fields that have already been rendered as attributes and - // fields that should not be rendered - if (hitOptionsForProvider.getAttributeName(entry.getKey()) == null - && hitOptionsForProvider.shouldRenderField(entry.getKey())) - renderProviderField(context, hit, entry, writer); - } - } - - /** - * Renders one field under /result/meta/provider. - */ - protected void renderProviderField(Context context, Hit hit, - Map.Entry<String, Object> entry, XMLWriter writer) throws IOException { - - String name = entry.getKey(); - FormattingOptions.SubtypeField subtypeField = hitOptionsForProvider.getSubtype(name); - if (subtypeField == null) - subtypeField = hitOptionsForProvider.getSubtypeWithPrefix(name); - - if (subtypeField != null) { - writer.openTag(subtypeField.tagName); - if (subtypeField.attributeValue != null) { - writer.attribute(subtypeField.attributeName,subtypeField.attributeValue); - } - else if (subtypeField instanceof SubtypeFieldWithPrefix) { - // This is a subtype field that was defined using a prefix - // get the remaining part of the field name - writer.attribute(subtypeField.attributeName, - name.substring(((SubtypeFieldWithPrefix)subtypeField).prefixLength)); - } - } else { - writer.openTag(name); - } - writer.escapedContent(asXML(hit.getField(name)),false).closeTag(); - } - - private String asXML(Object value) { - if (value == null) - return "(null)"; - else if (value instanceof HitField) - return ((HitField)value).quotedContent(false); - else if (value instanceof StructuredData || value instanceof XMLString || value instanceof JSONString) - return value.toString(); - else - return XML.xmlEscape(value.toString(), false, '\u001f'); - } - - public String toString() { return "tiled result template"; } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/UserTemplate.java b/container-search/src/main/java/com/yahoo/prelude/templates/UserTemplate.java deleted file mode 100644 index bcdf3311c1f..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/UserTemplate.java +++ /dev/null @@ -1,312 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates; - -import com.yahoo.io.ByteWriter; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.fastsearch.XMLField; -import com.yahoo.search.Result; -import com.yahoo.text.Utf8String; -import com.yahoo.text.XMLWriter; - -import java.io.IOException; -import java.io.Writer; -import java.util.logging.Logger; - - -/** - * A wrapper for a template set, suitable for subclassing. - * - * <p> - * A subclass of UserTemplate must implement header(), footer(), hit(), - * hitFooter(), error() and noHits(). - * - * @deprecated use a renderer instead - * @author Steinar Knutsen - */ -@SuppressWarnings("deprecation") -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -// TODO: Remove on Vespa 7 -public abstract class UserTemplate<T extends Writer> extends GenericTemplateSet { - - // & - private static final byte[] ampersand = new byte[] { 38, 97, 109, 112, 59 }; - - // < - private static final byte[] lessThan = new byte[] { 38, 108, 116, 59 }; - // > - private static final byte[] greaterThan = new byte[] { 38, 103, 116, 59 }; - - // \\u00 - private static final byte[] quotePrefix = new byte[] { 92, 117, 48, 48 }; - - private static final Logger log = Logger.getLogger(UserTemplate.class.getName()); - - /** - * The signature of this constructor is the one which is invoked - * in a production setting. - */ - public UserTemplate(String name, String mimeType, - String encoding) { - super(name, mimeType, encoding); - } - - public UserTemplate(String name) { - this(name, - DEFAULT_MIMETYPE, - DEFAULT_ENCODING - ); - } - - /** - * This is called once before each result is rendered using this template. - * The returned writer is used in all subsequent calls. Use this if another (wrapper) - * writer of the raw incoming writer is desired in the implementation of this template. - * The class of the returned type must be given as a type argument to the template class, - * to be able to implement methods taking this wrapper writer as the argument type. - * This default implementation returns an XMLWriter. - */ - @SuppressWarnings("unchecked") - public T wrapWriter(Writer writer) { - //FIXME: Hack - return (T) XMLWriter.from(writer, 10, -1); - } - - /** - * Creates a new context suitable for this template. - * The context may be reused for several evaluations, but not multiple - * concurrent evaluations - */ - public Context createContext() { - return new MapContext(); - } - - - /** - * For internal use only - * TODO: get rid of this method * - */ - public boolean isDefaultTemplateSet() { - return getClass().equals(TemplateSet.getDefault().getClass()); - } - - /** - * Render the result set header. - * - * <p> - * The result set is available in the context object under the name - * "result". - * - * @param context - * wrapper which will contain, among other thing, the result - * set instance - * @param writer - * the destination for rendering the result - * @throws IOException - * may be propagated from the writer - */ - public abstract void header(Context context, T writer) - throws IOException; - - /** - * Render the result set footer. - * - * <p> - * The result set is available in the context object under the name - * "result". - * - * @param context - * wrapper which will contain, among other thing, the result - * set instance - * @param writer - * the destination for rendering the result - * @throws IOException - * may be propagated from the writer - */ - public abstract void footer(Context context, T writer) - throws IOException; - - /** - * Render a single top level hit. - * - * <p> - * The result set is available in the context object under the name - * "result". The hit itself as "hit", the index of the hit as "hitno", and - * all the fields under their normal names. - * - * @param context - * wrapper which will contain, among other thing, the hit - * instance - * @param writer - * the destination for rendering the hit - * @throws IOException - * may be propagated from the writer - */ - public abstract void hit(Context context, T writer) throws IOException; - - /** - * Render a footer for a single top level hit. A typical implementation may - * do nothing. - * - * <p> - * The result set is available in the context object under the name - * "result". The hit itself as "hit", the index of the hit as "hitno", and - * all the fields under their normal names. - * - * @param context - * wrapper which will contain, among other thing, the hit - * instance - * @param writer - * the destination for rendering the hit - * @throws IOException - * may be propagated from the writer - */ - public abstract void hitFooter(Context context, T writer) - throws IOException; - - /** - * Render the error message for a result set. - * - * <p> - * The result set is available in the context object under the name - * "result". - * - * @param context - * wrapper which will contain, among other thing, main error - * and result set instances. - * @param writer - * the destination for rendering the hit - * @throws IOException - * may be propagated from the writer - */ - public abstract void error(Context context, T writer) - throws IOException; - - /** - * Invoked when the result set has no hits. - * - * <p> - * The result set is available in the context object under the name - * "result". - * - * @param context - * wrapper which will contain, among other thing, the result - * set instance - * @param writer - * the destination for rendering the hit - * @throws IOException - * may be propagated from the writer - */ - public abstract void noHits(Context context, T writer) - throws IOException; - - /** - * Override this to add custom rendering for the query context of the result. - * Only called when the query context is present. - * - * <p> - * The result set is available in the context object under the name - * "result". The query context is retrieved from the result by calling - * result.getQuery.getContext(false) - * - * @param context - * wrapper which will contain, among other things, the result - * set instance - * @param writer - * the destination for rendering the hit - * @throws IOException - * may be propagated from the writer - */ - public void queryContext(Context context, T writer) throws IOException { - Result result = (Result) context.get("result"); - result.getContext(false).render(writer); - } - - /** - * Dump UTF-8 byte array to writer, but escape low ASCII codes except - * TAB, NL and CR, and escape ampersand, less than and greater than. - * - * <p> - * It is presumed the writer is buffered (which is the case in normal - * result rendering), as the method may perform a large number of write - * operations. - * - * <p> - * public only for testing. - */ - public static void dumpAndXMLQuoteUTF8(ByteWriter writer, byte[] utf) throws java.io.IOException { - int startDump = 0; - - for (int i = 0; i < utf.length; ++i) { - byte b = utf[i]; - if (b < 0) { - // Not ASCII, above character 127 - // Don't try to do something smart with UNICODE characters, - // just pass them through. - } else if (b < 32) { - switch (b) { - case 9: - case 10: - case 13: - break; - default: - writer.append(utf, startDump, i - startDump); - startDump = i + 1; - quoteByte(writer, b); - break; - } - } else { - // printable ASCII - // quote special characters, otherwise do nothing - switch (b) { - // case 34: // double quote - // writer.append(utf, startDump, i - startDump); - // startDump = i + 1; - // writer.append(doubleQuote); - // break; - case 38: // ampersand - writer.append(utf, startDump, i - startDump); - startDump = i + 1; - writer.append(ampersand); - break; - case 60: // less than - writer.append(utf, startDump, i - startDump); - startDump = i + 1; - writer.append(lessThan); - break; - case 62: // greater than - writer.append(utf, startDump, i - startDump); - startDump = i + 1; - writer.append(greaterThan); - break; - } - } - } - if (startDump < utf.length) { - writer.append(utf, startDump, utf.length - startDump); - } - } - - /** - * If the field is available as a UTF-8 byte array, - * dump it to the writer. - */ - public static boolean dumpBytes(ByteWriter writer, - FastHit hit, - String fieldName) throws java.io.IOException { - return false; - } - - private static void quoteByte(ByteWriter writer, byte b) throws java.io.IOException { - byte[] quoted = new byte[2]; - writer.append(quotePrefix); - quoted[0] = (byte) ((b >>> 4) + 0x30); - if (quoted[0] > 0x39) { - quoted[0] = (byte) (quoted[0] + 7); - } - quoted[1] = (byte) ((b & 0x0f) + 0x30); - if (quoted[1] > 0x39) { - quoted[1] = (byte) (quoted[1] + 7); - } - writer.append(quoted); - } -} diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/package-info.java b/container-search/src/main/java/com/yahoo/prelude/templates/package-info.java deleted file mode 100644 index 2363a12c78d..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/templates/package-info.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -@PublicApi -package com.yahoo.prelude.templates; - -import com.yahoo.api.annotations.PublicApi; -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index bf0920a2aa5..73335037b94 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -139,7 +139,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { // The timeout to be used when dumping rank features private static final long dumpTimeout = (6 * 60 * 1000); // 6 minutes - private static final long defaultTimeout = 5000; + private static final long defaultTimeout = 500; /** The timeout of the query, in milliseconds */ private long timeout = defaultTimeout; @@ -527,17 +527,8 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { } /** - * @deprecated do not use - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public boolean requestHasProperty(String name) { - return httpRequest.hasProperty(name); - } - - /** * Returns the number of milliseconds to wait for a response from a search backend - * before timing it out. Default is 5000. + * before timing it out. Default is 500. * <p> * Note: If Ranking.RANKFEATURES is turned on, this is hardcoded to 6 minutes. * @@ -549,7 +540,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** * Sets the number of milliseconds to wait for a response from a search backend - * before time out. Default is 5000. + * before time out. Default is 500. */ public void setTimeout(long timeout) { if (timeout > 1000000000 || timeout < 0) @@ -639,24 +630,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { setHits(hits); } - /** - * This is ignored - compression is controlled at the network level. - * - * @deprecated this is ignored - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void setCompress(boolean ignored) { } - - /** - * Returns false. - * - * @deprecated this always returns false - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public boolean getCompress() { return false; } - /** Returns a string describing this query */ @Override public String toString() { @@ -885,7 +858,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { yql.append(" limit ").append(Integer.toString(getHits())); } } - if (getTimeout() != 5000L) { + if (getTimeout() != defaultTimeout) { yql.append(" timeout ").append(Long.toString(getTimeout())); } yql.append(';'); @@ -1009,22 +982,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { */ public HttpRequest getHttpRequest() { return httpRequest; } - /** - * Returns the unique and stable session id of this query. - * - * @param create if true this is created if not already set - * @return the session id of this query, or null if not set and create is false - * @deprecated use getSessionId() or getSessionId(serverId) instead - */ - @Deprecated - public SessionId getSessionId(boolean create) { - if ( ! create) return getSessionId(); - - if (requestId == null) - requestId = UniqueRequestId.next(); - return new SessionId(requestId, getRanking().getProfile()); - } - /** Returns the session id of this query, or null if none is assigned */ public SessionId getSessionId() { if (requestId == null) return null; diff --git a/container-search/src/main/java/com/yahoo/search/Result.java b/container-search/src/main/java/com/yahoo/search/Result.java index e70916369a3..364e60e6263 100644 --- a/container-search/src/main/java/com/yahoo/search/Result.java +++ b/container-search/src/main/java/com/yahoo/search/Result.java @@ -48,11 +48,6 @@ public final class Result extends com.yahoo.processing.Response implements Clone */ private ListMap<String,String> headers = null; - /** - * Result rendering infrastructure. - */ - private final Templating templating; - /** Creates a new Result where the top level hit group has id "toplevel" */ public Result(Query query) { this(query, new HitGroup("toplevel")); @@ -75,7 +70,6 @@ public final class Result extends com.yahoo.processing.Response implements Clone if (query.getRanking().getSorting() != null) { setHitOrderer(new HitSortOrderer(query.getRanking().getSorting())); } - templating = new Templating(this); } /** Create a result containing an error */ @@ -92,8 +86,6 @@ public final class Result extends com.yahoo.processing.Response implements Clone */ @SuppressWarnings("deprecation") public void mergeWith(Result result) { - if (templating.usesDefaultTemplate()) - templating.setRenderer(result.templating.getRenderer()); totalHitCount += result.getTotalHitCount(); deepHitCount += result.getDeepHitCount(); timeAccountant.merge(result.getElapsedTime()); @@ -228,7 +220,6 @@ public final class Result extends com.yahoo.processing.Response implements Clone resultClone.hits = hits.clone(); - resultClone.getTemplating().setRenderer(null); // TODO: Remove on Vespa 7 resultClone.setElapsedTime(new ElapsedTime()); return resultClone; } @@ -299,9 +290,7 @@ public final class Result extends com.yahoo.processing.Response implements Clone * @param hit * the hit to be analyzed */ - protected void traceExtraHitProperties(StringBuilder hitBuffer, Hit hit) { - return; - } + protected void traceExtraHitProperties(StringBuilder hitBuffer, Hit hit) { } /** Returns the context of this result - this is equal to getQuery().getContext(create) */ public QueryContext getContext(boolean create) { return getQuery().getContext(create); } @@ -337,17 +326,4 @@ public final class Result extends com.yahoo.processing.Response implements Clone return headers; } - /** - * The Templating object contains helper methods and data containers for - * result rendering. - * - * @return helper object for result rendering - * @deprecated use renderers - */ - // TODO: Remove on Vespa 7 - @Deprecated// OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) - public Templating getTemplating() { - return templating; - } - } diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java index 5de0c5eff74..ac0c8375f04 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java @@ -35,13 +35,6 @@ public class ClusterMonitor<T> { /** A map from Node to corresponding MonitoredNode */ private final Map<T, BaseNodeMonitor<T>> nodeMonitors = Collections.synchronizedMap(new java.util.LinkedHashMap<>()); - /** @deprecated use the constructor with just the first argument instead */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public ClusterMonitor(NodeManager<T> manager, String ignored) { - this(manager); - } - public ClusterMonitor(NodeManager<T> manager) { nodeManager = manager; monitorThread = new MonitorThread("search.clustermonitor"); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/RpcFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/RpcFillInvoker.java index fb72e8c77c9..578c447dfbe 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/RpcFillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/RpcFillInvoker.java @@ -110,7 +110,7 @@ public class RpcFillInvoker extends FillInvoker { Query query = result.getQuery(); String rankProfile = query.getRanking().getProfile(); byte[] serializedSlime = BinaryFormat - .encode(toSlime(rankProfile, summaryClass, query.getModel().getDocumentDb(), query.getSessionId(false), hits)); + .encode(toSlime(rankProfile, summaryClass, query.getModel().getDocumentDb(), query.getSessionId(), hits)); double timeoutSeconds = ((double) query.getTimeLeft() - 3.0) / 1000.0; Compressor.Compression compressionResult = resourcePool.compressor().compress(compression, serializedSlime); resourcePool.client().getDocsums(hits, node, compressionResult.type(), serializedSlime.length, compressionResult.data(), diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/RpcResourcePool.java b/container-search/src/main/java/com/yahoo/search/dispatch/RpcResourcePool.java index 8ab80ec17dd..bf626312b13 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/RpcResourcePool.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/RpcResourcePool.java @@ -3,7 +3,6 @@ package com.yahoo.search.dispatch; import com.google.common.collect.ImmutableMap; import com.yahoo.compress.Compressor; -import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.processing.request.CompoundName; @@ -52,9 +51,8 @@ public class RpcResourcePool { if (query.properties().getBoolean(dispatchSummaries, true) && ! searcher.summaryNeedsQuery(query) && query.getRanking().getLocation() == null - && ! searcher.getCacheControl().useCache(query) - && ! legacyEmulationConfigIsSet(documentDb)) { - + && ! searcher.getCacheControl().useCache(query)) + { return Optional.of(new RpcFillInvoker(this, documentDb)); } else { return Optional.empty(); @@ -66,17 +64,6 @@ public class RpcResourcePool { return new RpcFillInvoker(this, documentDb); } - private boolean legacyEmulationConfigIsSet(DocumentDatabase db) { - LegacyEmulationConfig config = db.getDocsumDefinitionSet().legacyEmulationConfig(); - if (config.forceFillEmptyFields()) - return true; - if (config.stringBackedFeatureData()) - return true; - if (config.stringBackedStructuredData()) - return true; - return false; - } - public Compressor compressor() { return compressor; } diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java deleted file mode 100644 index 1607d108722..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import java.util.Collections; - -import com.yahoo.component.ComponentId; -import com.yahoo.search.federation.ProviderConfig; -import com.yahoo.search.Result; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; - - -/** - * Superclass for http client searchers which depends on config. All this is doing is translating - * the provider and cache configurations to parameters which are passed upwards. - * - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class ConfiguredHTTPClientSearcher extends HTTPClientSearcher { - - /** Create this from a configuraton */ - public ConfiguredHTTPClientSearcher(final ComponentId id, final ProviderConfig providerConfig, Statistics manager) { - super(id, ConfiguredSearcherHelper.toConnectionList(providerConfig), new HTTPParameters(providerConfig), manager); - } - - /** Create an instance from direct parameters having a single connection. Useful for testing */ - public ConfiguredHTTPClientSearcher(String idString,String host,int port,String path, Statistics manager) { - super(new ComponentId(idString), Collections.singletonList(new Connection(host,port)),path, manager); - } - - /** Forwards to the next in chain fill(result,summaryName) */ - @Override - public void fill(Result result,String summaryName, Execution execution,Connection connection) { - execution.fill(result,summaryName); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java deleted file mode 100644 index 9f99a790df0..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.component.ComponentId; -import com.yahoo.search.federation.ProviderConfig; -import com.yahoo.search.cache.QrBinaryCacheConfig; -import com.yahoo.search.cache.QrBinaryCacheRegionConfig; -import com.yahoo.search.Result; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; - -import java.util.Collections; - - -/** - * Superclass for http provider searchers which depends on config. All this is doing is translating - * the provider and cache configurations to parameters which are passed upwards. - * - * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> - * @author bratseth - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class ConfiguredHTTPProviderSearcher extends HTTPProviderSearcher { - - /** Create this from a configuraton */ - public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig, Statistics manager) { - super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),new HTTPParameters(providerConfig), manager); - } - - /** Create this from a configuraton */ - public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig, - HTTPParameters parameters, Statistics manager) { - super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),parameters, manager); - } - - /** Create this from a configuraton with a configured cache */ - public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig, - final QrBinaryCacheConfig cacheConfig, - final QrBinaryCacheRegionConfig regionConfig, Statistics manager) { - super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),new HTTPParameters(providerConfig), manager); - configureCache(cacheConfig,regionConfig); - } - - /** Create this from a configuraton with a configured cache */ - public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig, - final QrBinaryCacheConfig cacheConfig, - final QrBinaryCacheRegionConfig regionConfig, HTTPParameters parameters, Statistics manager) { - super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),parameters, manager); - configureCache(cacheConfig,regionConfig); - } - - /** Create an instance from direct parameters having a single connection. Useful for testing */ - public ConfiguredHTTPProviderSearcher(String idString,String host,int port,String path, Statistics manager) { - super(new ComponentId(idString), Collections.singletonList(new Connection(host,port)),path, manager); - } - - /** Create an instance from direct parameters having a single connection. Useful for testing */ - public ConfiguredHTTPProviderSearcher(String idString,String host,int port,HTTPParameters parameters, Statistics manager) { - super(new ComponentId(idString), Collections.singletonList(new Connection(host,port)),parameters, manager); - } - - /** - * Override this to provider multi-phase result filling towards a backend. - * This default implementation does nothing. - */ - @Override - public void fill(Result result,String summaryName, Execution execution,Connection connection) { - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredSearcherHelper.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredSearcherHelper.java deleted file mode 100644 index 0fa235709f9..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredSearcherHelper.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import java.util.ArrayList; -import java.util.List; - -import com.yahoo.search.federation.ProviderConfig; - -/** - * Some static helper classes for configured*Searcher classes - * - * @author bratseth - */ -// TODO: Remove on Vespa 7 -class ConfiguredSearcherHelper { - - /** No instantiation */ - private ConfiguredSearcherHelper() { } - - public static List<Connection> toConnectionList(ProviderConfig providerConfig) { - List<Connection> connections=new ArrayList<>(); - for(ProviderConfig.Node node : providerConfig.node()) { - connections.add(new Connection(node.host(), node.port())); - } - return connections; - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/Connection.java b/container-search/src/main/java/com/yahoo/search/federation/http/Connection.java deleted file mode 100644 index 4f624a12547..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/Connection.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -/** - * Represents a connection to a particular node (host/port). - * Right now this is just a container of connection parameters, but might be extended to - * contain an open connection later. - * The host and port state is immutable. - * - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class Connection { - - private String host; - private int port; - - public Connection(String host,int port) { - this.host=host; - this.port=port; - } - - public String getHost() { return host; } - - public int getPort() { return port; } - - public String toString() { - return "http connection '" + host + ":" + port + "'"; - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/GzipDecompressingEntity.java b/container-search/src/main/java/com/yahoo/search/federation/http/GzipDecompressingEntity.java deleted file mode 100644 index 0407ddeca1b..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/GzipDecompressingEntity.java +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import org.apache.http.HttpEntity; -import org.apache.http.entity.HttpEntityWrapper; - -import java.io.BufferedInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.zip.GZIPInputStream; - -/** - * Used by HTTPSearcher when talking to services returning compressed content. - * - * @author Mainak Mandal - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class GzipDecompressingEntity extends HttpEntityWrapper { - - private static class Resources { - - byte [] buffer; - int total; - - Resources() { - total = 0; - buffer = new byte[65536]; - } - void drain(InputStream zipStream) throws IOException { - int numRead = zipStream.read(buffer, total, buffer.length); - while (numRead != -1) { - total += numRead; - if ((total + 65536) > buffer.length) { - buffer = Arrays.copyOf(buffer, buffer.length + numRead); - } - numRead = zipStream.read(buffer, total, buffer.length - total); - } - } - - } - - private final Resources resources = new Resources(); - - public GzipDecompressingEntity(final HttpEntity entity) throws IllegalStateException, IOException { - super(entity); - GZIPInputStream gz = new GZIPInputStream(entity.getContent()); - InputStream zipStream = new BufferedInputStream(gz); - try { - resources.drain(zipStream); - } catch (IOException e) { - throw e; - } finally { - zipStream.close(); - } - } - - @Override - public InputStream getContent() throws IOException, IllegalStateException { - - final ByteBuffer buff = ByteBuffer.wrap(resources.buffer, 0, resources.total); - return new InputStream() { - - @Override - public int available() throws IOException { - return buff.remaining(); - } - - @Override - public int read() throws IOException { - if (buff.hasRemaining()) - return buff.get() & 0xFF; - - return -1; - } - - @Override - public int read(byte[] b) throws IOException { - if (!buff.hasRemaining()) - return -1; - - int len = b.length; - if (len > buff.remaining()) - len = buff.remaining(); - buff.get(b, 0, len); - return len; - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - if (!buff.hasRemaining()) - return -1; - - if (len > buff.remaining()) - len = buff.remaining(); - buff.get(b, off, len); - return len; - } - - @Override - public long skip(long n) throws IOException { - if (!buff.hasRemaining()) - return -1; - - if (n > buff.remaining()) - n = buff.remaining(); - - buff.position(buff.position() + (int) n); - return n; - } - }; - } - - @Override - public long getContentLength() { - return resources.total; - } - - @Override - public void writeTo(OutputStream outstream) throws IOException { - outstream.write(resources.buffer, 0, resources.total); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPClientSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPClientSearcher.java deleted file mode 100644 index ee093adc6fc..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPClientSearcher.java +++ /dev/null @@ -1,279 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.component.ComponentId; -import com.yahoo.jdisc.http.CertificateStore; -import com.yahoo.yolean.Exceptions; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; - -import org.apache.http.HttpEntity; - -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.logging.Logger; - -/** - * A utility parent for searchers which gets data from web services which is incorporated into the query. - * This searcher will take care of implementing the search method while the extending class implements - * {@link #getQueryMap} and {@link #handleResponse} to create the http request and handle the response, respectively. - * - * <p>This class automatically adds a meta hit containing latency and other - * meta information about the obtained HTTP data using createRequestMeta(). - * The fields available in the hit are:</p> - * - * <dl><dt> - * HTTPSearcher.LOG_LATENCY_START - * <dd> - * The latency of the external provider answering a request. - * <dt> - * HTTPSearcher.LOG_LATENCY_FINISH - * <dd> - * Total time of the HTTP traffic, but also decoding of the data, is this - * happens at the same time. - * <dt> - * HTTPSearcher.LOG_URI - * <dd> - * The complete URI used for external service. - * <dt> - * HTTPSearcher.LOG_SCHEME - * <dd> - * The scheme of the request URI sent. - * <dt> - * HTTPSearcher.LOG_HOST - * <dd> - * The host used for the request URI sent. - * <dt> - * HTTPSearcher.LOG_PORT - * <dd> - * The port used for the request URI sent. - * <dt> - * HTTPSearcher.LOG_PATH - * <dd> - * Path element of the request URI sent. - * <dt> - * HTTPSearcher.LOG_STATUS - * <dd> - * Status code of the HTTP response. - * <dt> - * HTTPSearcher.LOG_PROXY_TYPE - * <dd> - * The proxy type used, if any. Default is "http". - * <dt> - * HTTPSearcher.LOG_PROXY_HOST - * <dd> - * The proxy host, if any. - * <dt> - * HTTPSearcher.LOG_PROXY_PORT - * <dd> - * The proxy port, if any. - * <dt> - * HTTPSearcher.LOG_HEADER_PREFIX prepended to request header field name - * <dd> - * The content of any additional request header fields. - * <dt> - * HTTPSearcher.LOG_RESPONSE_HEADER_PREFIX prepended to response header field name - * <dd> - * The content of any additional response header fields. - * </dl> - - * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class HTTPClientSearcher extends HTTPSearcher { - - static final CompoundName REQUEST_META_CARRIER = new CompoundName("com.yahoo.search.federation.http.HTTPClientSearcher_requestMeta"); - - protected final static Logger log = Logger.getLogger(HTTPClientSearcher.class.getName()); - - /** - * Creates a client searcher - * - * @param id the id of this instance - * @param connections the connections this will load balance and fail over between - * @param path the path portion of the url to be used - */ - public HTTPClientSearcher(ComponentId id, List<Connection> connections,String path,Statistics statistics) { - super(id, connections, path, statistics); - } - - public HTTPClientSearcher(ComponentId id, List<Connection> connections,String path,Statistics statistics, - CertificateStore certificateStore) { - super(id, connections, path, statistics, certificateStore); - } - - public HTTPClientSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters, Statistics statistics) { - super(id, connections, parameters, statistics); - } - /** - * Creates a client searcher - * - * @param id the id of this instance - * @param connections the connections this will load balance and fail over between - * @param parameters the parameters to use when making http calls - * @param certificateStore the certificate store to use to pass certificates in requests - */ - public HTTPClientSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters, - Statistics statistics, CertificateStore certificateStore) { - super(id, connections, parameters, statistics, certificateStore); - } - - /** Overridden to avoid interfering with errors from nested searchers, which is inappropriate for a <i>client</i> */ - @Override - public Result robustSearch(Query query, Execution execution, Connection connection) { - return search(query,execution,connection); - } - - /** Implements a search towards the connection chosen by the cluster searcher for this query */ - @Override - public Result search(Query query, Execution execution, Connection connection) { - Hit requestMeta = doHttpRequest(query, connection); - Result result = execution.search(query); - result.hits().add(requestMeta); - return result; - } - - private Hit doHttpRequest(Query query, Connection connection) { - URI uri; - // Create default meta hit for holding logging information - Hit requestMeta = createRequestMeta(); - query.properties().set(REQUEST_META_CARRIER, requestMeta); - - query.trace("Created request information hit",false,9); - try { - uri = getURI(query, connection); - } catch (MalformedURLException e) { - query.errors().add(createMalformedUrlError(query,e)); - return requestMeta; - } catch (URISyntaxException e) { - query.errors().add(createMalformedUrlError(query,e)); - return requestMeta; - } - - HttpEntity entity; - try { - if (query.getTraceLevel()>=1) - query.trace("Fetching " + uri.toString(), false, 1); - entity = getEntity(uri, requestMeta, query); - } catch (IOException e) { - query.errors().add(ErrorMessage.createBackendCommunicationError( - "Error when trying to connect to HTTP backend in " + this + " using " + connection + " for " + - query + ": " + Exceptions.toMessageString(e))); - return requestMeta; - } catch (TimeoutException e) { - query.errors().add(ErrorMessage.createTimeout("HTTP traffic timed out in " - + this + " for " + query + ": " + e.getMessage())); - return requestMeta; - } - if (entity==null) { - query.errors().add(ErrorMessage.createBackendCommunicationError( - "No result from connecting to HTTP backend in " + this + " using " + connection + " for " + query)); - return requestMeta; - } - - try { - query = handleResponse(entity,query); - } - catch (IOException e) { - query.errors().add(ErrorMessage.createBackendCommunicationError( - "Error when trying to consume input in " + this + ": " + Exceptions.toMessageString(e))); - } finally { - cleanupHttpEntity(entity); - } - return requestMeta; - } - - /** Overrides to pass the query on to the next searcher */ - @Override - public Result search(Query query, Execution execution, ErrorMessage error) { - query.errors().add(error); - return execution.search(query); - } - - /** Do nothing on fill in client searchers */ - @Override - public void fill(Result result,String summaryClass,Execution execution,Connection connection) { - } - - /** - * Convenience hook for unmarshalling the response and adding the information to the query. - * Implement this or <code>handleResponse(entity,query)</code> in any subclass. - * This default implementation throws an exception. - * - * @param inputStream the stream containing the data from the http service - * @param contentLength the length of the content in the stream in bytes, or a negative number if not known - * @param query the current query, to which information from the stream should be added - * @return query the query to propagate down the chain. This should almost always be the - * query instance given as a parameter. - */ - public Query handleResponse(InputStream inputStream, long contentLength, Query query) throws IOException { - throw new UnsupportedOperationException("handleResponse must be implemented by " + this); - } - - /** - * Unmarshals the response and adds the resulting data to the given query. - * This default implementation calls - * <code>return handleResponse(entity.getContent(), entity.getContentLength(), query);</code> - * (and does some detailed query tracing). - * - * @param query the current query, to which information from the stream should be added - * @return query the query to propagate down the chain. This should almost always be the - * query instance given as a parameter. - */ - public Query handleResponse(HttpEntity entity, Query query) throws IOException { - long len = entity.getContentLength(); - if (query.getTraceLevel()>=4) - query.trace("Received " + len + " bytes response in " + this, false, 4); - query = handleResponse(entity.getContent(), len, query); - if (query.getTraceLevel()>=2) - query.trace("Handled " + len + " bytes response in " + this, false, 2); - return query; - } - - /** Never retry individual queries to clients for now */ - @Override - protected boolean shouldRetry(Query query,Result result) { return false; } - - /** - * numHits and offset should not be part of the cache key as cache supports - * partial read/write that is only one cache entry is maintained per query - * irrespective of the offset and numhits. - */ - public abstract Map<String, String> getCacheKey(Query q); - - /** - * Adds all key-values starting by "service." + getClientName() in query.properties(). - * Returns the empty map if {@link #getServiceName} is not overridden. - */ - @Override - public Map<String,String> getQueryMap(Query query) { - LinkedHashMap<String, String> queryMap=new LinkedHashMap<>(); - if (getServiceName().isEmpty()) return queryMap; - - for (Map.Entry<String,Object> objectProperty : query.properties().listProperties("service." + getServiceName()).entrySet()) // TODO: Make more efficient using CompoundName - queryMap.put(objectProperty.getKey(),objectProperty.getValue().toString()); - return queryMap; - } - - /** - * Override this to return the name of the service this is a client of. - * This is used to look up service specific properties as service.getServiceName.serviceSpecificProperty. - * This default implementation returns "", which means service specific parameters will not be used. - */ - protected String getServiceName() { return ""; } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPParameters.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPParameters.java deleted file mode 100644 index a7b32678c1a..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPParameters.java +++ /dev/null @@ -1,318 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.google.common.base.Preconditions; -import com.yahoo.search.federation.ProviderConfig.PingOption; -import org.apache.http.conn.params.ConnManagerParams; -import org.apache.http.conn.params.ConnPerRouteBean; -import org.apache.http.params.BasicHttpParams; -import org.apache.http.params.HttpConnectionParams; -import org.apache.http.params.HttpParams; - -import com.yahoo.search.federation.ProviderConfig; - -/** - * A set of parameters for talking to an http backend - * - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public final class HTTPParameters { - - public static final String RETRIES = "com.yahoo.search.federation.http.retries"; - - private boolean frozen=false; - - // All timing parameters below are in milliseconds - /** The url request path portion */ - private String path="/"; - private int connectionTimeout=2000; - private int readTimeout=5000; - private boolean persistentConnections=true; - private boolean enableProxy = false; - private String proxyHost = "localhost"; - private int proxyPort = 1080; - private String method = "GET"; - private String schema = "http"; - private String inputEncoding = "utf-8"; - private String outputEncoding = "utf-8"; - private int maxTotalConnections=10000; - private int maxConnectionsPerRoute=10000; - private int socketBufferSizeBytes=-1; - private int retries = 1; - private int configuredReadTimeout = -1; - private int configuredConnectionTimeout = -1; - private int connectionPoolTimeout = -1; - private String certificateProxy = null; - private int certificatePort = 0; - private String certificateApplicationId = null; - private boolean certificateUseProxy = false; - private long certificateTtl = 0L; - private long certificateRetry = 0L; - - private PingOption.Enum pingOption = PingOption.NORMAL; - - - private boolean followRedirects = true; - - public HTTPParameters() {} - - public HTTPParameters(String path) { - setPath(path); - } - - public HTTPParameters(ProviderConfig providerConfig) { - configuredReadTimeout = (int) (providerConfig.readTimeout() * 1000.0d); - configuredConnectionTimeout = (int) (providerConfig.connectionTimeout() * 1000.0d); - connectionPoolTimeout = (int) (providerConfig.connectionPoolTimeout() * 1000.0d); - retries = providerConfig.retries(); - setPath(providerConfig.path()); - certificateUseProxy = providerConfig.yca().useProxy(); - if (certificateUseProxy) { - certificateProxy = providerConfig.yca().host(); - certificatePort = providerConfig.yca().port(); - } - certificateApplicationId = providerConfig.yca().applicationId(); - certificateTtl = providerConfig.yca().ttl() * 1000L; - certificateRetry = providerConfig.yca().retry() * 1000L; - followRedirects = providerConfig.followRedirects(); - pingOption = providerConfig.pingOption(); - } - - /** - * Set the url path to use in queries to this. If the argument is null or empty the path is set to "/". - * If a leading "/" is missing, it is added automatically. - */ - public final void setPath(String path) { - if (path==null || path.isEmpty()) path="/"; - - if (! path.startsWith("/")) - path="/" + path; - this.path = path; - } - - public PingOption.Enum getPingOption() { - return pingOption; - } - - public void setPingOption(PingOption.Enum pingOption) { - Preconditions.checkNotNull(pingOption); - ensureNotFrozen(); - this.pingOption = pingOption; - } - - /** Returns the url path. Default is "/". */ - public String getPath() { return path; } - - public boolean getFollowRedirects() { - return followRedirects; - } - - public void setFollowRedirects(boolean followRedirects) { - ensureNotFrozen(); - this.followRedirects = followRedirects; - } - - - public void setConnectionTimeout(int connectionTimeout) { - ensureNotFrozen(); - this.connectionTimeout=connectionTimeout; - } - - /** Returns the connection timeout in milliseconds. Default is 2000. */ - public int getConnectionTimeout() { return connectionTimeout; } - - public void setReadTimeout(int readTimeout) { - ensureNotFrozen(); - this.readTimeout=readTimeout; - } - - /** Returns the read timeout in milliseconds. Default is 5000. */ - public int getReadTimeout() { return readTimeout; } - - /** - * <b>Note: This is currently largely a noop: Connections are reused even when this is set to true. - * The setting will change from sharing connections between threads to only reusing it within a thread - * but it is still reused.</b> - */ - public void setPersistentConnections(boolean persistentConnections) { - ensureNotFrozen(); - this.persistentConnections=persistentConnections; - } - - /** Returns whether this should use persistent connections. Default is true. */ - public boolean getPersistentConnections() { return persistentConnections; } - - /** Returns whether proxying should be enabled. Default is false. */ - public boolean getEnableProxy() { return enableProxy; } - - public void setEnableProxy(boolean enableProxy ) { - ensureNotFrozen(); - this.enableProxy=enableProxy; - } - - /** Returns the proxy type to use (if enabled). Default is "http". */ - public String getProxyType() { - return "http"; - } - - public void setProxyHost(String proxyHost) { - ensureNotFrozen(); - this.proxyHost=proxyHost; - } - - /** Returns the proxy host to use (if enabled). Default is "localhost". */ - public String getProxyHost() { return proxyHost; } - - public void setProxyPort(int proxyPort) { - ensureNotFrozen(); - this.proxyPort=proxyPort; - } - - /** Returns the proxy port to use (if enabled). Default is 1080. */ - public int getProxyPort() { return proxyPort; } - - public void setMethod(String method) { - ensureNotFrozen(); - this.method=method; - } - - /** Returns the http method to use. Default is "GET". */ - public String getMethod() { return method; } - - public void setSchema(String schema) { - ensureNotFrozen(); - this.schema=schema; - } - - /** Returns the schema to use. Default is "http". */ - public String getSchema() { return schema; } - - public void setInputEncoding(String inputEncoding) { - ensureNotFrozen(); - this.inputEncoding=inputEncoding; - } - - /** Returns the input encoding. Default is "utf-8". */ - public String getInputEncoding() { return inputEncoding; } - - public void setOutputEncoding(String outputEncoding) { - ensureNotFrozen(); - this.outputEncoding=outputEncoding; - } - - /** Returns the output encoding. Default is "utf-8". */ - public String getOutputEncoding() { return outputEncoding; } - - /** Make this unmodifiable. Note that any thread synchronization must be done outside this object. */ - public void freeze() { - frozen=true; - } - - private void ensureNotFrozen() { - if (frozen) throw new IllegalStateException("Cannot modify frozen " + this); - } - - /** - * Returns the eligible subset of this as a HttpParams snapshot - * AND configures the Apache HTTP library with the parameters of this - */ - public HttpParams toHttpParams() { - return toHttpParams(connectionTimeout, readTimeout); - } - - /** - * Returns the eligible subset of this as a HttpParams snapshot - * AND configures the Apache HTTP library with the parameters of this - */ - public HttpParams toHttpParams(int connectionTimeout, int readTimeout) { - HttpParams params = new BasicHttpParams(); - // force use of configured value if available - if (configuredConnectionTimeout > 0) { - HttpConnectionParams.setConnectionTimeout(params, configuredConnectionTimeout); - } else { - HttpConnectionParams.setConnectionTimeout(params, connectionTimeout); - } - if (configuredReadTimeout > 0) { - HttpConnectionParams.setSoTimeout(params, configuredReadTimeout); - } else { - HttpConnectionParams.setSoTimeout(params, readTimeout); - } - if (socketBufferSizeBytes > 0) { - HttpConnectionParams.setSocketBufferSize(params, socketBufferSizeBytes); - } - if (connectionPoolTimeout > 0) { - ConnManagerParams.setTimeout(params, connectionPoolTimeout); - } - ConnManagerParams.setMaxTotalConnections(params, maxTotalConnections); - ConnManagerParams.setMaxConnectionsPerRoute(params, new ConnPerRouteBean(maxConnectionsPerRoute)); - if (retries >= 0) { - params.setIntParameter(RETRIES, retries); - } - params.setParameter("http.protocol.handle-redirects", followRedirects); - return params; - } - - public int getMaxTotalConnections() { - return maxTotalConnections; - } - - public void setMaxTotalConnections(int maxTotalConnections) { - ensureNotFrozen(); - this.maxTotalConnections = maxTotalConnections; - } - - public int getMaxConnectionsPerRoute() { - return maxConnectionsPerRoute; - } - - public void setMaxConnectionsPerRoute(int maxConnectionsPerRoute) { - ensureNotFrozen(); - this.maxConnectionsPerRoute = maxConnectionsPerRoute; - } - - public int getSocketBufferSizeBytes() { - return socketBufferSizeBytes; - } - - public void setSocketBufferSizeBytes(int socketBufferSizeBytes) { - ensureNotFrozen(); - this.socketBufferSizeBytes = socketBufferSizeBytes; - } - - public int getRetries() { - return retries; - } - - public void setRetries(int retries) { - ensureNotFrozen(); - this.retries = retries; - } - - public String getYcaProxy() { - return certificateProxy; - } - - public int getYcaPort() { - return certificatePort; - } - - public String getYcaApplicationId() { - return certificateApplicationId; - } - - public boolean getYcaUseProxy() { - return certificateUseProxy; - } - - public long getYcaTtl() { - return certificateTtl; - } - - public long getYcaRetry() { - return certificateRetry; - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPProviderSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPProviderSearcher.java deleted file mode 100644 index c72c2f26a1c..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPProviderSearcher.java +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.google.common.collect.ImmutableList; -import com.yahoo.component.ComponentId; -import com.yahoo.jdisc.http.CertificateStore; -import com.yahoo.search.cache.QrBinaryCacheConfig; -import com.yahoo.search.cache.QrBinaryCacheRegionConfig; -import com.yahoo.yolean.Exceptions; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.federation.FederationSearcher; -import com.yahoo.search.query.Properties; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Counter; -import com.yahoo.statistics.Statistics; -import com.yahoo.statistics.Value; - -import org.apache.http.HttpEntity; - -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.*; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Superclass of searchers which talks to HTTP backends. Implement a subclass to talk to a backend - * over HTTP which is not supported by the platform out of the box. - * <p> - * Implementations must override one of the <code>unmarshal</code> methods to unmarshal the response. - * </p> - * - * @author Arne Bergene Fossaa - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class HTTPProviderSearcher extends HTTPSearcher { - - private final Counter emptyResults; - private final Value hitsPerQuery; - private final Value responseLatency; - private final Counter readTimeouts; - - private final static List<String> excludedSourceProperties = ImmutableList.of("offset", "hits", "provider"); - - protected final static Logger log = Logger.getLogger(HTTPProviderSearcher.class.getName()); - - /** The name of the cache used (which is just getid().stringValue(), or null if no cache is used */ - protected String cacheName=null; - - public HTTPProviderSearcher(ComponentId id, List<Connection> connections,String path, Statistics statistics) { - this(id,connections,new HTTPParameters(path), statistics); - } - - /** Creates a http provider searcher using id.getName as provider name */ - public HTTPProviderSearcher(ComponentId id, List<Connection> connections, String path, - Statistics statistics, CertificateStore certificateStore) { - this(id, connections, new HTTPParameters(path), statistics, certificateStore); - } - - public HTTPProviderSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters, - Statistics statistics) { - this(id, connections, parameters, statistics, new ThrowingCertificateStore()); - } - - /** - * Creates a provider searcher - * - * @param id the id of this instance - * @param connections the connections this will load balance and fail over between - * @param parameters the parameters to use when making http calls - */ - public HTTPProviderSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters, - Statistics statistics, CertificateStore certificateStore) { - super(id, connections, parameters, statistics, certificateStore); - String suffix = "_" + getId().getName().replace('.', '_'); - hitsPerQuery = new Value("hits_per_query" + suffix, statistics, - new Value.Parameters().setLogRaw(false).setNameExtension(false).setLogMean(true)); - responseLatency = new Value(LOG_LATENCY_START + suffix, statistics, - new Value.Parameters().setLogRaw(false).setLogMean(true).setNameExtension(false)); - emptyResults = new Counter("empty_results" + suffix, statistics, false); - readTimeouts = new Counter(LOG_READ_TIMEOUT_PREFIX + suffix, statistics, false); - } - - /** @deprecated this method does nothing */ - @Deprecated // OK - protected void configureCache(final QrBinaryCacheConfig cacheConfig,final QrBinaryCacheRegionConfig regionConfig) { - } - - /** - * Unmarshal the stream by converting it to hits and adding the hits to the given result. - * A convenience hook called by the default <code>unmarshal(entity,result).</code> - * Override this in subclasses which does not override <code>unmarshal(entity,result).</code> - * <p> - * This default implementation throws an exception. - * - * @param stream the stream of data returned - * @param contentLength the length of the content in bytes if known, or a negative number if unknown - * @param result the result to which unmarshalled data should be added - */ - public void unmarshal(final InputStream stream, long contentLength, final Result result) throws IOException { - throw new UnsupportedOperationException("Unmarshal must be implemented by " + this); - } - - /** - * Unmarshal the result from an http entity. This default implementation calls - * <code>unmarshal(entity.getContent(), entity.getContentLength(), result)</code> - * (and does some detailed query tracing). - * - * @param entity the entity containing the data to unmarshal - * @param result the result to which unmarshalled data should be added - */ - public void unmarshal(HttpEntity entity,Result result) throws IOException { - Query query=result.getQuery(); - long len = entity.getContentLength(); - if (query.getTraceLevel()>=4) - query.trace("Received " + len + " bytes response in " + this, false, 4); - query.trace("Unmarshaling result.", false, 6); - unmarshal(entity.getContent(), len, result); - - if (query.getTraceLevel()>=2) - query.trace("Handled " + len + " bytes response in " + this, false, 2); - - } - - protected void addNonExcludedSourceProperties(Query query, Map<String, String> queryMap) { - Properties sourceProperties = FederationSearcher.getSourceProperties(query); - if (sourceProperties != null) { - for(Map.Entry<String, Object> entry : sourceProperties.listProperties("").entrySet()) { - if (!excludedSourceProperties.contains(entry.getKey())) { - queryMap.put(entry.getKey(), entry.getValue().toString()); - } - } - } - } - - /** - * Hook called at the moment the result is returned from this searcher. This default implementation - * does <code>return result</code>. - * - * @param result the result which is to be returned - * @param requestMeta the request information hit, or null if none was created (e.g if this was a cache lookup) - * @param e the exception caused during execution of this query, or null if none - * @return the result which is returned upwards - */ - protected Result inspectAndReturnFinalResult(Result result, Hit requestMeta, Exception e) { - return result; - } - - private Result statisticsBeforeInspection(Result result, Hit requestMeta, Exception e) { - int hitCount = result.getConcreteHitCount(); - if (hitCount == 0) { - emptyResults.increment(); - } - hitsPerQuery.put((double) hitCount); - - if (requestMeta != null) { - requestMeta.setField(LOG_HITCOUNT, Integer.valueOf(hitCount)); - } - - return inspectAndReturnFinalResult(result, requestMeta, e); - } - - - @Override - protected void logResponseLatency(long latency) { - responseLatency.put((double) latency); - } - - @Override - public Result search(Query query, Execution execution,Connection connection) { - // Create default meta hit for holding logging information - Hit requestMeta = createRequestMeta(); - Result result = new Result(query); - result.hits().add(requestMeta); - query.trace("Created request information hit", false, 9); - - try { - URI uri = getURI(query, requestMeta, connection); - if (query.getTraceLevel()>=1) - query.trace("Fetching " + uri.toString(), false, 1); - long requestStartTime = System.currentTimeMillis(); - - HttpEntity entity = getEntity(uri, requestMeta, query); - - // Why should consumeEntity call inspectAndReturnFinalResult itself? - // Seems confusing to me. - return entity == null - ? statisticsBeforeInspection(result, requestMeta, null) - : consumeEntity(entity, query, result, requestMeta, requestStartTime); - - } catch (MalformedURLException|URISyntaxException e) { - result.hits().addError(createMalformedUrlError(query,e)); - return statisticsBeforeInspection(result, requestMeta, e); - } catch (TimeoutException e) { - result.hits().addError(ErrorMessage.createTimeout("No time left for HTTP traffic in " - + this - + " for " + query + ": " + e.getMessage())); - return statisticsBeforeInspection(result, requestMeta, e); - } catch (IOException e) { - result.hits().addError(ErrorMessage.createBackendCommunicationError( - "Error when trying to connect to HTTP backend in " + this - + " for " + query + ": " + Exceptions.toMessageString(e))); - return statisticsBeforeInspection(result, requestMeta, e); - } - } - - private Result consumeEntity(HttpEntity entity, Query query, Result result, Hit logHit, long requestStartTime) { - - try { - // remove some time from timeout to allow for close calls with return result - unmarshal(new TimedHttpEntity(entity, query.getStartTime(), Math.max(1, query.getTimeout() - 10)), result); - logHit.setField(LOG_LATENCY_FINISH, System.currentTimeMillis() - requestStartTime); - return statisticsBeforeInspection(result, logHit, null); - } catch (IOException e) { - result.hits().addError(ErrorMessage.createBackendCommunicationError( - "Error when trying to consume input in " + this + ": " + Exceptions.toMessageString(e))); - return statisticsBeforeInspection(result, logHit, e); - } catch (TimeoutException e) { - readTimeouts.increment(); - result.hits().addError(ErrorMessage - .createTimeout("Timed out while reading/unmarshaling from backend in " - + this + " for " + query - + ": " + e.getMessage())); - return statisticsBeforeInspection(result, logHit, e); - } finally { // TODO: The scope of this finally must be enlarged to release the connection also on errors - cleanupHttpEntity(entity); - } - } - - /** - * Returns the key-value pairs that should be added as properties to the request url sent to the service. - * Must be overridden in subclasses to add the key-values expected by the service in question, unless - * {@link #getURI} (from which this is called) is overridden. - * <p> - * This default implementation returns the query.properties() prefixed by - * "source.[sourceName]" or "property.[propertyName]" - * (by calling {@link #addNonExcludedSourceProperties}). - */ - @Override - public Map<String,String> getQueryMap(Query query) { - Map<String,String> queryMap = super.getQueryMap(query); - addNonExcludedSourceProperties(query, queryMap); - return queryMap; - } - - /** - * @deprecated the cache key is ignored as there is no built-in caching support - */ - @Deprecated // OK - public abstract Map<String, String> getCacheKey(Query q); - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java deleted file mode 100644 index edf347bd84e..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java +++ /dev/null @@ -1,961 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.google.inject.Inject; -import com.yahoo.component.ComponentId; -import com.yahoo.jdisc.http.CertificateStore; -import com.yahoo.log.LogLevel; -import com.yahoo.prelude.Ping; -import com.yahoo.prelude.Pong; -import com.yahoo.yolean.Exceptions; -import com.yahoo.search.Query; -import com.yahoo.search.cluster.ClusterSearcher; -import com.yahoo.search.federation.ProviderConfig.PingOption; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.statistics.Counter; -import com.yahoo.statistics.Statistics; -import com.yahoo.text.Utf8; - -import org.apache.http.*; -import org.apache.http.client.HttpClient; -import org.apache.http.client.HttpRequestRetryHandler; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.conn.ClientConnectionManager; -import org.apache.http.conn.ConnectTimeoutException; -import org.apache.http.conn.params.ConnManagerParams; -import org.apache.http.conn.params.ConnRoutePNames; -import org.apache.http.conn.routing.HttpRoutePlanner; -import org.apache.http.conn.scheme.PlainSocketFactory; -import org.apache.http.conn.scheme.Scheme; -import org.apache.http.conn.scheme.SchemeRegistry; -import org.apache.http.conn.ssl.SSLSocketFactory; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.DefaultHttpRoutePlanner; -import org.apache.http.impl.conn.SingleClientConnManager; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; -import org.apache.http.params.HttpParams; -import org.apache.http.params.HttpProtocolParams; -import org.apache.http.protocol.BasicHttpContext; -import org.apache.http.protocol.ExecutionContext; -import org.apache.http.protocol.HttpContext; -import org.apache.http.protocol.HttpRequestExecutor; -import org.apache.http.util.EntityUtils; - -import javax.net.ssl.SSLHandshakeException; -import java.io.IOException; -import java.io.InterruptedIOException; -import java.io.UnsupportedEncodingException; -import java.net.*; -import java.nio.charset.Charset; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Generic superclass of searchers making connections to some HTTP service. This - * supports clustered connections - a list of alternative servers may be given, - * requests will be hashed across these and failed over in case some are down. - * <p> - * This simply provides some utility methods for working with http connections - * and implements ping against the service. - * - * <p>This searcher contains code from the Apache httpcomponents client library, - * licensed to the Apache Software Foundation under the Apache License, Version - * 2.0. Please refer to http://www.apache.org/licenses/LICENSE-2.0 for details. - * - * <p>This class automatically adds a meta hit containing latency and other - * meta information about the obtained HTTP data using createRequestMeta(). - * The fields available in the hit are:</p> - * - * <dl><dt> - * HTTPSearcher.LOG_LATENCY_START - * <dd> - * The latency of the external provider answering a request. - * <dt> - * HTTPSearcher.LOG_LATENCY_FINISH - * <dd> - * Total time of the HTTP traffic, but also decoding of the data, as this - * happens at the same time. - * <dt> - * HTTPSearcher.LOG_HITCOUNT - * <dd> - * Number of concrete hits in the result returned by this provider. - * <dt> - * HTTPSearcher.LOG_URI - * <dd> - * The complete URI used for external service. - * <dt> - * HTTPSearcher.LOG_SCHEME - * <dd> - * The scheme of the request URI sent. - * <dt> - * HTTPSearcher.LOG_HOST - * <dd> - * The host used for the request URI sent. - * <dt> - * HTTPSearcher.LOG_PORT - * <dd> - * The port used for the request URI sent. - * <dt> - * HTTPSearcher.LOG_PATH - * <dd> - * Path element of the request URI sent. - * <dt> - * HTTPSearcher.LOG_STATUS - * <dd> - * Status code of the HTTP response. - * <dt> - * HTTPSearcher.LOG_PROXY_TYPE - * <dd> - * The proxy type used, if any. Default is "http". - * <dt> - * HTTPSearcher.LOG_PROXY_HOST - * <dd> - * The proxy host, if any. - * <dt> - * HTTPSearcher.LOG_PROXY_PORT - * <dd> - * The proxy port, if any. - * <dt> - * HTTPSearcher.LOG_HEADER_PREFIX prepended to request header field name - * <dd> - * The content of any additional request header fields. - * <dt> - * HTTPSearcher.LOG_RESPONSE_HEADER_PREFIX prepended to response header field name - * <dd> - * The content of any additional response header fields. - * </dl> - * - * @author Arne Bergene Fossaa - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class HTTPSearcher extends ClusterSearcher<Connection> { - - protected static final String YCA_HTTP_HEADER = "Yahoo-App-Auth"; - - private static final Charset iso8859Charset = Charset.forName("ISO-8859-1"); - - // Logging field name constants - public static final String LOG_PATH = "path"; - public static final String LOG_PORT = "port"; - public static final String LOG_HOST = "host"; - public static final String LOG_IP_ADDRESS = "ip_address"; - public static final String IP_ADDRESS_UNKNOWN = "unknown"; - - public static final String LOG_SCHEME = "scheme"; - public static final String LOG_URI = "uri"; - public static final String LOG_PROXY_PORT = "proxy_port"; - public static final String LOG_PROXY_HOST = "proxy_host"; - public static final String LOG_PROXY_TYPE = "proxy_type"; - public static final String LOG_STATUS = "status"; - public static final String LOG_LATENCY_FINISH = "latency_finish"; - public static final String LOG_LATENCY_START = "latency_start"; - public static final String LOG_LATENCY_CONNECT = "latency_connect"; - public static final String LOG_QUERY_PARAM_PREFIX = "query_param_"; - public static final String LOG_HEADER_PREFIX = "header_"; - public static final String LOG_RESPONSE_HEADER_PREFIX = "response_header_"; - public static final String LOG_HITCOUNT = "hit_count"; - public static final String LOG_CONNECT_TIMEOUT_PREFIX = "connect_timeout_"; - public static final String LOG_READ_TIMEOUT_PREFIX = "read_timeout_"; - - protected final Logger log = Logger.getLogger(HTTPSearcher.class.getName()); - - /** The HTTP parameters to use. Assigned in the constructor */ - private HTTPParameters httpParameters; - - private final Counter connectTimeouts; - - /** Whether to use certificates */ - protected boolean useCertificate = false; - - private final CertificateStore certificateStore; - - /** The (optional) certificate application ID. */ - private String certificateApplicationId = null; - - /** The (optional) certificate server proxy */ - protected HttpHost certificateProxy = null; - - /** Certificate cache TTL in ms */ - private long certificateTtl = 0L; - - /** Certificate server retry rate in the cache if no cert is found, in ms */ - private long certificateRetry = 0L; - - /** Set at construction if this is using persistent connections */ - private ClientConnectionManager sharedConnectionManager = null; - - /** Set at construction if using non-persistent connections */ - private ThreadLocal<SingleClientConnManager> singleClientConnManagerThreadLocal = null; - - private static final SchemeRegistry schemeRegistry = new SchemeRegistry(); - - static { - schemeRegistry.register(new Scheme("http", PlainSocketFactory - .getSocketFactory(), 80)); - schemeRegistry.register(new Scheme("https", SSLSocketFactory - .getSocketFactory(), 443)); - } - - public HTTPSearcher(ComponentId componentId, List<Connection> connections,String path, Statistics statistics) { - this(componentId, connections, new HTTPParameters(path), statistics, new ThrowingCertificateStore()); - } - - /** Creates a http searcher with default connection and read timeouts (currently 2 and 5s respectively) */ - public HTTPSearcher(ComponentId componentId, List<Connection> connections,String path, Statistics statistics, - CertificateStore certificateStore) { - this(componentId, connections, new HTTPParameters(path), statistics, certificateStore); - } - - public HTTPSearcher(ComponentId componentId, List<Connection> connections, HTTPParameters parameters, - Statistics statistics) { - this(componentId, connections, parameters, statistics, new ThrowingCertificateStore()); - } - /** - * Creates a http searcher - * - * @param componentId the id of this instance - * @param connections the connections to establish to the backend nodes - * @param parameters the http parameters to use. This object will be frozen if it isn't already - */ - @Inject - public HTTPSearcher(ComponentId componentId, List<Connection> connections, HTTPParameters parameters, - Statistics statistics, CertificateStore certificateStore) { - super(componentId,connections,false); - String suffix = "_" + getId().getName().replace('.', '_'); - - connectTimeouts = new Counter(LOG_CONNECT_TIMEOUT_PREFIX + suffix, statistics, false); - - parameters.freeze(); - this.httpParameters = parameters; - this.certificateStore = certificateStore; - - if (parameters.getPersistentConnections()) { - HttpParams params=parameters.toHttpParams(); - HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1); - ConnManagerParams.setTimeout(params, 10); - sharedConnectionManager = new ThreadSafeClientConnManager(params, schemeRegistry); - Thread connectionPurgerThread = new Thread(() -> { - //this is the default value in yahoo jvm installations - long DNSTTLSec = 120; - while (true) { - try { - Thread.sleep(DNSTTLSec * 1000); - if (sharedConnectionManager == null) - continue; - - sharedConnectionManager.closeExpiredConnections(); - DNSTTLSec = Long.valueOf(java.security.Security - .getProperty("networkaddress.cache.ttl")); - //No DNS TTL, no need to close idle connections - if (DNSTTLSec <= 0) { - DNSTTLSec = 120; - continue; - } - sharedConnectionManager.closeIdleConnections(2 * DNSTTLSec, TimeUnit.SECONDS); - } catch (InterruptedException e) { - return; - } catch (NumberFormatException e) { - continue; - } - } - }); - connectionPurgerThread.setDaemon(true); - connectionPurgerThread.start(); - - } - else { - singleClientConnManagerThreadLocal =new ThreadLocal<>(); - } - - initializeCertificate(httpParameters, certificateStore); - } - - /** - * Initialize certificate store and proxy if they have been set to non-null, - * non-empty values. It will wrap thrown exceptions from the certificate store into - * RuntimeException and propagate them. - */ - private void initializeCertificate(HTTPParameters parameters, CertificateStore certificateStore) { - String applicationId = parameters.getYcaApplicationId(); - String proxy = parameters.getYcaProxy(); - int port = parameters.getYcaPort(); - long ttl = parameters.getYcaTtl(); - long retry = parameters.getYcaRetry(); - - if (applicationId != null && !applicationId.trim().isEmpty()) { - initializeCertificate(applicationId, ttl, retry, certificateStore); - } - - if (parameters.getYcaUseProxy()) { - initializeProxy(proxy, port); - } - } - - /** Returns the HTTP parameters used in this. This is always frozen */ - public HTTPParameters getParameters() { return httpParameters; } - - /** - * Returns the key-value pairs that should be added as properties to the request url sent to the service. - * Must be overridden in subclasses to add the key-values expected by the service in question, unless - * {@link #getURI} (from which this is called) is overridden. - * <p> - * This default implementation returns an empty LinkedHashMap. - */ - public Map<String,String> getQueryMap(Query query) { - return new LinkedHashMap<>(); - } - - /** - * Initialize the certificate. - * This will warn but not throw if certificates could not be loaded, as the certificates - * are external state which can fail independently. - */ - private void initializeCertificate(String applicationId, long ttl, long retry, CertificateStore certificateStore) { - try { - // get the certificate, i.e. init the cache and check integrity - String certificate = certificateStore.getCertificate(applicationId, ttl, retry); - if (certificate == null) { - getLogger().log(LogLevel.WARNING, "No certificate found for application '" + applicationId + "'"); - return; - } - - this.useCertificate = true; - this.certificateApplicationId = applicationId; - this.certificateTtl = ttl; - this.certificateRetry = retry; - getLogger().log(LogLevel.CONFIG, "Got certificate: " + certificate); - } - catch (Exception e) { - getLogger().log(LogLevel.WARNING,"Exception while initializing certificate for application '" + - applicationId + "' in " + this, e); - } - } - - /** - * Initialize the certificate proxy setting. - */ - private void initializeProxy(String host, int port) { - certificateProxy = new HttpHost(host, port); - getLogger().log(LogLevel.CONFIG, "Proxy is configured; will use proxy: " + certificateProxy); - } - - /** - * Same a {@code getURI(query, offset, hits, null)}. - * @see #getURI(Query, Hit, Connection) - */ - protected URI getURI(Query query,Connection connection) throws MalformedURLException, URISyntaxException { - Hit requestMeta; - try { - requestMeta = (Hit) query.properties().get(HTTPClientSearcher.REQUEST_META_CARRIER); - } catch (ClassCastException e) { - requestMeta = null; - } - return getURI(query, requestMeta, connection); - } - - /** - * Creates the URI for a query. - * Populates the {@code requestMeta} meta hit with the created URI HTTP properties. - * - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - */ - protected URI getURI(Query query, Hit requestMeta, Connection connection) - throws MalformedURLException, URISyntaxException { - StringBuilder parameters = new StringBuilder(); - - Map<String, String> queries = getQueryMap(query); - if (queries.size() > 0) { - Iterator<Map.Entry<String, String>> mapIterator = queries.entrySet().iterator(); - parameters.append("?"); - try { - Map.Entry<String, String> entry; - while (mapIterator.hasNext()) { - entry = mapIterator.next(); - - if (requestMeta != null) - requestMeta.setField(LOG_QUERY_PARAM_PREFIX - + entry.getKey(), entry.getValue()); - - parameters.append(entry.getKey() + "=" + URLEncoder.encode(entry.getValue(), - httpParameters.getInputEncoding())); - if (mapIterator.hasNext()) { - parameters.append("&"); - } - } - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("Unknown input encoding set in " + this, e); - } - } - - URI uri = new URL(httpParameters.getSchema(), connection.getHost(), - connection.getPort(), getPath() + parameters.toString()).toURI(); - if (requestMeta != null) { - requestMeta.setField(LOG_URI, uri.toString()); - requestMeta.setField(LOG_SCHEME, uri.getScheme()); - requestMeta.setField(LOG_HOST, uri.getHost()); - requestMeta.setField(LOG_PORT, uri.getPort()); - requestMeta.setField(LOG_PATH, uri.getPath()); - } - return uri; - } - - /** - * Called by getURI() to get the path of the URI for the external service. - * The default implementation returns httpParameters.getPath(); subclasses - * which only wants to override the path from httpParameters may use this - * method instead of overriding all of getURI(). - * - * @return the path to use for getURI - */ - protected String getPath() { - return httpParameters.getPath(); - } - - /** - * The URI that is used to check if the provider is up or down. This will again be used in the - * checkPing method by checking that we get a response that has a good status code (below 300). If better - * validation than just status code checking is needed, override the checkPing method. - */ - protected URI getPingURI(Connection connection) throws MalformedURLException, URISyntaxException { - return new URL(httpParameters.getSchema(),connection.getHost(),connection.getPort(),getPingPath()).toURI(); - } - - /** - * Called by getPingURI() to get the path of the URI for pinging the - * external service. The default implementation returns - * httpParameters.getPath(); subclasses which only wants to override the - * path from httpParameters may use this method instead of overriding all of - * getPingURI(). - * - * @return the path to use for getPingURI - */ - protected String getPingPath() { - return httpParameters.getPath(); - } - - /** - * Checks if the response is valid. - * @param response The response from the ping request - * @param pong The pong result to return back to the calling method. This method - * will add an error to the pong result (using addError) if the status of the HTTP response is 300 or above. - */ - protected void checkPing(HttpResponse response, Pong pong) { - if (response.getStatusLine().getStatusCode() >= 300) { - pong.addError(com.yahoo.search.result.ErrorMessage.createBackendCommunicationError( - "Got error " + response.getStatusLine().getStatusCode() - + " when contacting backend") - ); - } - } - - /** - * Pinging in HTTPBackend is done by creating a PING uri from http://host:port/path. - * If this returns a status that is below 300, the ping is considered good. - * - * If another uri is needed for pinging, reimplement getPingURI. - * - * Override either this method to change how ping - */ - @Override - public Pong ping(Ping ping, Connection connection) { - URI uri = null; - Pong pong = new Pong(); - HttpResponse response = null; - - if (httpParameters.getPingOption() == PingOption.DISABLE) - return pong; - - try { - uri = getPingURI(connection); - if (uri == null) - pong.addError(ErrorMessage.createIllegalQuery("Ping uri is null")); - if (uri.getHost()==null) { - pong.addError(ErrorMessage.createIllegalQuery("Ping uri has no host")); - uri=null; - } - } catch (MalformedURLException | URISyntaxException e) { - pong.addError(ErrorMessage.createIllegalQuery("Malformed ping uri '" + uri + "': " + - Exceptions.toMessageString(e))); - } catch (RuntimeException e) { - log.log(Level.WARNING,"Unexpected exception while attempting to ping " + connection + - " using uri '" + uri + "'",e); - pong.addError(ErrorMessage.createIllegalQuery("Unexpected problem with ping uri '" + uri + "': " + - Exceptions.toMessageString(e))); - } - - if (uri == null) return pong; - pong.setPingInfo("using uri '" + uri + "'"); - - try { - response = getPingResponse(uri, ping); - checkPing(response, pong); - } catch (IOException e) { - // We do not have a valid ping - pong.addError(ErrorMessage.createBackendCommunicationError( - "Exception thrown when pinging with url '" + uri + "': " + Exceptions.toMessageString(e))); - } catch (TimeoutException e) { - pong.addError(ErrorMessage.createTimeout("Timeout for ping " + uri + " in " + this + ": " + e.getMessage())); - } catch (RuntimeException e) { - log.log(Level.WARNING,"Unexpected exception while attempting to ping " + connection + " using uri '" + uri + "'",e); - pong.addError(ErrorMessage.createIllegalQuery("Unexpected problem with ping uri '" + uri + "': " + - Exceptions.toMessageString(e))); - } finally { - if (response != null) { - cleanupHttpEntity(response.getEntity()); - } - } - - return pong; - } - - private HttpResponse getPingResponse(URI uri, Ping ping) throws IOException { - long timeLeft = ping.getTimeout(); - int connectionTimeout = (int) (timeLeft / 4L); - int readTimeout = (int) (timeLeft * 3L / 4L); - - Map<String, String> requestHeaders = null; - if (httpParameters.getPingOption() == PingOption.YCA) - requestHeaders = generateYCAHeaders(); - - return getResponse(uri, null, requestHeaders, null, connectionTimeout, readTimeout); - } - - /** - * Same a {@code getEntity(uri, null)}. - * @param uri resource to fetch - * @param query the originating query - * @throws TimeoutException If query.timeLeft() equal to or lower than 0 - */ - protected HttpEntity getEntity(URI uri, Query query) throws IOException{ - return getEntity(uri, null, query); - } - - - /** - * Gets the HTTP entity that holds the response contents. - * @param uri the request URI. - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - * @param query the originating query - * @return the http entity, or null if none - * @throws java.io.IOException Whenever HTTP status code is in the 300 or higher range. - * @throws TimeoutException If query.timeLeft() equal to or lower than 0 - */ - protected HttpEntity getEntity(URI uri, Hit requestMeta, Query query) throws IOException { - if (query.getTimeLeft() <= 0) { - throw new TimeoutException("No time left for querying external backend."); - } - HttpResponse response = getResponse(uri, requestMeta, query); - StatusLine statusLine = response.getStatusLine(); - - // Logging - if (requestMeta != null) { - requestMeta.setField(LOG_STATUS, statusLine.getStatusCode()); - for (HeaderIterator headers = response.headerIterator(); headers.hasNext(); ) { - Header h = headers.nextHeader(); - requestMeta.setField(LOG_RESPONSE_HEADER_PREFIX + h.getName(), h.getValue()); - } - } - - if (statusLine.getStatusCode() >= 300) { - HttpEntity entity = response.getEntity(); - String message = createServerReporterErrorMessage(statusLine, entity); - cleanupHttpEntity(response.getEntity()); - throw new IOException(message); - } - - return response.getEntity(); - } - - private String createServerReporterErrorMessage(StatusLine statusLine, HttpEntity entity) { - String message = "Error when trying to connect to HTTP backend: " - + statusLine.getStatusCode() + " : " + statusLine.getReasonPhrase(); - - try { - if (entity != null) { - message += "(Message = " + EntityUtils.toString(entity) + ")"; - } - } catch (Exception e) { - log.log(LogLevel.WARNING, "Could not get message.", e); - } - - return message; - } - - /** - * Creates a meta hit dedicated to holding logging information. This hit has - * the 'logging:[searcher's ID]' type. - */ - protected Hit createRequestMeta() { - Hit requestMeta = new Hit("logging:" + getId().toString()); - requestMeta.setMeta(true); - requestMeta.types().add("logging"); - return requestMeta; - } - - protected void cleanupHttpEntity(HttpEntity entity) { - if (entity == null) return; - - try { - entity.consumeContent(); - } catch (IOException e) { - // It is ok if do not consume it, the resource will be freed after - // timeout. - // But log it just in case. - log.log(LogLevel.getVespaLogLevel(LogLevel.DEBUG), - "Not able to consume after processing: " + Exceptions.toMessageString(e)); - } - } - - /** - * Same as {@code getResponse(uri, null)}. - */ - protected HttpResponse getResponse(URI uri, Query query) throws IOException{ - return getResponse(uri, null, query); - } - - /** - * Executes an HTTP request and gets the response. - * @param uri the request URI. - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - * @param query the originating query, used to calculate timeouts - */ - protected HttpResponse getResponse(URI uri, Hit requestMeta, Query query) throws IOException { - long timeLeft = query.getTimeLeft(); - int connectionTimeout = (int) (timeLeft / 4L); - int readTimeout = (int) (timeLeft * 3L / 4L); - connectionTimeout = connectionTimeout <= 0 ? 1 : connectionTimeout; - readTimeout = readTimeout <= 0 ? 1 : readTimeout; - HttpEntity reqEntity = getRequestEntity(query, requestMeta); - Map<String, String> reqHeaders = getRequestHeaders(query, requestMeta); - if ((reqEntity == null) && (reqHeaders == null)) { - return getResponse(uri, requestMeta, connectionTimeout, readTimeout); - } else { - return getResponse(uri, reqEntity, reqHeaders, requestMeta, connectionTimeout, readTimeout); - } - } - - /** - * Returns the set of headers to be passed in the http request to provider backend. The default - * implementation returns null, unless certificates are in use. If certificates are used, it will return a map - * only containing the needed certificate headers. - */ - protected Map<String, String> getRequestHeaders(Query query, Hit requestMeta) { - if (useCertificate) { - return generateYCAHeaders(); - } - return null; - } - - /** - * Returns the HTTP request entity to use when making the request for this query. - * This default implementation returns null. - * - * <p> Do return a repeatable entity if HTTP retry is active. - * - * @return the http request entity to use, or null to use the default entity - */ - protected HttpEntity getRequestEntity(Query query, Hit requestMeta) { - return null; - } - - /** - * Executes an HTTP request and gets the response. - * @param uri the request URI. - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - * @param connectionTimeout how long to wait for getting a connection - * @param readTimeout timeout for reading HTTP data - */ - protected HttpResponse getResponse(URI uri, Hit requestMeta, int connectionTimeout, int readTimeout) - throws IOException { - return getResponse(uri, null, null, requestMeta, connectionTimeout, readTimeout); - } - - - /** - * Executes an HTTP request and gets the response. - * @param uri the request URI. - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - * @param connectionTimeout how long to wait for getting a connection - * @param readTimeout timeout for reading HTTP data - */ - protected HttpResponse getResponse(URI uri, HttpEntity reqEntity, - Map<String, String> reqHeaders, Hit requestMeta, - int connectionTimeout, int readTimeout) throws IOException { - - HttpParams httpParams = httpParameters.toHttpParams(connectionTimeout, readTimeout); - HttpClient httpClient = createClient(httpParams); - long start = 0L; - HttpUriRequest request; - if (httpParameters.getEnableProxy() && "http".equals(httpParameters.getProxyType())) { - HttpHost proxy = new HttpHost(httpParameters.getProxyHost(), - httpParameters.getProxyPort(), httpParameters.getProxyType()); - httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy); - // Logging - if (requestMeta != null) { - requestMeta.setField(LOG_PROXY_TYPE, httpParameters.getProxyType()); - requestMeta.setField(LOG_PROXY_HOST, httpParameters.getProxyHost()); - requestMeta.setField(LOG_PROXY_PORT, httpParameters.getProxyPort()); - } - } - if (reqEntity == null) { - request = createRequest(httpParameters.getMethod(), uri); - } else { - request = createRequest(httpParameters.getMethod(), uri, reqEntity); - } - - if (reqHeaders != null) { - for (Entry<String, String> entry : reqHeaders.entrySet()) { - if (entry.getValue() == null || isAscii(entry.getValue())) { - request.addHeader(entry.getKey(), entry.getValue()); - } else { - byte[] asBytes = Utf8.toBytes(entry.getValue()); - String asLyingString = new String(asBytes, 0, asBytes.length, iso8859Charset); - request.addHeader(entry.getKey(), asLyingString); - } - } - } - - // Logging - if (requestMeta != null) { - for (HeaderIterator headers = request.headerIterator(); headers.hasNext();) { - Header h = headers.nextHeader(); - requestMeta.setField(LOG_HEADER_PREFIX + h.getName(), h.getValue()); - } - start = System.currentTimeMillis(); - } - - HttpResponse response; - - try { - HttpContext context = new BasicHttpContext(); - response = httpClient.execute(request, context); - - if (requestMeta != null) { - requestMeta.setField(LOG_IP_ADDRESS, getIpAddress(context)); - } - } catch (ConnectTimeoutException e) { - connectTimeouts.increment(); - throw e; - } - - // Logging - long latencyStart = System.currentTimeMillis() - start; - if (requestMeta != null) { - requestMeta.setField(LOG_LATENCY_START, latencyStart); - } - logResponseLatency(latencyStart); - return response; - } - - private String getIpAddress(HttpContext context) { - HttpConnection connection = (HttpConnection) context.getAttribute(ExecutionContext.HTTP_CONNECTION); - if (connection instanceof HttpInetConnection) { - InetAddress address = ((HttpInetConnection) connection).getRemoteAddress(); - String hostAddress = address.getHostAddress(); - return hostAddress == null ? - IP_ADDRESS_UNKNOWN: - hostAddress; - } else { - getLogger().log(LogLevel.DEBUG, "Unexpected connection type: " + connection.getClass().getName()); - return IP_ADDRESS_UNKNOWN; - } - } - - private boolean isAscii(String value) { - char[] scanBuffer = new char[value.length()]; - value.getChars(0, value.length(), scanBuffer, 0); - for (char c: scanBuffer) - if (c > 127) return false; - return true; - } - - protected void logResponseLatency(long latency) { } - - /** - * Creates a http client for one request. Override to customize the client - * to use, e.g for testing. This default implementation will add a certificate store - * proxy to params if is necessary, and then do - * <code>return new SearcherHttpClient(getConnectionManager(params), params);</code> - */ - protected HttpClient createClient(HttpParams params) { - if (certificateProxy != null) { - params.setParameter(ConnRoutePNames.DEFAULT_PROXY, certificateProxy); - } - return new SearcherHttpClient(getConnectionManager(params), params); - } - - /** - * Creates a HttpRequest. Override to customize the request. - * This default implementation does <code>return new HttpRequest(method,uri);</code> - */ - protected HttpUriRequest createRequest(String method,URI uri) { - return createRequest(method, uri, null); - } - - /** - * Creates a HttpRequest. Override to customize the request. - * This default implementation does <code>return new HttpRequest(method,uri);</code> - */ - protected HttpUriRequest createRequest(String method,URI uri, HttpEntity entity) { - return new SearcherHttpRequest(method,uri); - } - - /** Get a connection manager which may be used safely from this thread */ - protected ClientConnectionManager getConnectionManager(HttpParams params) { - if (sharedConnectionManager != null) {// We are using shared connections - return sharedConnectionManager; - } else { - SingleClientConnManager singleClientConnManager = singleClientConnManagerThreadLocal.get(); - if (singleClientConnManager == null) { - singleClientConnManager = new SingleClientConnManager(params, schemeRegistry); - singleClientConnManagerThreadLocal.set(singleClientConnManager); - } - return singleClientConnManager; - } - } - - /** Utility method for creating error messages when a url is incorrect */ - protected ErrorMessage createMalformedUrlError(Query query,Exception e) { - return ErrorMessage.createErrorInPluginSearcher("Malformed url in " + this + " for " + query + - ": " + Exceptions.toMessageString(e)); - } - - private Map<String, String> generateYCAHeaders() { - Map<String, String> headers = new HashMap<>(); - String certificate = certificateStore.getCertificate(certificateApplicationId, certificateTtl, certificateRetry); - headers.put(YCA_HTTP_HEADER, certificate); - return headers; - } - - protected static class SearcherHttpClient extends DefaultHttpClient { - - private final int retries; - - public SearcherHttpClient(final ClientConnectionManager conman, final HttpParams params) { - super(conman, params); - retries = params.getIntParameter(HTTPParameters.RETRIES, 1); - addRequestInterceptor((request, context) -> { - if (!request.containsHeader("Accept-Encoding")) { - request.addHeader("Accept-Encoding", "gzip"); - } - }); - addResponseInterceptor((response, context) -> { - HttpEntity entity = response.getEntity(); - if (entity == null) return; - Header ceheader = entity.getContentEncoding(); - if (ceheader == null) return; - for (HeaderElement codec : ceheader.getElements()) { - if (codec.getName().equalsIgnoreCase("gzip")) { - response.setEntity(new GzipDecompressingEntity(response.getEntity())); - return; - } - } - }); - } - - @Override - protected HttpRequestExecutor createRequestExecutor() { - return new HttpRequestExecutor(); - } - - @Override - protected HttpRoutePlanner createHttpRoutePlanner() { - return new DefaultHttpRoutePlanner(getConnectionManager().getSchemeRegistry()); - } - - @Override - protected HttpRequestRetryHandler createHttpRequestRetryHandler() { - return new SearcherHttpRequestRetryHandler(retries); - } - } - - /** A retry handler which avoids retrying forever on errors misclassified as transient */ - private static class SearcherHttpRequestRetryHandler implements HttpRequestRetryHandler { - private final int retries; - - public SearcherHttpRequestRetryHandler(int retries) { - this.retries = retries; - } - - @Override - public boolean retryRequest(IOException e, int executionCount, HttpContext httpContext) { - if (e == null) { - throw new IllegalArgumentException("Exception parameter may not be null"); - } - if (executionCount > retries) { - return false; - } - if (e instanceof NoHttpResponseException) { - // Retry if the server dropped connection on us - return true; - } - if (e instanceof InterruptedIOException) { - // Timeout from federation layer - return false; - } - if (e instanceof UnknownHostException) { - // Unknown host - return false; - } - if (e instanceof SSLHandshakeException) { - // SSL handshake exception - return false; - } - return true; - } - - - } - - private static class SearcherHttpRequest extends HttpRequestBase { - String method; - - public SearcherHttpRequest(String method, final URI uri) { - super(); - this.method = method; - setURI(uri); - } - - @Override - public String getMethod() { - return method; - } - } - - /** - * Only for testing. - */ - public void shutdownConnectionManagers() { - ClientConnectionManager manager; - if (sharedConnectionManager != null) { - manager = sharedConnectionManager; - } else { - manager = singleClientConnManagerThreadLocal.get(); - } - if (manager != null) { - manager.shutdown(); - } - } - - protected static final class ThrowingCertificateStore implements CertificateStore { - - @Override - public String getCertificate(String key, long ttl, long retry) { - throw new UnsupportedOperationException("A certificate store is not available"); - } - - } - -} - diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/TimedHttpEntity.java b/container-search/src/main/java/com/yahoo/search/federation/http/TimedHttpEntity.java deleted file mode 100644 index 03ffe2b8a9c..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/TimedHttpEntity.java +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -import org.apache.http.Header; -import org.apache.http.HttpEntity; - -/** - * Wrapper for adding timeout to an HttpEntity instance. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class TimedHttpEntity implements HttpEntity { - /** - * The wrapped entity. Never null. - */ - private final HttpEntity entity; - private final long startTime; - private final long timeout; - - public TimedHttpEntity(HttpEntity entity, long startTime, long timeout) { - if (entity == null) { - throw new IllegalArgumentException("TimedHttpEntity cannot be instantiated with null HttpEntity."); - } - this.entity = entity; - this.startTime = startTime; - this.timeout = timeout; - } - - - @Override - public InputStream getContent() throws IOException, IllegalStateException { - InputStream content = entity.getContent(); - if (content == null) { - return null; - } else { - return new TimedStream(content, startTime, timeout); - } - } - - - // START OF PURE FORWARDING METHODS - @Override - public void consumeContent() throws IOException { - entity.consumeContent(); - } - - - @Override - public Header getContentEncoding() { - return entity.getContentEncoding(); - } - - @Override - public long getContentLength() { - return entity.getContentLength(); - } - - @Override - public Header getContentType() { - return entity.getContentType(); - } - - @Override - public boolean isChunked() { - return entity.isChunked(); - } - - @Override - public boolean isRepeatable() { - return entity.isRepeatable(); - } - - @Override - public boolean isStreaming() { - return entity.isStreaming(); - } - - @Override - public void writeTo(OutputStream outstream) throws IOException { - entity.writeTo(outstream); - } - // END OF PURE FORWARDING METHODS - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/TimedStream.java b/container-search/src/main/java/com/yahoo/search/federation/http/TimedStream.java deleted file mode 100644 index 77a42ee0a34..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/TimedStream.java +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import java.io.IOException; -import java.io.InputStream; - -/** - * A stream which throws a TimeoutException if query timeout has been reached. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class TimedStream extends InputStream { - - /** - * A time barrier value, the point in time from which on read operations will cause an exception. - */ - private final long limit; - - /** - * A wrapped InputStream instance. - */ - private final InputStream content; - - /** - * Wrap an InputStream to make read operations potentially fire off - * TimeoutException. - * - * <p>Typical use would be<br> - * <code>new TimedStream(httpEntity.getContent(), query.getStartTime(), query.getTimeout())</code> - * - * @param content - * the InputStream to wrap - * @param startTime - * start time of query - * @param timeout - * how long the query is allowed to run - */ - public TimedStream(InputStream content, long startTime, long timeout) { - if (content == null) { - throw new IllegalArgumentException("Cannot instantiate TimedStream with null InputStream"); - } - this.content = content; - // The reasion for doing it in here instead of outside the constructor - // is this makes the usage of the class more intuitive IMHO - this.limit = startTime + timeout; - } - - private void checkTime(String message) { - if (System.currentTimeMillis() >= limit) { - throw new TimeoutException(message); - } - } - - // START FORWARDING METHODS: - // All methods below are forwarding methods to the contained stream, where - // some do a timeout check. - @Override - public int read() throws IOException { - int data = content.read(); - checkTime("Timed out during read()."); - return data; - } - - @Override - public int available() throws IOException { - return content.available(); - } - - @Override - public void close() throws IOException { - content.close(); - } - - @Override - public synchronized void mark(int readlimit) { - content.mark(readlimit); - } - - @Override - public boolean markSupported() { - return content.markSupported(); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - int length = content.read(b, off, len); - checkTime("Timed out during read(byte[], int, int)"); - return length; - } - - @Override - public int read(byte[] b) throws IOException { - int length = content.read(b); - checkTime("Timed out during read(byte[])"); - return length; - } - - @Override - public synchronized void reset() throws IOException { - content.reset(); - } - - @Override - public long skip(long n) throws IOException { - long skipped = content.skip(n); - checkTime("Timed out during skip(long)"); - return skipped; - } - // END FORWARDING METHODS - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/TimeoutException.java b/container-search/src/main/java/com/yahoo/search/federation/http/TimeoutException.java deleted file mode 100644 index 18dd59ec37b..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/TimeoutException.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -/** - * Timeout marker for slow HTTP connections. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class TimeoutException extends RuntimeException { - - /** - * Auto-generated version ID. - */ - private static final long serialVersionUID = 7084147598258586559L; - - public TimeoutException(String message) { - super(message); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/package-info.java b/container-search/src/main/java/com/yahoo/search/federation/http/package-info.java deleted file mode 100644 index af334b35221..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.search.federation.http; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/QueryMarshaller.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/QueryMarshaller.java deleted file mode 100644 index 3a0db9b76de..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/vespa/QueryMarshaller.java +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa; - -import java.util.Iterator; - -import com.yahoo.prelude.query.*; - -/** - * Marshal a query stack into an advanced query string suitable for - * passing to another QRS. - * - * @author Steinar Knutsen - * @author Rong-En Fan - * @deprecated use YQL - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class QueryMarshaller { - - private boolean atRoot = true; - - public String marshal(Item root) { - if (root == null || root instanceof NullItem) { - return null; - } - StringBuilder s = new StringBuilder(); - marshal(root, s); - atRoot = true; - return s.toString(); - } - - /** - * We do not yet care about exact match indices - */ - private void marshal(Item root, StringBuilder s) { - switch (root.getItemType()) { - case OR: - marshalOr((OrItem) root, s); - break; - case AND: - marshalAnd((CompositeItem) root, s); - break; - case NOT: - marshalNot((NotItem) root, s); - break; - case RANK: - marshalRank((RankItem) root, s); - break; - case WORD: - case INT: - case PREFIX: - case SUBSTRING: - case SUFFIX: - marshalWord((TermItem) root, s); - break; - case PHRASE: - // PhraseItem and PhraseSegmentItem don't add quotes for segmented - // termse - if (root instanceof PhraseSegmentItem) { - marshalPhrase((PhraseSegmentItem) root, s); - } else { - marshalPhrase((PhraseItem) root, s); - } - break; - case NEAR: - marshalNear((NearItem) root, s); - break; - case ONEAR: - marshalNear((ONearItem) root, s); - break; - case WEAK_AND: - marshalWeakAnd((WeakAndItem)root, s); - break; - default: - break; - } - } - - - private void marshalWord(TermItem item, StringBuilder s) { - String index = item.getIndexName(); - if (index.length() != 0) { - s.append(item.getIndexName()).append(':'); - } - s.append(item.stringValue()); - if (item.getWeight() != Item.DEFAULT_WEIGHT) - s.append("!").append(item.getWeight()); - } - - private void marshalRank(RankItem root, StringBuilder s) { - marshalComposite("RANK", root, s); - } - - private void marshalNot(NotItem root, StringBuilder s) { - marshalComposite("ANDNOT", root, s); - } - - private void marshalOr(OrItem root, StringBuilder s) { - marshalComposite("OR", root, s); - } - - /** - * Dump WORD items, and add space between each of them unless those - * words came from segmentation. - * - * @param root CompositeItem - * @param s current marshaled query - */ - private void dumpWords(CompositeItem root, StringBuilder s) { - for (Iterator<Item> i = root.getItemIterator(); i.hasNext();) { - Item word = i.next(); - boolean useSeparator = true; - if (word instanceof TermItem) { - s.append(((TermItem) word).stringValue()); - if (word instanceof WordItem) { - useSeparator = !((WordItem) word).isFromSegmented(); - } - } else { - dumpWords((CompositeItem) word, s); - } - if (useSeparator && i.hasNext()) { - s.append(' '); - } - } - } - - private void marshalPhrase(PhraseItem root, StringBuilder s) { - marshalPhrase(root, s, root.isExplicit(), false); - } - - private void marshalPhrase(PhraseSegmentItem root, StringBuilder s) { - marshalPhrase(root, s, root.isExplicit(), true); - } - - private void marshalPhrase(IndexedItem root, StringBuilder s, boolean isExplicit, boolean isSegmented) { - String index = root.getIndexName(); - if (index.length() != 0) { - s.append(root.getIndexName()).append(':'); - } - if (isExplicit || !isSegmented) s.append('"'); - dumpWords((CompositeItem) root, s); - if (isExplicit || !isSegmented) s.append('"'); - } - - private void marshalNear(NearItem root, StringBuilder s) { - marshalComposite(root.getName() + "(" + root.getDistance() + ")", root, s); - } - - // Not only AndItem returns ItemType.AND - private void marshalAnd(CompositeItem root, StringBuilder s) { - marshalComposite("AND", root, s); - } - - private void marshalWeakAnd(WeakAndItem root, StringBuilder s) { - marshalComposite("WAND(" + root.getN() + ")", root, s); - } - - private void marshalComposite(String operator, CompositeItem root, StringBuilder s) { - boolean useParen = !atRoot; - if (useParen) { - s.append("( "); - } else { - atRoot = false; - } - for (Iterator<Item> i = root.getItemIterator(); i.hasNext();) { - Item item = i.next(); - marshal(item, s); - if (i.hasNext()) - s.append(' ').append(operator).append(' '); - } - if (useParen) { - s.append(" )"); - } - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/ResultBuilder.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/ResultBuilder.java deleted file mode 100644 index a6aec30b496..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/vespa/ResultBuilder.java +++ /dev/null @@ -1,642 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa; - -import com.yahoo.log.LogLevel; -import com.yahoo.prelude.hitfield.XMLString; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import com.yahoo.search.result.Relevance; -import com.yahoo.text.XML; -import org.xml.sax.*; -import org.xml.sax.helpers.DefaultHandler; -import org.xml.sax.helpers.XMLReaderFactory; - -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Deque; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.logging.Logger; - -import static com.yahoo.text.Lowercase.toLowerCase; - -/** - * Parse Vespa XML results and create Result instances. - * - * @author Steinar Knutsen - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -@SuppressWarnings("deprecation") -public class ResultBuilder extends DefaultHandler { - private static final String ERROR = "error"; - - private static final String FIELD = "field"; - - private static Logger log = Logger.getLogger(ResultBuilder.class.getName()); - - /** Namespaces feature id (http://xml.org/sax/features/namespaces). */ - protected static final String NAMESPACES_FEATURE_ID = "http://xml.org/sax/features/namespaces"; - - /** - * Namespace prefixes feature id - * (http://xml.org/sax/features/namespace-prefixes). - */ - protected static final String NAMESPACE_PREFIXES_FEATURE_ID = "http://xml.org/sax/features/namespace-prefixes"; - - /** Validation feature id (http://xml.org/sax/features/validation). */ - protected static final String VALIDATION_FEATURE_ID = "http://xml.org/sax/features/validation"; - - /** - * Schema validation feature id - * (http://apache.org/xml/features/validation/schema). - */ - protected static final String SCHEMA_VALIDATION_FEATURE_ID = "http://apache.org/xml/features/validation/schema"; - - /** - * Dynamic validation feature id - * (http://apache.org/xml/features/validation/dynamic). - */ - protected static final String DYNAMIC_VALIDATION_FEATURE_ID = "http://apache.org/xml/features/validation/dynamic"; - - // default settings - - /** Default parser name. */ - protected static final String DEFAULT_PARSER_NAME = "org.apache.xerces.parsers.SAXParser"; - - /** Default namespaces support (false). */ - protected static final boolean DEFAULT_NAMESPACES = false; - - /** Default namespace prefixes (false). */ - protected static final boolean DEFAULT_NAMESPACE_PREFIXES = false; - - /** Default validation support (false). */ - protected static final boolean DEFAULT_VALIDATION = false; - - /** Default Schema validation support (false). */ - protected static final boolean DEFAULT_SCHEMA_VALIDATION = false; - - /** Default dynamic validation support (false). */ - protected static final boolean DEFAULT_DYNAMIC_VALIDATION = false; - - private StringBuilder fieldContent; - - private String fieldName; - - private int fieldLevel = 0; - - private boolean hasLiteralTags = false; - - private Map<String, Object> hitFields = new HashMap<>(); - private String hitType; - private String hitRelevance; - private String hitSource; - - private int offset = 0; - - private List<Tag> tagStack = new ArrayList<>(); - - private final XMLReader parser; - - private Query query; - - private Result result; - - private static enum ResultPart { - ROOT, ERRORDETAILS, HIT, HITGROUP; - } - - Deque<ResultPart> location = new ArrayDeque<>(10); - - private String currentErrorCode; - - private String currentError; - - private Deque<HitGroup> hitGroups = new ArrayDeque<>(5); - - private static class Tag { - public final String name; - - /** - * Offset is a number which is generated for all data and tags inside - * fields, used to determine whether a tag was closed without enclosing - * any characters or other tags. - */ - public final int offset; - - public Tag(final String name, final int offset) { - this.name = name; - this.offset = offset; - } - - @Override - public String toString() { - return name + '(' + Integer.valueOf(offset) + ')'; - } - } - - /** Default constructor. */ - public ResultBuilder() throws RuntimeException { - this(createParser()); - } - - public ResultBuilder(XMLReader parser) { - this.parser = parser; - this.parser.setContentHandler(this); - this.parser.setErrorHandler(this); - } - - public static XMLReader createParser() { - ClassLoader savedContextClassLoader = Thread.currentThread().getContextClassLoader(); - Thread.currentThread().setContextClassLoader(org.apache.xerces.parsers.SAXParser.class.getClassLoader()); - - try { - XMLReader reader = XMLReaderFactory.createXMLReader(DEFAULT_PARSER_NAME); - setParserFeatures(reader); - return reader; - } catch (Exception e) { - throw new RuntimeException("error: Unable to instantiate parser (" - + DEFAULT_PARSER_NAME + ")", e); - } finally { - Thread.currentThread().setContextClassLoader(savedContextClassLoader); - } - } - - private static void setParserFeatures(XMLReader reader) { - try { - reader.setFeature(NAMESPACES_FEATURE_ID, DEFAULT_NAMESPACES); - } catch (SAXException e) { - log.log(LogLevel.WARNING, "warning: Parser does not support feature (" - + NAMESPACES_FEATURE_ID + ")"); - } - try { - reader.setFeature(NAMESPACE_PREFIXES_FEATURE_ID, - DEFAULT_NAMESPACE_PREFIXES); - } catch (SAXException e) { - log.log(LogLevel.WARNING, "warning: Parser does not support feature (" - + NAMESPACE_PREFIXES_FEATURE_ID + ")"); - } - try { - reader.setFeature(VALIDATION_FEATURE_ID, DEFAULT_VALIDATION); - } catch (SAXException e) { - log.log(LogLevel.WARNING, "warning: Parser does not support feature (" - + VALIDATION_FEATURE_ID + ")"); - } - try { - reader.setFeature(SCHEMA_VALIDATION_FEATURE_ID, - DEFAULT_SCHEMA_VALIDATION); - } catch (SAXNotRecognizedException e) { - log.log(LogLevel.WARNING, "warning: Parser does not recognize feature (" - + SCHEMA_VALIDATION_FEATURE_ID + ")"); - - } catch (SAXNotSupportedException e) { - log.log(LogLevel.WARNING, "warning: Parser does not support feature (" - + SCHEMA_VALIDATION_FEATURE_ID + ")"); - } - - try { - reader.setFeature(DYNAMIC_VALIDATION_FEATURE_ID, - DEFAULT_DYNAMIC_VALIDATION); - } catch (SAXNotRecognizedException e) { - log.log(LogLevel.WARNING, "warning: Parser does not recognize feature (" - + DYNAMIC_VALIDATION_FEATURE_ID + ")"); - - } catch (SAXNotSupportedException e) { - log.log(LogLevel.WARNING, "warning: Parser does not support feature (" - + DYNAMIC_VALIDATION_FEATURE_ID + ")"); - } - } - - @Override - public void startDocument() throws SAXException { - reset(); - result = new Result(query); - hitGroups.addFirst(result.hits()); - location.addFirst(ResultPart.ROOT); - return; - } - - private void reset() { - result = null; - fieldLevel = 0; - hasLiteralTags = false; - tagStack = null; - fieldContent = null; - offset = 0; - currentError = null; - currentErrorCode = null; - hitGroups.clear(); - location.clear(); - } - - @Override - public void startElement(String uri, String local, String raw, - Attributes attrs) throws SAXException { - // "Everybody" wants this switch to be moved into the - // enum class instead, but in this case, I find the classic - // approach more readable. - switch (location.peekFirst()) { - case HIT: - if (fieldLevel > 0) { - tagInField(raw, attrs, FIELD); - ++offset; - return; - } - if (FIELD.equals(raw)) { - ++fieldLevel; - fieldName = attrs.getValue("name"); - fieldContent = new StringBuilder(); - hasLiteralTags = false; - } - break; - case ERRORDETAILS: - if (fieldLevel > 0) { - tagInField(raw, attrs, ERROR); - ++offset; - return; - } - if (ERROR.equals(raw)) { - if (attrs != null) { - currentErrorCode = attrs.getValue("code"); - currentError = attrs.getValue("error"); - } - ++fieldLevel; - fieldContent = new StringBuilder(); - hasLiteralTags = false; - } - break; - case HITGROUP: - if ("hit".equals(raw)) { - startHit(attrs); - } else if ("group".equals(raw)) { - startHitGroup(attrs); - } - break; - case ROOT: - if ("hit".equals(raw)) { - startHit(attrs); - } else if ("errordetails".equals(raw)) { - location.addFirst(ResultPart.ERRORDETAILS); - } else if ("result".equals(raw)) { - if (attrs != null) { - String total = attrs.getValue("total-hit-count"); - if (total != null) { - result.setTotalHitCount(Long.valueOf(total)); - } - } - } else if ("group".equals(raw)) { - startHitGroup(attrs); - } else if (ERROR.equals(raw)) { - if (attrs != null) { - currentErrorCode = attrs.getValue("code"); - fieldContent = new StringBuilder(); - } - } - break; - } - ++offset; - } - - private void startHitGroup(Attributes attrs) { - HitGroup g = new HitGroup(); - Set<String> types = g.types(); - - final String source; - if (attrs != null) { - String groupType = attrs.getValue("type"); - if (groupType != null) { - for (String s : groupType.split(" ")) { - if (s.length() > 0) { - types.add(s); - } - } - } - - source = attrs.getValue("source"); - } else { - source = null; - } - - g.setId((source != null) ? source : "dummy"); - - hitGroups.peekFirst().add(g); - hitGroups.addFirst(g); - location.addFirst(ResultPart.HITGROUP); - } - - private void startHit(Attributes attrs) { - hitFields.clear(); - location.addFirst(ResultPart.HIT); - if (attrs != null) { - hitRelevance = attrs.getValue("relevancy"); - hitSource = attrs.getValue("source"); - hitType = attrs.getValue("type"); - } else { - hitRelevance = null; - hitSource = null; - hitType = null; - } - } - - private void tagInField(String tag, Attributes attrs, String enclosingTag) { - if (!hasLiteralTags) { - hasLiteralTags = true; - String fieldTillNow = XML.xmlEscape(fieldContent.toString(), false); - fieldContent = new StringBuilder(fieldTillNow); - tagStack = new ArrayList<>(); - } - if (enclosingTag.equals(tag)) { - ++fieldLevel; - } - if (tagStack.size() > 0) { - Tag prevTag = tagStack.get(tagStack.size() - 1); - if (prevTag != null && (prevTag.offset + 1) == offset) { - fieldContent.append(">"); - } - } - fieldContent.append("<").append(tag); - if (attrs != null) { - int attrCount = attrs.getLength(); - for (int i = 0; i < attrCount; i++) { - fieldContent.append(" ").append(attrs.getQName(i)) - .append("=\"").append( - XML.xmlEscape(attrs.getValue(i), true)).append( - "\""); - } - } - tagStack.add(new Tag(tag, offset)); - } - - private void endElementInField(String qName, String enclosingTag) { - Tag prevTag = tagStack.get(tagStack.size() - 1); - if (qName.equals(prevTag.name) && offset == (prevTag.offset + 1)) { - fieldContent.append(" />"); - } else { - fieldContent.append("</").append(qName).append('>'); - } - if (prevTag.name.equals(qName)) { - tagStack.remove(tagStack.size() - 1); - } - } - - private void endElementInHitField(String qName) { - if (FIELD.equals(qName) && --fieldLevel == 0) { - Object content; - if (hasLiteralTags) { - content = new XMLString(fieldContent.toString()); - } else { - content = fieldContent.toString(); - } - hitFields.put(fieldName, content); - if ("collapseId".equals(fieldName)) { - hitFields.put(fieldName, Integer.valueOf(content.toString())); - } - fieldName = null; - fieldContent = null; - tagStack = null; - } else { - Tag prevTag = tagStack.get(tagStack.size() - 1); - if (qName.equals(prevTag.name) && offset == (prevTag.offset + 1)) { - fieldContent.append(" />"); - } else { - fieldContent.append("</").append(qName).append('>'); - } - if (prevTag.name.equals(qName)) { - tagStack.remove(tagStack.size() - 1); - } - } - } - @Override - public void characters(char ch[], int start, int length) - throws SAXException { - - switch (location.peekFirst()) { - case ERRORDETAILS: - case HIT: - if (fieldLevel > 0) { - if (hasLiteralTags) { - if (tagStack.size() > 0) { - Tag tag = tagStack.get(tagStack.size() - 1); - if (tag != null && (tag.offset + 1) == offset) { - fieldContent.append(">"); - } - } - fieldContent.append( - XML.xmlEscape(new String(ch, start, length), false)); - } else { - fieldContent.append(ch, start, length); - } - } - break; - default: - if (fieldContent != null) { - fieldContent.append(ch, start, length); - } - break; - } - ++offset; - } - - @Override - public void ignorableWhitespace(char ch[], int start, int length) - throws SAXException { - return; - } - - @Override - public void processingInstruction(String target, String data) - throws SAXException { - return; - } - - @Override - public void endElement(String namespaceURI, String localName, String qName) - throws SAXException { - switch (location.peekFirst()) { - case HITGROUP: - if ("group".equals(qName)) { - hitGroups.removeFirst(); - location.removeFirst(); - } - break; - case HIT: - if (fieldLevel > 0) { - endElementInHitField(qName); - } else if ("hit".equals(qName)) { - //assert(hitKeys.size() == hitValues.size()); - //We try to get either uri or documentID and use that as id - Object docId = extractDocumentID(); - Hit newHit = new Hit(docId.toString()); - if (hitRelevance != null) newHit.setRelevance(new Relevance(Double.parseDouble(hitRelevance))); - if(hitSource != null) newHit.setSource(hitSource); - if(hitType != null) { - for(String type: hitType.split(" ")) { - newHit.types().add(type); - } - } - for(Map.Entry<String, Object> field : hitFields.entrySet()) { - newHit.setField(field.getKey(), field.getValue()); - } - - hitGroups.peekFirst().add(newHit); - location.removeFirst(); - } - break; - case ERRORDETAILS: - if (fieldLevel == 1 && ERROR.equals(qName)) { - ErrorMessage error = new ErrorMessage(Integer.valueOf(currentErrorCode), - currentError, - fieldContent.toString()); - hitGroups.peekFirst().addError(error); - currentError = null; - currentErrorCode = null; - fieldContent = null; - tagStack = null; - fieldLevel = 0; - } else if (fieldLevel > 0) { - endElementInField(qName, ERROR); - } else if ("errordetails".equals(qName)) { - location.removeFirst(); - } - break; - case ROOT: - if (ERROR.equals(qName)) { - ErrorMessage error = new ErrorMessage(Integer.valueOf(currentErrorCode), - fieldContent.toString()); - hitGroups.peekFirst().addError(error); - currentErrorCode = null; - fieldContent = null; - } - break; - default: - break; - } - ++offset; - } - - private Object extractDocumentID() { - Object docId = null; - if (hitFields.containsKey("uri")) { - docId = hitFields.get("uri"); - } else { - final String documentId = "documentId"; - if (hitFields.containsKey(documentId)) { - docId = hitFields.get(documentId); - } else { - final String lcDocumentId = toLowerCase(documentId); - for (Map.Entry<String, Object> e : hitFields.entrySet()) { - String key = e.getKey(); - // case insensitive matching, checking length first hoping to avoid some lowercasing - if (documentId.length() == key.length() && lcDocumentId.equals(toLowerCase(key))) { - docId = e.getValue(); - break; - } - } - } - } - if (docId == null) { - docId = "dummy"; - log.info("Results from vespa backend did not contain either uri or documentId"); - } - return docId; - } - - @Override - public void warning(SAXParseException ex) throws SAXException { - printError("Warning", ex); - } - - @Override - public void error(SAXParseException ex) throws SAXException { - printError("Error", ex); - } - - @Override - public void fatalError(SAXParseException ex) throws SAXException { - printError("Fatal Error", ex); - // throw ex; - } - - /** Prints the error message. */ - protected void printError(String type, SAXParseException ex) { - StringBuilder errorMessage = new StringBuilder(); - - errorMessage.append(type); - if (ex != null) { - String systemId = ex.getSystemId(); - if (systemId != null) { - int index = systemId.lastIndexOf('/'); - if (index != -1) - systemId = systemId.substring(index + 1); - errorMessage.append(' ').append(systemId); - } - } - errorMessage.append(':') - .append(ex.getLineNumber()) - .append(':') - .append(ex.getColumnNumber()) - .append(": ") - .append(ex.getMessage()); - log.log(LogLevel.WARNING, errorMessage.toString()); - - } - - public Result parse(String identifier, Query query) { - Result toReturn; - - setQuery(query); - try { - parser.parse(identifier); - } catch (SAXParseException e) { - // ignore - } catch (Exception e) { - log.log(LogLevel.WARNING, "Error parsing result from Vespa",e); - Exception se = e; - if (e instanceof SAXException) { - se = ((SAXException) e).getException(); - } - if (se != null) - se.printStackTrace(System.err); - else - e.printStackTrace(System.err); - } - toReturn = result; - reset(); - return toReturn; - } - - public Result parse(InputSource input, Query query) { - Result toReturn; - - setQuery(query); - try { - parser.parse(input); - } catch (SAXParseException e) { - // ignore - } catch (Exception e) { - log.log(LogLevel.WARNING, "Error parsing result from Vespa",e); - Exception se = e; - if (e instanceof SAXException) { - se = ((SAXException) e).getException(); - } - if (se != null) - se.printStackTrace(System.err); - else - e.printStackTrace(System.err); - } - toReturn = result; - reset(); - return toReturn; - } - - - private void setQuery(Query query) { - this.query = query; - } -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java deleted file mode 100644 index 246732d0970..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java +++ /dev/null @@ -1,259 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa; - -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.Map; -import java.util.Set; - -import com.yahoo.search.federation.http.ConfiguredHTTPProviderSearcher; -import org.xml.sax.InputSource; -import org.xml.sax.XMLReader; - -import com.google.inject.Inject; -import com.yahoo.collections.Tuple2; -import com.yahoo.component.ComponentId; -import com.yahoo.component.Version; -import com.yahoo.component.chain.dependencies.After; -import com.yahoo.component.chain.dependencies.Provides; -import com.yahoo.language.Linguistics; -import com.yahoo.log.LogLevel; -import com.yahoo.prelude.query.Item; -import com.yahoo.prelude.query.QueryCanonicalizer; -import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.cache.QrBinaryCacheConfig; -import com.yahoo.search.cache.QrBinaryCacheRegionConfig; -import com.yahoo.search.federation.ProviderConfig; -import com.yahoo.search.federation.http.Connection; -import com.yahoo.search.query.QueryTree; -import com.yahoo.search.query.textserialize.TextSerialize; -import com.yahoo.search.yql.MinimalQueryInserter; -import com.yahoo.statistics.Statistics; - -import edu.umd.cs.findbugs.annotations.Nullable; - -/** - * Backend searcher for external Vespa clusters (queried over http). - * - * <p>If the "sources" argument should be honored on an external cluster - * when using YQL+, override {@link #chooseYqlSources(Set)}.</p> - * - * @author Arne Bergene Fossaa - * @author Steinar Knutsen - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated -@Provides("Vespa") -@After("*") -public class VespaSearcher extends ConfiguredHTTPProviderSearcher { - - private final ThreadLocal<XMLReader> readerHolder = new ThreadLocal<>(); - private final Query.Type queryType; - private final Tuple2<String, Version> segmenterVersion; - - private static final CompoundName select = new CompoundName("select"); - private static final CompoundName streamingUserid = new CompoundName("streaming.userid"); - private static final CompoundName streamingGroupname = new CompoundName("streaming.groupname"); - private static final CompoundName streamingSelection = new CompoundName("streaming.selection"); - - /** Create an instance from configuration */ - public VespaSearcher(ComponentId id, ProviderConfig config, QrBinaryCacheConfig c, - QrBinaryCacheRegionConfig r, Statistics statistics) { - this(id, config, c, r, statistics, null); - } - - /** - * Create an instance from configuration - * - * @param linguistics used for generating meta info for YQL+ - */ - @Inject - public VespaSearcher(ComponentId id, ProviderConfig config, - QrBinaryCacheConfig c, QrBinaryCacheRegionConfig r, - Statistics statistics, @Nullable Linguistics linguistics) { - super(id, config, c, r, statistics); - queryType = toQueryType(config.queryType()); - if (linguistics == null) { - segmenterVersion = null; - } else { - segmenterVersion = linguistics.getVersion(Linguistics.Component.SEGMENTER); - } - } - - /** - * Create an instance from direct parameters having a single connection. - * Useful for testing - */ - public VespaSearcher(String idString, String host, int port, String path) { - super(idString, host, port, path, Statistics.nullImplementation); - queryType = toQueryType(ProviderConfig.QueryType.LEGACY); - segmenterVersion = null; - } - - void addProperty(Map<String, String> queryMap, Query query, CompoundName property) { - Object o = query.properties().get(property); - if (o != null) { - queryMap.put(property.toString(), o.toString()); - } - } - - @Override - public Map<String, String> getQueryMap(Query query) { - Map<String, String> queryMap = getQueryMapWithoutHitsOffset(query); - queryMap.put("offset", Integer.toString(query.getOffset())); - queryMap.put("hits", Integer.toString(query.getHits())); - queryMap.put("presentation.format", "xml"); - - addProperty(queryMap, query, select); - addProperty(queryMap, query, streamingUserid); - addProperty(queryMap, query, streamingGroupname); - addProperty(queryMap, query, streamingSelection); - return queryMap; - } - - @Override - public Map<String, String> getCacheKey(Query q) { - return getQueryMapWithoutHitsOffset(q); - } - - private Map<String, String> getQueryMapWithoutHitsOffset(Query query) { - Map<String, String> queryMap = super.getQueryMap(query); - if (queryType == Query.Type.YQL) { - queryMap.put(MinimalQueryInserter.YQL.toString(), marshalQuery(query)); - } else { - queryMap.put("query", marshalQuery(query.getModel().getQueryTree())); - queryMap.put("type", queryType.toString()); - } - - addNonExcludedSourceProperties(query, queryMap); - return queryMap; - } - - Query.Type toQueryType(ProviderConfig.QueryType.Enum providerQueryType) { - if (providerQueryType == ProviderConfig.QueryType.LEGACY) { - return Query.Type.ADVANCED; - } else if (providerQueryType == ProviderConfig.QueryType.PROGRAMMATIC) { - return Query.Type.PROGRAMMATIC; - } else if (providerQueryType == ProviderConfig.QueryType.YQL) { - return Query.Type.YQL; - } else if (providerQueryType == ProviderConfig.QueryType.SELECT) { - return Query.Type.SELECT; - } else { - throw new RuntimeException("Query type " + providerQueryType + " unsupported."); - } - } - - /** - * Serialize the query parameter for outgoing queries. For YQL+ queries, - * sources and fields will be set to all sources and all fields, to follow - * the behavior of other query types. - * - * @param query - * the current, outgoing query - * @return a string to include in an HTTP request - */ - public String marshalQuery(Query query) { - if (queryType != Query.Type.YQL) { - return marshalQuery(query.getModel().getQueryTree()); - } - - query.getModel().getQueryTree(); // performance: parse query before cloning such that it is only done once - Query workQuery = query.clone(); - String error = QueryCanonicalizer.canonicalize(workQuery); - if (error != null) { - getLogger().log(LogLevel.WARNING,"Could not normalize [" + query.toString() + "]: " + error); - // Just returning null here is the pattern from existing code... - return null; - } - chooseYqlSources(workQuery.getModel().getSources()); - chooseYqlSummaryFields(workQuery.getPresentation().getSummaryFields()); - return workQuery.yqlRepresentation(getSegmenterVersion(), false); - } - - public String marshalQuery(QueryTree root) { - QueryTree rootClone = root.clone(); // TODO: Why? - String error = QueryCanonicalizer.canonicalize(rootClone); - if (error != null) return null; - - return marshalRoot(rootClone.getRoot()); - } - - private String marshalRoot(Item root) { - switch (queryType) { - case ADVANCED: return new QueryMarshaller().marshal(root); - case PROGRAMMATIC: return TextSerialize.serialize(root); - default: throw new RuntimeException("Unsupported query type."); - } - } - - private XMLReader getReader() { - XMLReader reader = readerHolder.get(); - if (reader == null) { - reader = ResultBuilder.createParser(); - readerHolder.set(reader); - } - return reader; - } - - @Override - public void unmarshal(InputStream stream, long contentLength, Result result) { - ResultBuilder parser = new ResultBuilder(getReader()); - Result mResult = parser.parse(new InputSource(stream), - result.getQuery()); - result.mergeWith(mResult); - result.hits().addAll(mResult.hits().asUnorderedHits()); - } - - /** Returns the canonical Vespa ping URI, http://host:port/status.html */ - @Override - public URI getPingURI(Connection connection) throws MalformedURLException, URISyntaxException { - return new URL(getParameters().getSchema(), connection.getHost(), - connection.getPort(), "/status.html").toURI(); - } - - /** - * Get the segmenter version data used when creating YQL queries. Useful if - * overriding {@link #marshalQuery(Query)}. - * - * @return a tuple with the name of the segmenting engine in use, and its - * version - */ - protected Tuple2<String, Version> getSegmenterVersion() { - return segmenterVersion; - } - - /** - * Choose which source arguments to use for the external cluster when - * generating a YQL+ query string. This is called from - * {@link #marshalQuery(Query)}. The default implementation clears the set, - * i.e. requests all sources. Other implementations may modify the source - * set as they see fit, or simply do nothing. - * - * @param sources - * the set of source names to use for the outgoing query - */ - protected void chooseYqlSources(Set<String> sources) { - sources.clear(); - } - - /** - * Choose which summary fields to request from the external cluster when - * generating a YQL+ query string. This is called from - * {@link #marshalQuery(Query)}. The default implementation clears the set, - * i.e. requests all fields. Other implementations may modify the summary - * field set as they see fit, or simply do nothing. - * - * @param summaryFields - * the set of source names to use for the outgoing query - */ - protected void chooseYqlSummaryFields(Set<String> summaryFields) { - summaryFields.clear(); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/package-info.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/package-info.java deleted file mode 100644 index a1c9236ff9c..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/vespa/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.search.federation.vespa; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java index 83eca89dffa..13c23234910 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java @@ -121,20 +121,6 @@ public class GroupingRequest { } /** - * Sets the result {@link RootGroup} of this request. This is used by the executing grouping searcher, and should - * not be called by a requesting searcher. - * - * @param group the result to set. - * @return this, to allow chaining. - * @deprecated this is a noop - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public GroupingRequest setResultGroup(RootGroup group) { - return this; - } - - /** * Returns the list of {@link Continuation}s of this request. This is used by the executing grouping searcher to * allow pagination of grouping results. * @@ -156,21 +142,6 @@ public class GroupingRequest { return newRequest; } - /** - * Returns all instances of this class that have been attached to the given {@link Query}. If no requests have been - * attached to the {@link Query}, this method returns an empty list. - * - * @param query the query whose requests to return. - * @return the list of grouping requests. - * @deprecated use query.getSelect().getGrouping() - */ - @SuppressWarnings({ "unchecked" }) - // TODO: Remove on Vespa 7 - @Deprecated // OK - public static List<GroupingRequest> getRequests(Query query) { - return query.getSelect().getGrouping(); - } - @Override public String toString() { return root == null ? "(empty)" : root.toString(); diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/YmumValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/YmumValue.java deleted file mode 100644 index 8ca772b9188..00000000000 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/YmumValue.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.grouping.request; - -/** - * This class represents a document checksum in a {@link GroupingExpression}. It evaluates to the YMUM checksum of the - * input {@link com.yahoo.search.result.Hit}. - * - * @author Simon Thoresen Hult - * @author bratseth - */ -public class YmumValue extends DocumentValue { - - /** - * Constructs a new instance of this class. - */ - public YmumValue() { - this(null, null); - } - - private YmumValue(String label, Integer level) { - super("ymum()", label, level); - } - - @Override - public YmumValue copy() { - return new YmumValue(getLabel(), getLevelOrNull()); - } - -} - diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/BucketGroupId.java b/container-search/src/main/java/com/yahoo/search/grouping/result/BucketGroupId.java index 07729b4524a..e1f74df7172 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/result/BucketGroupId.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/result/BucketGroupId.java @@ -30,9 +30,9 @@ public abstract class BucketGroupId<T> extends GroupId { * * @param type The type of this id's value. * @param from The inclusive-from of the range. - * @param fromImage The String representation of the <tt>from</tt> argument. + * @param fromImage The String representation of the <code>from</code> argument. * @param to The exclusive-to of the range. - * @param toImage The String representation of the <tt>to</tt> argument. + * @param toImage The String representation of the <code>to</code> argument. */ public BucketGroupId(String type, T from, String fromImage, T to, String toImage) { super(type, fromImage, toImage); diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/ValueGroupId.java b/container-search/src/main/java/com/yahoo/search/grouping/result/ValueGroupId.java index bebd3fd85b0..a76af885180 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/result/ValueGroupId.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/result/ValueGroupId.java @@ -27,7 +27,7 @@ public abstract class ValueGroupId<T> extends GroupId { * * @param type The type of this id's value. * @param value The identifying value. - * @param valueImage The String representation of the <tt>value</tt> argument. + * @param valueImage The String representation of the <code>value</code> argument. */ public ValueGroupId(String type, T value, String valueImage) { super(type, valueImage); diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ExpressionConverter.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ExpressionConverter.java index d017fe2edb3..d5d1bb5b18e 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ExpressionConverter.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ExpressionConverter.java @@ -85,7 +85,6 @@ import com.yahoo.search.grouping.request.XorAggregator; import com.yahoo.search.grouping.request.XorBitFunction; import com.yahoo.search.grouping.request.XorFunction; import com.yahoo.search.grouping.request.YearFunction; -import com.yahoo.search.grouping.request.YmumValue; import com.yahoo.search.grouping.request.ZCurveXFunction; import com.yahoo.search.grouping.request.ZCurveYFunction; import com.yahoo.searchlib.aggregation.AggregationResult; @@ -115,7 +114,6 @@ import com.yahoo.searchlib.expression.FloatBucketResultNode; import com.yahoo.searchlib.expression.FloatBucketResultNodeVector; import com.yahoo.searchlib.expression.FloatResultNode; import com.yahoo.searchlib.expression.GetDocIdNamespaceSpecificFunctionNode; -import com.yahoo.searchlib.expression.GetYMUMChecksumFunctionNode; import com.yahoo.searchlib.expression.IntegerBucketResultNode; import com.yahoo.searchlib.expression.IntegerBucketResultNodeVector; import com.yahoo.searchlib.expression.IntegerResultNode; @@ -522,9 +520,6 @@ class ExpressionConverter { return new XorBitFunctionNode().setNumBits(((XorBitFunction)exp).getNumBits()) .addArg(toExpressionNode(((XorBitFunction)exp).getArg(0))); } - if (exp instanceof YmumValue) { - return new GetYMUMChecksumFunctionNode(); - } throw new UnsupportedOperationException("Can not convert '" + exp + "' of class " + exp.getClass().getName() + " to an expression."); } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java index b28e648be78..7c2e774f68b 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java @@ -217,7 +217,7 @@ public class GroupingExecutor extends Searcher { baseRoot = origRoot.clone(); } if (query.isTraceable(3) && query.getGroupingSessionCache()) { - query.trace("Grouping in " + (lastPass + 1) + " passes. SessionId='" + query.getSessionId(false) + "'.", 3); + query.trace("Grouping in " + (lastPass + 1) + " passes. SessionId='" + query.getSessionId() + "'.", 3); } for (int pass = 0; pass <= lastPass; ++pass) { boolean firstPass = (pass == 0); diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 6f965944bdf..d2490ec9532 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -13,13 +13,11 @@ import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.core.ContainerHttpConfig; -import com.yahoo.container.core.QrTemplatesConfig; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.container.jdisc.VespaHeaders; import com.yahoo.container.logging.AccessLog; -import com.yahoo.container.protect.FreezeDetector; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; import com.yahoo.language.Linguistics; @@ -33,6 +31,7 @@ import com.yahoo.prelude.query.parser.ParseException; import com.yahoo.prelude.query.parser.SpecialTokenRegistry; import com.yahoo.processing.rendering.Renderer; import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.query.ranking.SoftTimeout; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.vespa.config.SlimeUtils; @@ -164,48 +163,6 @@ public class SearchHandler extends LoggingRequestHandler { Optional.empty() : Optional.of( containerHttpConfig.hostResponseHeaderKey()); } - /** @deprecated use the constructor with ContainerHttpConfig */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public SearchHandler( - final ChainsConfig chainsConfig, - final IndexInfoConfig indexInfo, - final QrSearchersConfig clusters, - final SpecialtokensConfig specialtokens, - final Statistics statistics, - final Linguistics linguistics, - final Metric metric, - final ComponentRegistry<Renderer> renderers, - final Executor executor, - final AccessLog accessLog, - final QueryProfilesConfig queryProfileConfig, - final ComponentRegistry<Searcher> searchers) { - this (chainsConfig, indexInfo, clusters, specialtokens, statistics, linguistics, metric, renderers, executor, - accessLog, queryProfileConfig, searchers, new ContainerHttpConfig(new ContainerHttpConfig.Builder())); - } - - /** @deprecated use the constructor without deprecated parameters */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public SearchHandler( - final ChainsConfig chainsConfig, - final IndexInfoConfig indexInfo, - final QrSearchersConfig clusters, - final SpecialtokensConfig specialTokens, - final QrTemplatesConfig ignored, - final FreezeDetector ignored2, - final Statistics statistics, - final Linguistics linguistics, - final Metric metric, - final ComponentRegistry<Renderer> renderers, - final Executor executor, - final AccessLog accessLog, - final QueryProfilesConfig queryProfileConfig, - final ComponentRegistry<Searcher> searchers) { - this(chainsConfig, indexInfo, clusters, specialTokens, statistics, linguistics, metric, renderers, - executor, accessLog, queryProfileConfig, searchers); - } - @Override protected void destroy() { super.destroy(); @@ -263,8 +220,6 @@ public class SearchHandler extends LoggingRequestHandler { Result result = new Result(query, errorMessage); Renderer renderer = getRendererCopy(ComponentSpecification.fromString(request.getProperty("format"))); - result.getTemplating().setRenderer(renderer); // Pre-Vespa 6 Result.getEncoding() expects this TODO: Remove opn Vespa 7 - return new HttpSearchResponse(getHttpResponseStatus(request, result), result, query, renderer); } @@ -291,8 +246,12 @@ public class SearchHandler extends LoggingRequestHandler { Query query = new Query(request, requestMap, queryProfile); - boolean benchmarkOutput = VespaHeaders.benchmarkOutput(request); - boolean benchmarkCoverage = VespaHeaders.benchmarkCoverage(benchmarkOutput, request.getJDiscRequest().headers()); + boolean benchmarking = VespaHeaders.benchmarkOutput(request); + boolean benchmarkCoverage = VespaHeaders.benchmarkCoverage(benchmarking, request.getJDiscRequest().headers()); + + // Don't use soft timeout by default when benchmarking to avoid wrong conclusions by excluding nodes + if (benchmarking && ! request.hasProperty(SoftTimeout.enableProperty.toString())) + query.properties().set(SoftTimeout.enableProperty, false); // Find and execute search chain if we have a valid query String invalidReason = query.validate(); @@ -321,22 +280,14 @@ public class SearchHandler extends LoggingRequestHandler { result = search(pathAndQuery, query, searchChain, searchChainRegistry); } - Renderer renderer; - if (result.getTemplating().usesDefaultTemplate()) { // TODO: Remove on Vespa 7 - renderer = toRendererCopy(query.getPresentation().getRenderer()); - result.getTemplating().setRenderer(renderer); // pre-Vespa 6 Result.getEncoding() expects this to be set. - } - else { // somebody explicitly assigned a old style template // TODO: Remove on Vespa 7 - renderer = perRenderingCopy(result.getTemplating().getRenderer()); - } - // Transform result to response - HttpSearchResponse response = new HttpSearchResponse(getHttpResponseStatus(request, result), + Renderer renderer = toRendererCopy(query.getPresentation().getRenderer()); + HttpSearchResponse response = new HttpSearchResponse(getHttpResponseStatus(request, result), result, query, renderer); if (hostResponseHeaderKey.isPresent()) response.headers().add(hostResponseHeaderKey.get(), selfHostname); - if (benchmarkOutput) + if (benchmarking) VespaHeaders.benchmarkOutput(response.headers(), benchmarkCoverage, response.getTiming(), response.getHitCounts(), getErrors(result), response.getCoverage()); @@ -390,9 +341,6 @@ public class SearchHandler extends LoggingRequestHandler { } Result result = execution.search(query); - if (result.getTemplating() == null) // TODO: Remove on Vespa 7 - result.getTemplating().setRenderer(renderer); - ensureQuerySet(result, query); execution.fill(result, result.getQuery().getPresentation().getSummary()); diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java index 92e4bb7e5b8..0741e8effac 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java @@ -13,6 +13,7 @@ import com.yahoo.search.Result; import com.yahoo.search.pagetemplates.model.Renderer; import com.yahoo.search.pagetemplates.model.Source; import com.yahoo.search.query.context.QueryContext; +import com.yahoo.search.rendering.XmlRenderer; import com.yahoo.search.result.Coverage; import com.yahoo.search.result.DefaultErrorHit; import com.yahoo.search.result.ErrorHit; @@ -124,7 +125,7 @@ public class PageTemplatesXmlRenderer extends AsynchronousSectionedRenderer<Resu XMLWriter xmlWriter=XMLWriter.from(writer); xmlWriter.openTag("meta").attribute("type", QueryContext.ID); TraceNode traceRoot = owner.getModel().getExecution().trace().traceNode().root(); - traceRoot.accept(new com.yahoo.search.rendering.DefaultRenderer.RenderingVisitor(xmlWriter, owner.getStartTime())); + traceRoot.accept(new XmlRenderer.RenderingVisitor(xmlWriter, owner.getStartTime())); xmlWriter.closeTag(); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/Model.java b/container-search/src/main/java/com/yahoo/search/query/Model.java index fd52618ad85..28b605fa095 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Model.java +++ b/container-search/src/main/java/com/yahoo/search/query/Model.java @@ -59,7 +59,7 @@ public class Model implements Cloneable { argumentType.addField(new FieldDescription(QUERY_STRING, "string", "query")); argumentType.addField(new FieldDescription(TYPE, "string", "type")); argumentType.addField(new FieldDescription(FILTER, "string","filter")); - argumentType.addField(new FieldDescription(DEFAULT_INDEX, "string", "default-index def-idx defidx")); + argumentType.addField(new FieldDescription(DEFAULT_INDEX, "string", "default-index")); argumentType.addField(new FieldDescription(LANGUAGE, "string", "language lang")); argumentType.addField(new FieldDescription(ENCODING, "string", "encoding")); argumentType.addField(new FieldDescription(SOURCES, "string", "sources search")); @@ -93,34 +93,6 @@ public class Model implements Cloneable { setParent(query); } - /** - * Creates trace a message of language detection results into this Model - * instance's parent query. Do note this will give bogus results if the - * Execution instance is not set correctly. This is done automatically - * inside {@link Execution#search(Query)}. If tracing the same place as - * creating the query instance, {@link #setExecution(Execution)} has to be - * invoked first with the same Execution instance the query is intended to - * be run by. - * - * @deprecated do not use; language can now be assigned later and for parts of the query tree, making this quite useless - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void traceLanguage() { - if (getParent().getTraceLevel() < 2) return; - if (language != null) { - getParent().trace("Language " + getLanguage() + " specified directly as a parameter", false, 2); - } - else { - Language l = getParsingLanguage(); - // Don't include the query, it will trigger query parsing - getParent().trace("Detected language: " + l, false, 2); - getParent().trace("Language " + l + " determined by " + - (Language.fromEncoding(encoding) != Language.UNKNOWN ? "query encoding" : - "the characters in the terms") + ".", false, 2); - } - } - public Language getParsingLanguage() { return getParsingLanguage(queryString); } @@ -535,28 +507,4 @@ public class Model implements Cloneable { return false; } - /** - * Set the YTrace header value to use when transmitting this model to a - * search backend (of some kind). - * - * @param next string representation of header value - * @deprecated not used, ytrace has been discontinued - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void setYTraceHeaderToNext(String next) { } - - /** - * Get the YTrace header value to use when transmitting this model to a - * search backend (of some kind). Returns null if no ytrace data is not - * turned on. - * - * @deprecated not used, ytrace has been discontinued - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public String getYTraceHeaderToNext() { - return null; - } - } diff --git a/container-search/src/main/java/com/yahoo/search/query/Presentation.java b/container-search/src/main/java/com/yahoo/search/query/Presentation.java index ae179a2ba07..6b10fd0847c 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Presentation.java +++ b/container-search/src/main/java/com/yahoo/search/query/Presentation.java @@ -32,7 +32,6 @@ public class Presentation implements Cloneable { public static final String BOLDING = "bolding"; public static final String TIMING = "timing"; public static final String SUMMARY = "summary"; - public static final String REPORT_COVERAGE = "reportCoverage"; public static final String SUMMARY_FIELDS = "summaryFields"; /** The (short) name of the parameter holding the name of the return format to use */ @@ -45,7 +44,6 @@ public class Presentation implements Cloneable { argumentType.addField(new FieldDescription(BOLDING, "boolean", "bolding")); argumentType.addField(new FieldDescription(TIMING, "boolean", "timing")); argumentType.addField(new FieldDescription(SUMMARY, "string", "summary")); - argumentType.addField(new FieldDescription(REPORT_COVERAGE, "string", "reportcoverage")); argumentType.addField(new FieldDescription(FORMAT, "string", "format template")); argumentType.addField(new FieldDescription(SUMMARY_FIELDS, "string", "summaryFields")); argumentType.freeze(); @@ -96,16 +94,6 @@ public class Presentation implements Cloneable { /** Sets whether matching query terms should be bolded in the result */ public void setBolding(boolean bolding) { this.bolding = bolding; } - /** @deprecated coverage information is always returned */ - @Deprecated // OK - // TODO: Remove on Vespa 7 - public boolean getReportCoverage() { return true; } - - /** @deprecated coverage information is always returned */ - @Deprecated // OK - // TODO: Remove on Vespa 7 - public void setReportCoverage(boolean ignored) { } - /** Get the name of the format desired for result rendering. */ @NonNull public ComponentSpecification getRenderer() { return format; } diff --git a/container-search/src/main/java/com/yahoo/search/query/UniqueRequestId.java b/container-search/src/main/java/com/yahoo/search/query/UniqueRequestId.java index 49529936901..f57a5f4ab1e 100644 --- a/container-search/src/main/java/com/yahoo/search/query/UniqueRequestId.java +++ b/container-search/src/main/java/com/yahoo/search/query/UniqueRequestId.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query; -import com.yahoo.container.Server; - import java.util.concurrent.atomic.AtomicLong; /** @@ -27,17 +25,6 @@ public class UniqueRequestId { /** * Creates a session id which is unique across the cluster this runtime is a member of each time this is called. * Calling this causes synchronization. - * - * @deprecated use nextId(serverId) instead - */ - @Deprecated - public static UniqueRequestId next() { - return new UniqueRequestId(Server.get().getServerDiscriminator(), System.currentTimeMillis(), sequenceCounter.getAndIncrement()); - } - - /** - * Creates a session id which is unique across the cluster this runtime is a member of each time this is called. - * Calling this causes synchronization. */ public static UniqueRequestId next(String serverId) { return new UniqueRequestId(serverId, System.currentTimeMillis(), sequenceCounter.getAndIncrement()); diff --git a/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java b/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java index e2a025c8fae..6b28b166150 100644 --- a/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java +++ b/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java @@ -3,7 +3,7 @@ package com.yahoo.search.query.context; import com.yahoo.processing.execution.Execution; import com.yahoo.search.Query; -import com.yahoo.search.rendering.DefaultRenderer; +import com.yahoo.search.rendering.XmlRenderer; import com.yahoo.text.XMLWriter; import com.yahoo.yolean.trace.TraceNode; @@ -88,7 +88,7 @@ public class QueryContext implements Cloneable { XMLWriter xmlWriter=XMLWriter.from(writer); xmlWriter.openTag("meta").attribute("type",ID); TraceNode traceRoot=owner.getModel().getExecution().trace().traceNode().root(); - traceRoot.accept(new DefaultRenderer.RenderingVisitor(xmlWriter,owner.getStartTime())); + traceRoot.accept(new XmlRenderer.RenderingVisitor(xmlWriter, owner.getStartTime())); xmlWriter.closeTag(); } return true; diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java index 7aefa6f0cf2..ca437fb9def 100644 --- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java +++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java @@ -18,9 +18,7 @@ import com.yahoo.search.searchchain.Execution; public final class ParserEnvironment { private IndexFacts indexFacts = new IndexFacts(); - - @SuppressWarnings("deprecation") - private Linguistics linguistics = new SimpleLinguistics(false); + private Linguistics linguistics = new SimpleLinguistics(); private SpecialTokens specialTokens = new SpecialTokens(); public IndexFacts getIndexFacts() { diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java index 69d46527255..a57c0f98b45 100644 --- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java +++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java @@ -18,7 +18,7 @@ public final class ParserFactory { } /** - * Creates a {@link Parser} appropriate for the given <tt>Query.Type</tt>, providing the Parser with access to + * Creates a {@link Parser} appropriate for the given <code>Query.Type</code>, providing the Parser with access to * the {@link ParserEnvironment} given. * * @param type the query type for which to create a Parser diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index 60427aeb0af..9e043bc3dc9 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -29,12 +29,6 @@ import java.util.Map; */ public class QueryProperties extends Properties { - /** @deprecated use Query.nativeProperties */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public static final CompoundName[] PER_SOURCE_QUERY_PROPERTIES = - Query.nativeProperties.toArray(new CompoundName[] {}); - private Query query; private final CompiledQueryProfileRegistry profileRegistry; @@ -119,7 +113,6 @@ public class QueryProperties extends Properties { else if (key.size()==2 && key.first().equals(Presentation.PRESENTATION)) { if (key.last().equals(Presentation.BOLDING)) return query.getPresentation().getBolding(); if (key.last().equals(Presentation.SUMMARY)) return query.getPresentation().getSummary(); - if (key.last().equals(Presentation.REPORT_COVERAGE)) return true; // TODO: Remove this line on Vespa 7 if (key.last().equals(Presentation.FORMAT)) return query.getPresentation().getFormat(); if (key.last().equals(Presentation.TIMING)) return query.getPresentation().getTiming(); if (key.last().equals(Presentation.SUMMARY_FIELDS)) return query.getPresentation().getSummaryFields(); @@ -217,9 +210,9 @@ public class QueryProperties extends Properties { } else if (key.size() == 3 && key.get(1).equals(Ranking.SOFTTIMEOUT)) { SoftTimeout soft = ranking.getSoftTimeout(); - if (key.last().equals(SoftTimeout.ENABLE)) soft.setEnable(asBoolean(value, false)); - if (key.last().equals(SoftTimeout.FACTOR)) soft.setFactor(asDouble(value, 0.50)); - if (key.last().equals(SoftTimeout.TAILCOST)) soft.setTailcost(asDouble(value, 0.10)); + if (key.last().equals(SoftTimeout.ENABLE)) soft.setEnable(asBoolean(value, true)); + if (key.last().equals(SoftTimeout.FACTOR)) soft.setFactor(asDouble(value, null)); + if (key.last().equals(SoftTimeout.TAILCOST)) soft.setTailcost(asDouble(value, null)); } else if (key.size() == 3 && key.get(1).equals(Ranking.MATCHING)) { Matching matching = ranking.getMatching(); @@ -249,7 +242,7 @@ public class QueryProperties extends Properties { query.getPresentation().setTiming(asBoolean(value, true)); else if (key.last().equals(Presentation.SUMMARY_FIELDS)) query.getPresentation().setSummaryFields(asString(value,"")); - else if ( ! key.last().equals(Presentation.REPORT_COVERAGE)) // TODO: Change this line to "else" on Vespa 7 + else throwIllegalParameter(key.last(), Presentation.PRESENTATION); } else if (key.size()==2 && key.first().equals(Select.SELECT)) { diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java b/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java index ca6fd44af50..a05ecee8a1e 100644 --- a/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.ranking; +import com.yahoo.processing.request.CompoundName; import com.yahoo.search.query.Ranking; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; @@ -21,6 +22,10 @@ public class SoftTimeout implements Cloneable { public static final String FACTOR = "factor"; public static final String TAILCOST = "tailcost"; + /** The full property name for turning softtimeout on or off */ + public static final CompoundName enableProperty = + CompoundName.fromComponents(Ranking.RANKING, Ranking.SOFTTIMEOUT, ENABLE); + static { argumentType = new QueryProfileType(Ranking.SOFTTIMEOUT); argumentType.setStrict(true); @@ -31,14 +36,15 @@ public class SoftTimeout implements Cloneable { } public static QueryProfileType getArgumentType() { return argumentType; } - public Boolean enable = null; + private boolean enabled = true; private Double factor = null; private Double tailcost = null; - public void setEnable(boolean enable) { this.enable = enable; } + public void setEnable(boolean enable) { this.enabled = enable; } - public Boolean getEnable() { return enable; } + public Boolean getEnable() { return enabled; } + /** Override the adaptive factor determined on the content nodes */ public void setFactor(double factor) { if ((factor < 0.0) || (factor > 1.0)) { throw new IllegalArgumentException("factor must be in the range [0.0, 1.0], got " + factor); @@ -48,6 +54,7 @@ public class SoftTimeout implements Cloneable { public Double getFactor() { return factor; } + /** Override the tail cost factor determined on the content nodes */ public void setTailcost(double tailcost) { if ((tailcost < 0.0) || (tailcost > 1.0)) { throw new IllegalArgumentException("tailcost must be in the range [0.0, 1.0], got " + tailcost); @@ -59,15 +66,13 @@ public class SoftTimeout implements Cloneable { /** Internal operation - DO NOT USE */ public void prepare(RankProperties rankProperties) { - if (enable != null) { - rankProperties.put("vespa.softtimeout.enable", String.valueOf(enable)); - } - if (factor != null) { + if ( !enabled) return; + + rankProperties.put("vespa.softtimeout.enable", "true"); + if (factor != null) rankProperties.put("vespa.softtimeout.factor", String.valueOf(factor)); - } - if (tailcost != null) { + if (tailcost != null) rankProperties.put("vespa.softtimeout.tailcost", String.valueOf(tailcost)); - } } @Override @@ -83,7 +88,7 @@ public class SoftTimeout implements Cloneable { @Override public int hashCode() { int hash = 0; - if (enable != null) hash += 11 * enable.hashCode(); + if (enabled) hash += 11; if (factor != null) hash += 13 * factor.hashCode(); if (tailcost != null) hash += 17 * tailcost.hashCode(); return hash; @@ -95,7 +100,7 @@ public class SoftTimeout implements Cloneable { if ( ! (o instanceof SoftTimeout)) return false; SoftTimeout other = (SoftTimeout)o; - if ( ! Objects.equals(this.enable, other.enable)) return false; + if ( ! Objects.equals(this.enabled, other.enabled)) return false; if ( ! Objects.equals(this.factor, other.factor)) return false; if ( ! Objects.equals(this.tailcost, other.tailcost)) return false; return true; diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/LegacyCombinator.java b/container-search/src/main/java/com/yahoo/search/querytransform/LegacyCombinator.java deleted file mode 100644 index d6a88a8f4e7..00000000000 --- a/container-search/src/main/java/com/yahoo/search/querytransform/LegacyCombinator.java +++ /dev/null @@ -1,365 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.querytransform; - -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; - -import com.yahoo.component.chain.dependencies.Before; -import com.yahoo.language.Language; -import com.yahoo.log.LogLevel; -import com.yahoo.prelude.Index; -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.AndItem; -import com.yahoo.prelude.query.CompositeItem; -import com.yahoo.prelude.query.IndexedItem; -import com.yahoo.prelude.query.Item; -import com.yahoo.prelude.query.NotItem; -import com.yahoo.prelude.query.NullItem; -import com.yahoo.prelude.query.RankItem; -import com.yahoo.prelude.query.parser.CustomParser; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.yolean.Exceptions; -import com.yahoo.search.query.Properties; -import com.yahoo.search.query.QueryTree; -import com.yahoo.search.query.parser.ParserEnvironment; -import com.yahoo.search.query.parser.ParserFactory; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.searchchain.Execution; - -/** - * Compatibility layer to implement the old multi part query syntax, along with - * the features of QueryCombinator. Do <b>not</b> use both QueryCombinator and - * LegacyCombinator in a single search. - * - * <p> - * A searcher which grabs query parameters of the form - * "defidx.(identifier)=(index name)" and "query.(identifier)=(user query)", - * parses them and adds them as AND items to the query root. - * - * <p> - * If the given default index does not exist in the search definition, the query - * part will be parsed with the settings of the default index set to "". - * - * <p> - * If any of the following arguments exist, they will be used: - * - * <p> - * query.(identifier)=query string<br> - * query.(identifier).operator={"req", "rank", "not"}, where "req" is default<br> - * query.(identifier).defidx=default index<br> - * query.(identifier).type={"all", "any", "phrase", "adv", "web"} where "all" is - * default - * - * <p> - * If both defidx.(identifier) and any of - * query.(identifier).{operator,defidx,type} is present in the query, an - * InvalidQueryParameter error will be added, and the query will be passed - * through untransformed. - * - * @author Steinar Knutsen - */ -@Before({"transformedQuery", "com.yahoo.prelude.querytransform.StemmingSearcher"}) -public class LegacyCombinator extends Searcher { - - private static final String TYPESUFFIX = ".type"; - private static final String OPERATORSUFFIX = ".operator"; - private static final String DEFIDXSUFFIX = ".defidx"; - private static final String DEFIDXPREFIX = "defidx."; - private static final String QUERYPREFIX = "query."; - - private enum Combinator { - REQUIRED("req"), PREFERRED("rank"), EXCLUDED("not"); - - String parameterValue; - - private Combinator(String parameterValue) { - this.parameterValue = parameterValue; - } - - static Combinator getCombinator(String name) { - for (Combinator c : Combinator.values()) { - if (c.parameterValue.equals(name)) { - return c; - } - } - return REQUIRED; - } - } - - private static class QueryPart { - final String query; - final String defaultIndex; - final Combinator operator; - final String identifier; - final Query.Type syntax; - - QueryPart(String identifier, String defaultIndex, String oldIndex, - String operator, String query, String syntax) { - validateArguments(identifier, defaultIndex, oldIndex, - operator,syntax); - this.query = query; - if (defaultIndex != null) { - this.defaultIndex = defaultIndex; - } else { - this.defaultIndex = oldIndex; - } - this.operator = Combinator.getCombinator(operator); - this.identifier = identifier; - this.syntax = Query.Type.getType(syntax); - } - - private static void validateArguments(String identifier, String defaultIndex, - String oldIndex, String operator, String syntax) { - if (defaultIndex == null) { - return; - } - if (oldIndex != null) { - throw new IllegalArgumentException(createErrorMessage(identifier, DEFIDXSUFFIX)); - } - if (operator != null) { - throw new IllegalArgumentException(createErrorMessage(identifier, OPERATORSUFFIX)); - } - if (syntax != null) { - throw new IllegalArgumentException(createErrorMessage(identifier, TYPESUFFIX)); - } - } - - private static String createErrorMessage(String identifier, String legacyArgument) { - return "Cannot set both " + DEFIDXPREFIX + identifier + " and " - + QUERYPREFIX + identifier + legacyArgument + "."; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result - + ((identifier == null) ? 0 : identifier.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - QueryPart other = (QueryPart) obj; - if (identifier == null) { - if (other.identifier != null) - return false; - } else if (!identifier.equals(other.identifier)) - return false; - return true; - } - - @Override - public String toString() { - return "QueryPart(" + identifier + ", " + defaultIndex + ", " - + operator + ", " + syntax + ")"; - } - } - - @Override - public Result search(Query query, Execution execution) { - Set<QueryPart> pieces; - Set<String> usedSources; - IndexFacts indexFacts = execution.context().getIndexFacts(); - try { - pieces = findQuerySnippets(query.properties()); - } catch (IllegalArgumentException e) { - query.errors().add(ErrorMessage.createInvalidQueryParameter("LegacyCombinator got invalid parameters: " - + e.getMessage())); - return execution.search(query); - } - if (pieces.size() == 0) { - return execution.search(query); - } - IndexFacts.Session session = indexFacts.newSession(query); - Language language = query.getModel().getParsingLanguage(); - addAndItems(language, query, pieces, session, execution.context()); - addRankItems(language, query, pieces, session, execution.context()); - try { - addNotItems(language, query, pieces, session, execution.context()); - } catch (IllegalArgumentException e) { - query.errors().add(ErrorMessage.createInvalidQueryParameter("LegacyCombinator found only excluding terms, no including.")); - return execution.search(query); - } - query.trace("Adding extra query parts.", true, 2); - return execution.search(query); - } - - private void addNotItems(Language language, Query query, Set<QueryPart> pieces, - IndexFacts.Session session, Execution.Context context) { - for (QueryPart part : pieces) { - if (part.operator != Combinator.EXCLUDED) continue; - - String defaultIndex = defaultIndex(session, part); - Item item = parse(language, query, part, defaultIndex, context); - if (item == null) continue; - - setDefaultIndex(part, defaultIndex, item); - addNotItem(query.getModel().getQueryTree(), item); - } - - } - - private void addNotItem(QueryTree queryTree, Item item) { - Item root = queryTree.getRoot(); - // JavaDoc claims I can get null, code gives NullItem... well, well, well... - if (root instanceof NullItem || root == null) { - // errr... no positive branch at all? - throw new IllegalArgumentException("No positive terms for query."); - } else if (root.getClass() == NotItem.class) { - ((NotItem) root).addNegativeItem(item); - } else { - NotItem newRoot = new NotItem(); - newRoot.addPositiveItem(root); - newRoot.addNegativeItem(item); - queryTree.setRoot(newRoot); - } - } - - private void addRankItems(Language language, Query query, Set<QueryPart> pieces, IndexFacts.Session session, Execution.Context context) { - for (QueryPart part : pieces) { - if (part.operator != Combinator.PREFERRED) continue; - - String defaultIndex = defaultIndex(session, part); - Item item = parse(language, query, part, defaultIndex, context); - if (item == null) continue; - - setDefaultIndex(part, defaultIndex, item); - addRankItem(query.getModel().getQueryTree(), item); - } - } - - private void addRankItem(QueryTree queryTree, Item item) { - Item root = queryTree.getRoot(); - // JavaDoc claims I can get null, code gives NullItem... well, well, well... - if (root instanceof NullItem || root == null) { - queryTree.setRoot(item); - } else if (root.getClass() == RankItem.class) { - // if no clear recall terms, just set the rank term as recall - ((RankItem) root).addItem(item); - } else { - RankItem newRoot = new RankItem(); - newRoot.addItem(root); - newRoot.addItem(item); - queryTree.setRoot(newRoot); - } - } - - private void addAndItems(Language language, Query query, Iterable<QueryPart> pieces, IndexFacts.Session session, Execution.Context context) { - for (QueryPart part : pieces) { - if (part.operator != Combinator.REQUIRED) continue; - - String defaultIndex = defaultIndex(session, part); - Item item = parse(language, query, part, defaultIndex, context); - if (item == null) continue; - - setDefaultIndex(part, defaultIndex, item); - addAndItem(query.getModel().getQueryTree(), item); - } - } - - private void setDefaultIndex(QueryPart part, String defaultIndex, Item item) { - if (defaultIndex == null) { - assignDefaultIndex(item, part.defaultIndex); - } - } - - private Item parse(Language language, Query query, QueryPart part, String defaultIndex, Execution.Context context) { - Item item = null; - try { - CustomParser parser = (CustomParser)ParserFactory.newInstance( - part.syntax, ParserEnvironment.fromExecutionContext(context)); - item = parser.parse(part.query, null, language, query.getModel().getSources(), - context.getIndexFacts(), defaultIndex); - } catch (RuntimeException e) { - String err = Exceptions.toMessageString(e); - query.trace("Query parser threw an exception: " + err, true, 1); - getLogger().log(LogLevel.WARNING, - "Query parser threw exception in searcher LegacyCombinator for " - + query.getHttpRequest().toString() + ", query part " + part.query + ": " + err); - } - return item; - } - - private String defaultIndex(IndexFacts.Session indexFacts, QueryPart part) { - String defaultIndex; - if (indexFacts.getIndex(part.defaultIndex) == Index.nullIndex) { - defaultIndex = null; - } else { - defaultIndex = part.defaultIndex; - } - return defaultIndex; - } - - private static void addAndItem(QueryTree queryTree, Item item) { - Item root = queryTree.getRoot(); - // JavaDoc claims I can get null, code gives NullItem... well, well, well... - if (root instanceof NullItem || root == null) { - queryTree.setRoot(item); - } else if (root.getClass() == AndItem.class) { - ((AndItem) root).addItem(item); - } else { - AndItem newRoot = new AndItem(); - newRoot.addItem(root); - newRoot.addItem(item); - queryTree.setRoot(newRoot); - } - } - - private static void assignDefaultIndex(Item item, String defaultIndex) { - if (item instanceof IndexedItem) { - IndexedItem indexName = (IndexedItem) item; - - if ("".equals(indexName.getIndexName())) { - indexName.setIndexName(defaultIndex); - } - } else if (item instanceof CompositeItem) { - Iterator<Item> items = ((CompositeItem) item).getItemIterator(); - while (items.hasNext()) { - Item i = items.next(); - assignDefaultIndex(i, defaultIndex); - } - } - - } - - private static Set<QueryPart> findQuerySnippets(Properties properties) { - Set<QueryPart> pieces = new HashSet<>(); - for (Map.Entry<String, Object> k : properties.listProperties() - .entrySet()) { - String key = k.getKey(); - if (!key.startsWith(QUERYPREFIX)) { - continue; - } - String name = key.substring(QUERYPREFIX.length()); - if (hasDots(name)) { - continue; - } - String index = properties.getString(DEFIDXPREFIX + name); - String oldIndex = properties.getString(QUERYPREFIX + name - + DEFIDXSUFFIX); - String operator = properties.getString(QUERYPREFIX + name - + OPERATORSUFFIX); - String type = properties.getString(QUERYPREFIX + name + TYPESUFFIX); - pieces.add(new QueryPart(name, index, oldIndex, operator, k - .getValue().toString(), type)); - } - return pieces; - } - - private static boolean hasDots(String name) { - int index = name.indexOf('.', 0); - return index != -1; - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/QueryCombinator.java b/container-search/src/main/java/com/yahoo/search/querytransform/QueryCombinator.java deleted file mode 100644 index 67cb291a760..00000000000 --- a/container-search/src/main/java/com/yahoo/search/querytransform/QueryCombinator.java +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.querytransform; - -import com.yahoo.component.ComponentId; -import com.yahoo.language.Language; -import com.yahoo.log.LogLevel; -import com.yahoo.prelude.Index; -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.AndItem; -import com.yahoo.prelude.query.CompositeItem; -import com.yahoo.prelude.query.IndexedItem; -import com.yahoo.prelude.query.Item; -import com.yahoo.prelude.query.NullItem; -import com.yahoo.prelude.query.parser.CustomParser; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.query.Properties; -import com.yahoo.search.query.QueryTree; -import com.yahoo.search.query.parser.ParserEnvironment; -import com.yahoo.search.query.parser.ParserFactory; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.yolean.Exceptions; - -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; - -/** - * <p>A searcher which grabs query parameters of the form "defidx.(identifier)=(index name)" and - * "query.(identifier)=(user query)", * parses them and adds them as AND items to the query root.</p> - * - * <p>If the given default index does not exist in the search definition, the query part will be parsed with the - * settings of the default index set to the "".</p> - * - * @author Steinar Knutsen - * @deprecated use YQL - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class QueryCombinator extends Searcher { - - private static final String QUERYPREFIX = "query."; - - private static class QueryPart { - final String query; - final String defaultIndex; - - QueryPart(String query, String defaultIndex) { - this.query = query; - this.defaultIndex = defaultIndex; - } - } - - public QueryCombinator(ComponentId id) { - super(id); - } - - @Override - public Result search(Query query, Execution execution) { - Set<QueryPart> pieces = findQuerySnippets(query.properties()); - if (pieces.size() == 0) { - return execution.search(query); - } - addAndItems(query, pieces, execution.context()); - query.trace("Adding extra query parts.", true, 2); - return execution.search(query); - } - - private void addAndItems(Query query, Iterable<QueryPart> pieces, Execution.Context context) { - IndexFacts indexFacts = context.getIndexFacts(); - IndexFacts.Session session = indexFacts.newSession(query); - Set<String> usedSources = new HashSet<>(session.documentTypes()); - Language language = query.getModel().getParsingLanguage(); - for (QueryPart part : pieces) { - String defaultIndex; - Item item = null; - Index index = session.getIndex(part.defaultIndex); - if (index == Index.nullIndex) { - defaultIndex = null; - } else { - defaultIndex = part.defaultIndex; - } - try { - CustomParser parser = (CustomParser)ParserFactory.newInstance(query.getModel().getType(), - ParserEnvironment.fromExecutionContext(context)); - item = parser.parse(part.query, null, language, usedSources, indexFacts, defaultIndex); - } catch (RuntimeException e) { - String err = Exceptions.toMessageString(e); - query.trace("Query parser threw an exception: " + err, true, 1); - getLogger().log(LogLevel.WARNING, - "Query parser threw exception searcher QueryCombinator for " - + query.getHttpRequest().toString() + ", query part " + part.query + ": " + err); - } - if (item == null) { - continue; - } - if (defaultIndex == null) { - assignDefaultIndex(item, part.defaultIndex); - } - addAndItem(query.getModel().getQueryTree(), item); - } - } - - private static void addAndItem(QueryTree queryTree, Item item) { - Item root = queryTree.getRoot(); - // JavaDoc claims I can get null, code gives NullItem... well, well, well... - if (root instanceof NullItem || root == null) { - queryTree.setRoot(item); - } else if (root.getClass() == AndItem.class) { - ((AndItem) root).addItem(item); - } else { - AndItem newRoot = new AndItem(); - newRoot.addItem(root); - newRoot.addItem(item); - queryTree.setRoot(newRoot); - } - } - - private static void assignDefaultIndex(Item item, String defaultIndex) { - if (item instanceof IndexedItem) { - IndexedItem indexName = (IndexedItem) item; - - if ("".equals(indexName.getIndexName())) { - indexName.setIndexName(defaultIndex); - } - } else if (item instanceof CompositeItem) { - Iterator<Item> items = ((CompositeItem) item).getItemIterator(); - while (items.hasNext()) { - Item i = items.next(); - assignDefaultIndex(i, defaultIndex); - } - } - } - - private static Set<QueryPart> findQuerySnippets(Properties properties) { - Set<QueryPart> pieces = new HashSet<>(); - for (Map.Entry<String, Object> k : properties.listProperties().entrySet()) { - String key = k.getKey(); - if (!key.startsWith(QUERYPREFIX)) { - continue; - } - String name = key.substring(QUERYPREFIX.length()); - if (hasDots(name)) { - continue; - } - String index = properties.getString("defidx." + name); - pieces.add(new QueryPart(k.getValue().toString(), index)); - } - return pieces; - } - - private static boolean hasDots(String name) { - int index = name.indexOf('.', 0); - return index != -1; - } -} diff --git a/container-search/src/main/java/com/yahoo/search/rendering/DefaultRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/DefaultRenderer.java deleted file mode 100644 index 30695338741..00000000000 --- a/container-search/src/main/java/com/yahoo/search/rendering/DefaultRenderer.java +++ /dev/null @@ -1,427 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.rendering; - -import com.yahoo.concurrent.CopyOnWriteHashMap; -import com.yahoo.data.XmlProducer; -import com.yahoo.io.ByteWriter; -import com.yahoo.net.URI; -import com.yahoo.prelude.fastsearch.GroupingListHit; -import com.yahoo.prelude.hitfield.HitField; -import com.yahoo.prelude.hitfield.JSONString; -import com.yahoo.prelude.hitfield.XMLString; -import com.yahoo.processing.rendering.AsynchronousSectionedRenderer; -import com.yahoo.processing.response.Data; -import com.yahoo.processing.response.DataList; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.grouping.result.HitRenderer; -import com.yahoo.search.query.context.QueryContext; -import com.yahoo.search.result.*; -import com.yahoo.text.Utf8String; -import com.yahoo.text.XML; -import com.yahoo.text.XMLWriter; -import com.yahoo.yolean.trace.TraceNode; -import com.yahoo.yolean.trace.TraceVisitor; - -import java.io.IOException; -import java.io.OutputStream; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.io.Writer; -import java.nio.charset.Charset; -import java.nio.charset.CharsetEncoder; -import java.util.Iterator; -import java.util.concurrent.Executor; -import java.util.stream.Collectors; - -/** - * XML rendering of search results. This is NOT the default (but it once was). - * - * @author Tony Vaagenes - * @deprecated use JsonRenderer instead - */ -@SuppressWarnings({ "rawtypes", "deprecation" }) -@Deprecated // OK -// TODO: Rename to XmlRenderer on Vespa 7 -public final class DefaultRenderer extends AsynchronousSectionedRenderer<Result> { - - public static final String DEFAULT_MIMETYPE = "text/xml"; - public static final String DEFAULT_ENCODING = "utf-8"; - - private static final Utf8String RESULT = new Utf8String("result"); - private static final Utf8String GROUP = new Utf8String("group"); - private static final Utf8String ID = new Utf8String("id"); - private static final Utf8String FIELD = new Utf8String("field"); - private static final Utf8String HIT = new Utf8String("hit"); - private static final Utf8String ERROR = new Utf8String("error"); - private static final Utf8String TOTAL_HIT_COUNT = new Utf8String("total-hit-count"); - private static final Utf8String QUERY_TIME = new Utf8String("querytime"); - private static final Utf8String SUMMARY_FETCH_TIME = new Utf8String("summaryfetchtime"); - private static final Utf8String SEARCH_TIME = new Utf8String("searchtime"); - private static final Utf8String NAME = new Utf8String("name"); - private static final Utf8String CODE = new Utf8String("code"); - private static final Utf8String COVERAGE_DOCS = new Utf8String("coverage-docs"); - private static final Utf8String COVERAGE_NODES = new Utf8String("coverage-nodes"); - private static final Utf8String COVERAGE_FULL = new Utf8String("coverage-full"); - private static final Utf8String COVERAGE = new Utf8String("coverage"); - private static final Utf8String RESULTS_FULL = new Utf8String("results-full"); - private static final Utf8String RESULTS = new Utf8String("results"); - private static final Utf8String TYPE = new Utf8String("type"); - private static final Utf8String RELEVANCY = new Utf8String("relevancy"); - private static final Utf8String SOURCE = new Utf8String("source"); - - - // this is shared between umpteen threads by design - private final CopyOnWriteHashMap<String, Utf8String> fieldNameMap = new CopyOnWriteHashMap<>(); - - private XMLWriter writer; - - public DefaultRenderer() { - this(null); - } - - /** - * Creates an XML renderer using a custom executor. - * Using a custom executor is useful for tests to avoid creating new threads for each renderer registry. - */ - public DefaultRenderer(Executor executor) { - super(executor); - } - - @Override - public void init() { - super.init(); - writer = null; - } - - @Override - public String getEncoding() { - if (getResult() == null - || getResult().getQuery() == null - || getResult().getQuery().getModel().getEncoding() == null) { - return DEFAULT_ENCODING; - } else { - return getResult().getQuery().getModel().getEncoding(); - } - } - - @Override - public String getMimeType() { - return DEFAULT_MIMETYPE; - } - - private XMLWriter wrapWriter(Writer writer) { - return XMLWriter.from(writer, 10, -1); - } - - private void header(XMLWriter writer, Result result) throws IOException { - // TODO: move setting this to Result - writer.xmlHeader(getRequestedEncoding(result.getQuery())); - writer.openTag(RESULT).attribute(TOTAL_HIT_COUNT, String.valueOf(result.getTotalHitCount())); - renderCoverageAttributes(result.getCoverage(false), writer); - renderTime(writer, result); - writer.closeStartTag(); - } - - private void renderTime(XMLWriter writer, Result result) { - if ( ! result.getQuery().getPresentation().getTiming()) return; - - final String threeDecimals = "%.3f"; - final double milli = .001d; - final long now = System.currentTimeMillis(); - final long searchTime = now - result.getElapsedTime().first(); - final double searchSeconds = ((double) searchTime) * milli; - - if (result.getElapsedTime().firstFill() != 0L) { - final long queryTime = result.getElapsedTime().weightedSearchTime(); - final long summaryFetchTime = result.getElapsedTime().weightedFillTime(); - final double querySeconds = ((double) queryTime) * milli; - final double summarySeconds = ((double) summaryFetchTime) * milli; - writer.attribute(QUERY_TIME, String.format(threeDecimals, querySeconds)); - writer.attribute(SUMMARY_FETCH_TIME, String.format(threeDecimals, summarySeconds)); - } - writer.attribute(SEARCH_TIME, String.format(threeDecimals, searchSeconds)); - } - - protected static void renderCoverageAttributes(Coverage coverage, XMLWriter writer) throws IOException { - if (coverage == null) return; - writer.attribute(COVERAGE_DOCS,coverage.getDocs()); - writer.attribute(COVERAGE_NODES,coverage.getNodes()); - writer.attribute(COVERAGE_FULL,coverage.getFull()); - writer.attribute(COVERAGE,coverage.getResultPercentage()); - writer.attribute(RESULTS_FULL,coverage.getFullResultSets()); - writer.attribute(RESULTS,coverage.getResultSets()); - } - - public void error(XMLWriter writer, Result result) throws IOException { - ErrorMessage error = result.hits().getError(); - writer.openTag(ERROR).attribute(CODE,error.getCode()).content(error.getMessage(),false).closeTag(); - } - - @SuppressWarnings("UnusedParameters") - protected void emptyResult(XMLWriter writer, Result result) throws IOException {} - - @SuppressWarnings("UnusedParameters") - public void queryContext(XMLWriter writer, QueryContext queryContext, Query owner) throws IOException { - if (owner.getTraceLevel()!=0) { - XMLWriter xmlWriter=XMLWriter.from(writer); - xmlWriter.openTag("meta").attribute("type", QueryContext.ID); - TraceNode traceRoot = owner.getModel().getExecution().trace().traceNode().root(); - traceRoot.accept(new RenderingVisitor(xmlWriter, owner.getStartTime())); - xmlWriter.closeTag(); - } - } - - private void renderSingularHit(XMLWriter writer, Hit hit) { - writer.openTag(HIT); - renderHitAttributes(writer, hit); - writer.closeStartTag(); - renderHitFields(writer, hit); - } - - private void renderHitFields(XMLWriter writer, Hit hit) { - renderSyntheticRelevanceField(writer, hit); - hit.forEachField((name, value) -> renderField(writer, name, value)); - } - - private void renderField(XMLWriter writer, String name, Object value) { - if (name.startsWith("$")) return; - - writeOpenFieldElement(writer, name); - renderFieldContent(writer, value); - writeCloseFieldElement(writer); - } - - private void renderFieldContent(XMLWriter writer, Object value) { - writer.escapedContent(asXML(value), false); - } - - private String asXML(Object value) { - if (value == null) - return "(null)"; - else if (value instanceof XmlProducer) - return ((XmlProducer)value).toXML(); - else if (value instanceof HitField) - return ((HitField)value).quotedContent(false); - else if (value instanceof StructuredData || value instanceof XMLString || value instanceof JSONString) - return value.toString(); - else - return XML.xmlEscape(value.toString(), false, '\u001f'); - } - - private void renderSyntheticRelevanceField(XMLWriter writer, Hit hit) { - String relevancyFieldName = "relevancy"; - Relevance relevance = hit.getRelevance(); - - // skip depending on hit type - if (relevance != null) { - renderSimpleField(writer, relevancyFieldName, relevance); - } - } - - private void renderSimpleField(XMLWriter writer, String relevancyFieldName, Relevance relevance) { - writeOpenFieldElement(writer, relevancyFieldName); - writer.content(relevance.toString(), false); - writeCloseFieldElement(writer); - } - - private void writeCloseFieldElement(XMLWriter writer) { - writer.closeTag(); - } - - private void writeOpenFieldElement(XMLWriter writer, String relevancyFieldName) { - Utf8String utf8 = fieldNameMap.get(relevancyFieldName); - if (utf8 == null) { - utf8 = new Utf8String(relevancyFieldName); - fieldNameMap.put(relevancyFieldName, utf8); - } - writer.openTag(FIELD).attribute(NAME, utf8); - writer.closeStartTag(); - } - - private void renderHitAttributes(XMLWriter writer, Hit hit) { - writer.attribute(TYPE, hit.types().stream().collect(Collectors.joining(" "))); - if (hit.getRelevance() != null) - writer.attribute(RELEVANCY, hit.getRelevance().toString()); - writer.attribute(SOURCE, hit.getSource()); - } - - private void renderHitGroup(XMLWriter writer, HitGroup hit) throws IOException { - if (HitRenderer.renderHeader(hit, writer)) { - // empty - } else if (hit.types().contains("grouphit")) { - // TODO Keep this? - renderHitGroupOfTypeGroupHit(writer, hit); - } else { - renderGroup(writer, hit); - } - } - - private void renderGroup(XMLWriter writer, HitGroup hit) { - writer.openTag(GROUP); - renderHitAttributes(writer, hit); - writer.closeStartTag(); - } - - private void renderHitGroupOfTypeGroupHit(XMLWriter writer, HitGroup hit) { - writer.openTag(HIT); - renderHitAttributes(writer, hit); - renderId(writer, hit); - writer.closeStartTag(); - } - - private void renderId(XMLWriter writer, HitGroup hit) { - URI uri = hit.getId(); - if (uri != null) { - writer.openTag(ID).content(uri.stringValue(),false).closeTag(); - } - } - - private boolean simpleRenderHit(XMLWriter writer, Hit hit) throws IOException { - if (hit instanceof DefaultErrorHit) { - return simpleRenderDefaultErrorHit(writer, (DefaultErrorHit) hit); - } else if (hit instanceof GroupingListHit) { - return true; - } else { - return false; - } - } - - public static boolean simpleRenderDefaultErrorHit(XMLWriter writer, ErrorHit defaultErrorHit) throws IOException { - writer.openTag("errordetails"); - for (Iterator i = defaultErrorHit.errorIterator(); i.hasNext();) { - ErrorMessage error = (ErrorMessage) i.next(); - renderMessageDefaultErrorHit(writer, error); - } - writer.closeTag(); - return true; - } - - public static void renderMessageDefaultErrorHit(XMLWriter writer, ErrorMessage error) throws IOException { - writer.openTag("error"); - writer.attribute("source", error.getSource()); - writer.attribute("error", error.getMessage()); - writer.attribute("code", Integer.toString(error.getCode())); - writer.content(error.getDetailedMessage(), false); - if (error.getCause()!=null) { - writer.openTag("cause"); - writer.content("\n", true); - StringWriter stackTrace=new StringWriter(); - error.getCause().printStackTrace(new PrintWriter(stackTrace)); - writer.content(stackTrace.toString(), true); - writer.closeTag(); - } - writer.closeTag(); - } - - public static final class RenderingVisitor extends TraceVisitor { - - private static final String tag = "p"; - private final XMLWriter writer; - private long baseTime; - - public RenderingVisitor(XMLWriter writer,long baseTime) { - this.writer=writer; - this.baseTime=baseTime; - } - - @Override - public void entering(TraceNode node) { - if (node.isRoot()) return; - writer.openTag(tag); - } - - @Override - public void leaving(TraceNode node) { - if (node.isRoot()) return; - writer.closeTag(); - } - - @Override - public void visit(TraceNode node) { - if (node.isRoot()) return; - if (node.payload()==null) return; - - writer.openTag(tag); - if (node.timestamp()!=0) - writer.content(node.timestamp()-baseTime,false).content(" ms: ", false); - writer.content(node.payload().toString(),false); - writer.closeTag(); - } - - } - - private Result getResult() { - Result r; - try { - r = (Result) getResponse(); - } catch (ClassCastException e) { - throw new IllegalArgumentException( - "DefaultRenderer attempted used outside a search context, got a " - + getResponse().getClass().getName()); - } - return r; - } - - @Override - public void beginResponse(OutputStream stream) throws IOException { - Charset cs = Charset.forName(getRequestedEncoding(getResult().getQuery())); - CharsetEncoder encoder = cs.newEncoder(); - writer = wrapWriter(new ByteWriter(stream, encoder)); - - header(writer, getResult()); - if (getResult().hits().getError() != null || getResult().hits().getQuery().errors().size() > 0) { - error(writer, getResult()); - } - - if (getResult().getConcreteHitCount() == 0) { - emptyResult(writer, getResult()); - } - - if (getResult().getContext(false) != null) { - queryContext(writer, getResult().getContext(false), getResult().getQuery()); - } - - } - - /** Returns the encoding of the query, or the encoding given by the template if none is set */ - public final String getRequestedEncoding(Query query) { - String encoding = query.getModel().getEncoding(); - if (encoding != null) return encoding; - return getEncoding(); - } - - @Override - public void beginList(DataList<?> list) throws IOException { - if (getRecursionLevel() == 1) return; - - HitGroup hit = (HitGroup) list; - boolean renderedSimple = simpleRenderHit(writer, hit); - if (renderedSimple) return; - - renderHitGroup(writer, hit); - } - - @Override - public void data(Data data) throws IOException { - Hit hit = (Hit) data; - boolean renderedSimple = simpleRenderHit(writer, hit); - if (renderedSimple) return; - - renderSingularHit(writer, hit); - writer.closeTag(); - } - - @Override - public void endList(DataList<?> list) { - if (getRecursionLevel() > 1) - writer.closeTag(); - } - - @Override - public void endResponse() { - writer.closeTag(); - writer.close(); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index 7d60d7cf9ee..e8af150ce25 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -18,6 +18,7 @@ import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.json.JsonWriter; import com.yahoo.lang.MutableBoolean; +import com.yahoo.prelude.hitfield.HitField; import com.yahoo.processing.Response; import com.yahoo.processing.execution.Execution.Trace; import com.yahoo.processing.rendering.AsynchronousSectionedRenderer; @@ -473,7 +474,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { generator.writeNumberField(RELEVANCE, hit.getRelevance().getScore()); - if (hit.types().size() > 0) { // TODO: Remove types rendering on Vespa 7 + if (hit.types().size() > 0) { generator.writeArrayFieldStart(TYPES); for (String t : hit.types()) { generator.writeString(t); diff --git a/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java b/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java index 0f01261f6a4..783045babf4 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java @@ -4,12 +4,9 @@ package com.yahoo.search.rendering; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.provider.ComponentRegistry; -import com.yahoo.prelude.templates.PageTemplateSet; -import com.yahoo.prelude.templates.SearchRendererAdaptor; -import com.yahoo.prelude.templates.TiledTemplateSet; -import com.yahoo.prelude.templates.UserTemplate; import com.yahoo.processing.rendering.Renderer; import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.result.PageTemplatesXmlRenderer; import java.util.Collection; import java.util.Collections; @@ -23,14 +20,11 @@ import java.util.concurrent.Executor; */ public final class RendererRegistry extends ComponentRegistry<com.yahoo.processing.rendering.Renderer<Result>> { - public static final ComponentId xmlRendererId = ComponentId.fromString("DefaultRenderer"); - private static final ComponentId newXmlRendererId = ComponentId.fromString("XmlRenderer"); + public static final ComponentId xmlRendererId = ComponentId.fromString("XmlRenderer"); + public static final ComponentId pageRendererId = ComponentId.fromString("PageTemplatesXmlRenderer"); public static final ComponentId jsonRendererId = ComponentId.fromString("JsonRenderer"); public static final ComponentId defaultRendererId = jsonRendererId; - private final ComponentId tiledRendererId; - private final ComponentId pageRendererId; - /** Creates a registry containing the built-in renderers only */ public RendererRegistry() { this(Collections.emptyList()); @@ -61,23 +55,19 @@ public final class RendererRegistry extends ComponentRegistry<com.yahoo.processi register(jsonRenderer.getId(), jsonRenderer); // Add xml renderer - Renderer xmlRenderer = new DefaultRenderer(executor); + Renderer xmlRenderer = new XmlRenderer(executor); xmlRenderer.initId(xmlRendererId); register(xmlRenderer.getId(), xmlRenderer); - // Add new Vespa 7 xml renderer - Renderer newXmlRenderer = new XmlRenderer(executor); - newXmlRenderer.initId(newXmlRendererId); - register(newXmlRenderer.getId(), newXmlRenderer); + // Add page templates renderer + Renderer pageRenderer = new PageTemplatesXmlRenderer(executor); + pageRenderer.initId(pageRendererId); + register(pageRenderer.getId(), pageRenderer); // add application renderers for (Renderer renderer : renderers) register(renderer.getId(), renderer); - // add legacy "templates" converted to renderers // TODO: Remove on Vespa 7 - tiledRendererId = addTemplateSet(new TiledTemplateSet()); - pageRendererId = addTemplateSet(new PageTemplateSet()); - freeze(); } @@ -86,20 +76,9 @@ public final class RendererRegistry extends ComponentRegistry<com.yahoo.processi // deconstruct the renderers which was created by this getRenderer(jsonRendererId.toSpecification()).deconstruct(); getRenderer(xmlRendererId.toSpecification()).deconstruct(); - getRenderer(newXmlRendererId.toSpecification()).deconstruct(); - getRenderer(tiledRendererId.toSpecification()).deconstruct(); getRenderer(pageRendererId.toSpecification()).deconstruct(); } - @SuppressWarnings({"deprecation", "unchecked"}) - private ComponentId addTemplateSet(UserTemplate<?> templateSet) { - Renderer renderer = new SearchRendererAdaptor(templateSet); - ComponentId rendererId = new ComponentId(templateSet.getName()); - renderer.initId(rendererId); - register(rendererId, renderer); - return rendererId; - } - /** * Returns the default JSON renderer * @@ -120,6 +99,7 @@ public final class RendererRegistry extends ComponentRegistry<com.yahoo.processi if (format == null || format.stringValue().equals("default")) return getDefaultRenderer(); if (format.stringValue().equals("json")) return getComponent(jsonRendererId); if (format.stringValue().equals("xml")) return getComponent(xmlRendererId); + if (format.stringValue().equals("page")) return getComponent(pageRendererId); com.yahoo.processing.rendering.Renderer<Result> renderer = getComponent(format); if (renderer == null) diff --git a/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java index 5f99c531c95..5586fd2f996 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java @@ -349,15 +349,12 @@ public final class XmlRenderer extends AsynchronousSectionedRenderer<Result> { } private Result getResult() { - Result r; try { - r = (Result) getResponse(); + return (Result) getResponse(); } catch (ClassCastException e) { - throw new IllegalArgumentException( - "XmlRenderer attempted used outside a search context, got a " - + getResponse().getClass().getName()); + throw new IllegalArgumentException("XmlRenderer attempted used outside a search context, got a " + + getResponse().getClass().getName()); } - return r; } @Override diff --git a/container-search/src/main/java/com/yahoo/search/result/Coverage.java b/container-search/src/main/java/com/yahoo/search/result/Coverage.java index 7bc49fdb0b1..25829b70b5e 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Coverage.java +++ b/container-search/src/main/java/com/yahoo/search/result/Coverage.java @@ -23,20 +23,6 @@ public class Coverage extends com.yahoo.container.handler.Coverage { super(docs, active, nodes, resultSets); } - // TODO: Remove on Vespa 7 - /** @deprecated don't send a "full" boolean */ - @Deprecated // OK - public Coverage(long docs, int nodes, boolean full) { - this(docs, nodes, full, 1); - } - - // TODO: Remove on Vespa 7 - /** @deprecated don't send a "full" boolean */ - @Deprecated // OK - public Coverage(long docs, int nodes, boolean full, int resultSets) { - super(docs, nodes, full, resultSets); - } - /** * Will set number of documents present in ideal state * diff --git a/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java b/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java index 54b83544b1a..d8ec768325a 100644 --- a/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java +++ b/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java @@ -58,18 +58,6 @@ public class DefaultErrorHit extends Hit implements ErrorHit, Cloneable { } /** - * Returns the main error of this result, never null. - * - * @deprecated use {@link #errors()} - */ - @Override - @Deprecated // OK - // TODO: Remove on Vespa 7 - public ErrorMessage getMainError() { - return errors.get(0); - } - - /** * This is basically a way of making a list simulate a set. */ private void removeAndAdd(ErrorMessage error) { diff --git a/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java b/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java index 091c8d67793..9be80372ba9 100644 --- a/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java +++ b/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java @@ -14,15 +14,6 @@ public interface ErrorHit extends Cloneable { void setSource(String source); - /** - * Returns the main error of this result, never null - * - * @deprecated use errors().iterator().next() - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - ErrorMessage getMainError(); - /** * Adds an error to this. This may change the main error * and/or the list of detailed errors diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java index c5132549d05..db5b3e69803 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Hit.java +++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java @@ -106,9 +106,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi /** If this is true, then this hit will not be counted as a concrete hit */ private boolean auxiliary = false; - /** - * The hit field used to store rank features. TODO: Remove on Vespa 7 - */ + /** The hit field used to store rank features */ public static final String RANKFEATURES_FIELD = "rankfeatures"; public static final String SDDOCNAME_FIELD = "sddocname"; @@ -420,7 +418,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi */ public void forEachFieldAsRaw(RawUtf8Consumer consumer) { if (fields == null) return; - fields.forEach(consumer); // No utf-8 fields available in Hit + fields.forEach(consumer); } /** Returns the fields of this as a read-only map. This is more costly than fieldIterator() */ @@ -478,20 +476,6 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi return fields != null && ! fields.isEmpty(); } - /** - * Changes the key under which a value is found. This is useful because it allows keys to be changed - * without accessing the value (which may be lazily created). - * - * @deprecated do not use - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void changeFieldKey(String oldKey, String newKey) { - Map<String,Object> fieldMap = getFieldMap(); - Object value = fieldMap.remove(oldKey); - fieldMap.put(newKey, value); - } - private Map<String, Object> getFieldMap() { return getFieldMap(2); } @@ -515,42 +499,16 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi return unmodifiableFieldMap; } - /** Generate a HitField from a field if the field exists */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public HitField buildHitField(String key) { - return buildHitField(key, false); - } - - /** Generate a HitField from a field if the field exists */ - @SuppressWarnings("deprecation") - // TODO: Remove on Vespa 7 - @Deprecated // OK public HitField buildHitField(String key, boolean forceNoPreTokenize) { - return buildHitField(key, forceNoPreTokenize, false); - } - - // TODO: Remove third parameter on Vespa 7 (set always true) - @Deprecated // OK - public HitField buildHitField(String key, boolean forceNoPreTokenize, boolean forceStringHandling) { Object o = getField(key); if (o == null) return null; if (o instanceof HitField) return (HitField)o; HitField h; - if (forceNoPreTokenize) { - if (o instanceof XMLString && !forceStringHandling) { - h = new HitField(key, (XMLString) o, false); - } else { - h = new HitField(key, o.toString(), false); - } - } else { - if (o instanceof XMLString && !forceStringHandling) { - h = new HitField(key, (XMLString) o); - } else { - h = new HitField(key, o.toString()); - } - } + if (forceNoPreTokenize) + h = new HitField(key, o.toString(), false); + else + h = new HitField(key, o.toString()); h.setOriginal(o); getFieldMap().put(key, h); return h; @@ -559,34 +517,19 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi /** Returns the types of this as a modifiable set. Modifications to this set are directly reflected in this hit */ public Set<String> types() { return types; } - /** @deprecated do not use */ - // TODO: FRemove on Vespa 7 - @Deprecated // OK - public String getTypeString() { - return types().stream().collect(Collectors.joining(" ")); - } - /** * Returns the add number, assigned when adding the hit to a Result. * * Used to order equal relevant hit by add order. -1 if this hit * has never been added to a result. - * - * @deprecated do not use */ - // TODO: Make package private on Vespa 7 - @Deprecated // OK - public int getAddNumber() { return addNumber; } + int getAddNumber() { return addNumber; } /** * Sets the add number, assigned when adding the hit to a Result, * used to order equal relevant hit by add order. - * - * @deprecated do not use */ - // TODO: Make package private on Vespa 7 - @Deprecated // OK - public void setAddNumber(int addNumber) { this.addNumber = addNumber; } + void setAddNumber(int addNumber) { this.addNumber = addNumber; } /** * Returns whether this is a concrete hit, containing content of the requested @@ -610,16 +553,6 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi public void setAuxiliary(boolean auxiliary) { this.auxiliary = auxiliary; } - /** @deprecated do not use */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public int getSourceNumber() { return sourceNumber; } - - /** @deprecated do not use */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void setSourceNumber(int number) { this.sourceNumber = number; } - /** Returns the query which produced this hit, or null if not known */ public Query getQuery() { return query; } @@ -633,52 +566,6 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi } } - /** - * Returns a field of this hit XML escaped and without token - * delimiters. - * - * @deprecated do not use - * @return a field of this hit, or null if the property is not set - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public String getFieldXML(String key) { - Object p = getField(key); - - if (p == null) { - return null; - } else if (p instanceof HitField) { - return ((HitField)p).quotedContent(false); - } else if (p instanceof StructuredData || p instanceof XMLString || p instanceof JSONString) { - return p.toString(); - } else { - return XML.xmlEscape(p.toString(), false, '\u001f'); - } - } - - /** - * @deprecated do not use - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public String getUnboldedField(String key, boolean escape) { - Object p = getField(key); - - if (p == null) { - return null; - } else if (p instanceof HitField) { - return ((HitField) p).bareContent(escape, false); - } else if (p instanceof StructuredData) { - return p.toString(); - } else if (p instanceof XMLString || p instanceof JSONString) { - return p.toString(); - } else if (escape) { - return XML.xmlEscape(p.toString(), false, '\u001f'); - } else { - return stripCharacter('\u001F', p.toString()); - } - } - /** Attach some data to this hit for this searcher */ public void setSearcherSpecificMetaData(Searcher searcher, Object data) { if (searcherSpecificMetaData == null) { @@ -703,60 +590,21 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi return searcherSpecificMetaData != null ? searcherSpecificMetaData.get(searcher) : null; } - /** - * Internal - do not use - * - * @param filled the backing set - */ - // TODO: Make package private on Vespa 7 - protected final void setFilledInternal(Set<String> filled) { + final void setFilledInternal(Set<String> filled) { this.filled = filled; unmodifiableFilled = (filled != null) ? Collections.unmodifiableSet(filled) : null; } /** - * For internal use only. * Gives access to the modifiable backing set of filled summaries. * This set might be unmodifiable if the size is less than or equal to 1 * * @return the set of filled summaries. */ - // TODO: Make package private on Vespa 7 - protected final Set<String> getFilledInternal() { + final Set<String> getFilledInternal() { return filled; } - /** - * @deprecated do not use - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public static String stripCharacter(char strip, String toStripFrom) { - StringBuilder builder = null; - - int lastBadChar = 0; - for (int i = 0; i < toStripFrom.length(); i++) { - if (toStripFrom.charAt(i) == strip) { - if (builder == null) { - builder = new StringBuilder(toStripFrom.length()); - } - - builder.append(toStripFrom, lastBadChar, i); - lastBadChar = i + 1; - } - } - - if (builder == null) { - return toStripFrom; - } else { - if (lastBadChar < toStripFrom.length()) { - builder.append(toStripFrom, lastBadChar, toStripFrom.length()); - } - - return builder.toString(); - } - } - /** Releases the resources held by this, making it irreversibly unusable */ protected void close() { query = null; @@ -832,8 +680,8 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi * * @param fieldName the name of the field * @param utf8Data raw utf-8 data. The reciver <b>must not</b> modify this data - * @param offset the start index of the data to accept into the utf8Data array - * @param length the length of the data to accept into the utf8Data array + * @param offset the start index in the utf8Data array of the data to accept + * @param length the length starting from offset in the utf8Data array of the data to accept */ void accept(String fieldName, byte[] utf8Data, int offset, int length); diff --git a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java index 3d5cbab2865..af89220e504 100644 --- a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java +++ b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java @@ -383,17 +383,6 @@ public class HitGroup extends Hit implements DataList<Hit>, Cloneable, Iterable< return hit; } - /** - * Sets the main error of this result - * - * @deprecated prefer addError to add some error information. - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void setError(ErrorMessage error) { - addError(error); - } - /** Adds an error to this result */ public void addError(ErrorMessage error) { getError(); // update the list of errors diff --git a/container-search/src/main/java/com/yahoo/search/result/PositionsData.java b/container-search/src/main/java/com/yahoo/search/result/PositionsData.java index a07eaa2438d..483849a5435 100644 --- a/container-search/src/main/java/com/yahoo/search/result/PositionsData.java +++ b/container-search/src/main/java/com/yahoo/search/result/PositionsData.java @@ -17,8 +17,8 @@ public class PositionsData implements Inspectable, JsonProducer, XmlProducer { public PositionsData(Inspector value) { this.value = value; - if (value.type() != Type.ARRAY) { - throw new IllegalArgumentException("PositionsData expects an array of positions, got: "+value); + if (value.type() != Type.OBJECT && value.type() != Type.ARRAY) { + throw new IllegalArgumentException("PositionsData expects a position or an array of positions, got: "+value); } } @@ -38,27 +38,35 @@ public class PositionsData implements Inspectable, JsonProducer, XmlProducer { @Override public StringBuilder writeXML(StringBuilder target) { - for (int i = 0; i < value.entryCount(); i++) { - Inspector pos = value.entry(i); - target.append("<position "); - for (java.util.Map.Entry<String, Inspector> entry : pos.fields()) { - Inspector v = entry.getValue(); - if (v.type() == Type.STRING) { - target.append(entry.getKey()); - target.append("=\""); - target.append(entry.getValue().asString()); - target.append("\" "); - } - if (v.type() == Type.LONG) { - target.append(entry.getKey()); - target.append("=\""); - target.append(entry.getValue().asLong()); - target.append("\" "); - } + if (value.type() == Type.OBJECT) { + writeXML(value.inspect(), target); + } else { + for (int i = 0; i < value.entryCount(); i++) { + Inspector pos = value.entry(i); + writeXML(pos, target); } - target.append("/>"); } return target; } + private static void writeXML(Inspector pos, StringBuilder target) { + target.append("<position "); + for (java.util.Map.Entry<String, Inspector> entry : pos.fields()) { + Inspector v = entry.getValue(); + if (v.type() == Type.STRING) { + target.append(entry.getKey()); + target.append("=\""); + target.append(entry.getValue().asString()); + target.append("\" "); + } + if (v.type() == Type.LONG) { + target.append(entry.getKey()); + target.append("=\""); + target.append(entry.getValue().asLong()); + target.append("\" "); + } + } + target.append("/>"); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/result/Templating.java b/container-search/src/main/java/com/yahoo/search/result/Templating.java deleted file mode 100644 index a8b1eedf528..00000000000 --- a/container-search/src/main/java/com/yahoo/search/result/Templating.java +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.result; - -import java.util.Map; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.prelude.templates.SearchRendererAdaptor; -import com.yahoo.prelude.templates.TemplateSet; -import com.yahoo.prelude.templates.UserTemplate; -import com.yahoo.processing.rendering.Renderer; -import com.yahoo.search.Result; -import com.yahoo.search.query.Presentation; - -/** - * Helper methods and data store for result attributes geared towards result - * rendering and presentation. - * - * @author Steinar Knutsen - * @deprecated do not use - */ -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -// TODO: Remove on Vespa 7 -public class Templating { - - private final Result result; - private Renderer<Result> renderer; - - public Templating(Result result) { - super(); - this.result = result; - } - - /** - * Returns The first hit presented in the result as an index into the global - * list of all hits generated by the user query. - */ - public int getFirstHitNo() { - return result.getQuery().getOffset() + 1; - } - - /** - * Returns the first hit of the next result page, 0 if there aren't any more - * hits available - */ - public long getNextFirstHitNo() { - if (result.getQuery().getHits() > result.getConcreteHitCount()) { - return 0; - } - - return Math.min(getLastHitNo() + 1, result.getTotalHitCount()); - } - - /** - * Returns the first hit of the next result page, 0 if there aren't any more - * hits available - */ - public long getNextLastHitNo() { - if (result.getQuery().getHits() > result.getConcreteHitCount()) { - return 0; - } - - return Math.min(getLastHitNo() + result.getConcreteHitCount(), result.getTotalHitCount()); - } - - /** - * Returns the number of the last result of the current hit page. - */ - public int getLastHitNo() { - return getFirstHitNo() + result.getConcreteHitCount() - 1; - } - - /** - * The first hit presented on the previous result page as an index into the - * global list of all hits generated by the user query - */ - public int getPrevFirstHitNo() { - return Math.max(getFirstHitNo() - result.getQuery().getHits(), 1); - } - - /** - * The last hit presented on the previous result page as an index into the - * global list of all hits generated by the user query - */ - public int getPrevLastHitNo() { - return Math.max(getFirstHitNo() - 1, 0); - } - - /** - * An URL that may be used to obtain the next result page. - */ - public String getNextResultURL() { - HttpRequest request = result.getQuery().getHttpRequest(); - StringBuilder nextURL = new StringBuilder(); - - nextURL.append(getPath(request)).append("?"); - parametersExceptOffset(request, nextURL); - - int offset = getLastHitNo(); - - nextURL.append("&").append("offset=").append(Integer.toString(offset)); - return nextURL.toString(); - } - - /** - * An URL that may be used to obtain the previous result page. - */ - public String getPreviousResultURL() { - HttpRequest request = result.getQuery().getHttpRequest(); - StringBuilder prevURL = new StringBuilder(); - - prevURL.append(getPath(request)).append("?"); - parametersExceptOffset(request, prevURL); - int offset = getPrevFirstHitNo() - 1; - prevURL.append("&").append("offset=").append(Integer.toString(offset)); - return prevURL.toString(); - } - - public String getCurrentResultURL() { - HttpRequest request = result.getQuery().getHttpRequest(); - StringBuilder thisURL = new StringBuilder(); - - thisURL.append(getPath(request)).append("?"); - parameters(request, thisURL); - return thisURL.toString(); - } - - private String getPath(HttpRequest request) { - String path = request.getUri().getPath(); - if (path == null) { - path = ""; - } - return path; - } - - private void parametersExceptOffset(HttpRequest request, StringBuilder nextURL) { - int startLength = nextURL.length(); - for (Map.Entry<String, String> property : request.propertyMap().entrySet()) { - if (property.getKey().equals("offset")) continue; - - if (nextURL.length() > startLength) - nextURL.append("&"); - nextURL.append(property.getKey()).append("=").append(property.getValue()); - } - } - - private void parameters(HttpRequest request, StringBuilder nextURL) { - int startLength = nextURL.length(); - for (Map.Entry<String, String> property : request.propertyMap().entrySet()) { - if (nextURL.length() > startLength) - nextURL.append("&"); - nextURL.append(property.getKey()).append("=").append(property.getValue()); - } - } - - /** - * Returns the templates which will render the result. This is never null. - * If default rendering is used, it is a TemplateSet containing no - * templates. - * - * @deprecated use a renderer instead - */ - @SuppressWarnings("rawtypes") - // TODO: Remove on Vespa 7 - @Deprecated // OK - public UserTemplate getTemplates() { - if (renderer == null) { - return TemplateSet.getDefault(); - } else if (renderer instanceof SearchRendererAdaptor) { - return ((SearchRendererAdaptor) renderer).getAdaptee(); - } else { - throw new RuntimeException( - "Please use getTemplate() instead of getTemplates() when using the new template api."); - } - } - - /** - * Sets the template set which should render this result set - * - * @param templates - * the templates which should render this result, or null to - * use the default xml rendering - */ - @SuppressWarnings("deprecation") - public void setTemplates(@SuppressWarnings("rawtypes") UserTemplate templates) { - if (templates == null) { - setTemplates(TemplateSet.getDefault()); - } else { - setRenderer(new SearchRendererAdaptor(templates)); - } - } - - /** - * @deprecated since 5.1.21, use {@link Presentation#getRenderer()} - */ - public Renderer<Result> getRenderer() { - return renderer; - } - - /** - * @deprecated since 5.1.21, use {@link Presentation#setRenderer(com.yahoo.component.ComponentSpecification)} - */ - public void setRenderer(Renderer<Result> renderer) { - this.renderer = renderer; - } - - /** - * For internal use only. - */ - public boolean usesDefaultTemplate() { - return renderer == null || - (renderer instanceof SearchRendererAdaptor && - ((SearchRendererAdaptor) renderer).getAdaptee().isDefaultTemplateSet()); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java deleted file mode 100644 index a679b17b6fc..00000000000 --- a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.searchchain.model.federation; - -import com.yahoo.container.bundle.BundleInstantiationSpecification; -import net.jcip.annotations.Immutable; - -import com.yahoo.search.federation.http.HTTPProviderSearcher; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * Specifies how a http provider is to be set up. - * - * @author Tony Vaagenes - * @deprecated - */ -@Immutable -@Deprecated // OK -// TODO: Remove on Vespa 7 -public class HttpProviderSpec { - - public enum Type { - vespa(com.yahoo.search.federation.vespa.VespaSearcher.class); - - Type(Class<? extends HTTPProviderSearcher> searcherClass) { - className = searcherClass.getName(); - } - - final String className; - } - - // The default connection parameter values come from the config definition - public static class ConnectionParameters { - public final Double readTimeout; - public final Double connectionTimeout; - public final Double connectionPoolTimeout; - public final Integer retries; - - public ConnectionParameters(Double readTimeout, Double connectionTimeout, - Double connectionPoolTimeout, Integer retries) { - this.readTimeout = readTimeout; - this.connectionTimeout = connectionTimeout; - this.connectionPoolTimeout = connectionPoolTimeout; - this.retries = retries; - } - } - - public static class Node { - public final String host; - public final int port; - - public Node(String host, int port) { - this.host = host; - this.port = port; - } - - @Override - public String toString() { - return "Node{" + - "host='" + host + '\'' + - ", port=" + port + - '}'; - } - } - - public final ConnectionParameters connectionParameters; - - public final Integer cacheSizeMB; - - public final String path; - public final List<Node> nodes; - public final String ycaApplicationId; - public final Integer ycaCertificateTtl; - public final Integer ycaRetryWait; - public final Node ycaProxy; - - public static BundleInstantiationSpecification toBundleInstantiationSpecification(Type type) { - return BundleInstantiationSpecification.getInternalSearcherSpecificationFromStrings(type.className, null); - } - - public static boolean includesType(String typeString) { - for (Type type : Type.values()) { - if (type.name().equals(typeString)) { - return true; - } - } - return false; - } - - public HttpProviderSpec(Double cacheWeight, - String path, - List<Node> nodes, - String ycaApplicationId, - Integer ycaCertificateTtl, - Integer ycaRetryWait, - Node ycaProxy, - Integer cacheSizeMB, - ConnectionParameters connectionParameters) { - - this.path = path; - this.nodes = unmodifiable(nodes); - this.ycaApplicationId = ycaApplicationId; - this.ycaProxy = ycaProxy; - this.ycaCertificateTtl = ycaCertificateTtl; - this.ycaRetryWait = ycaRetryWait; - this.cacheSizeMB = cacheSizeMB; - - this.connectionParameters = connectionParameters; - } - - private List<HttpProviderSpec.Node> unmodifiable(List<HttpProviderSpec.Node> nodes) { - return nodes == null ? - Collections.emptyList() : - Collections.unmodifiableList(new ArrayList<>(nodes)); - } -} diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java index cfbea04a110..5288b28cad1 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java @@ -57,7 +57,6 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { private static final CompoundName streamingUserid=new CompoundName("streaming.userid"); private static final CompoundName streamingGroupname=new CompoundName("streaming.groupname"); private static final CompoundName streamingSelection=new CompoundName("streaming.selection"); - private static final CompoundName streamingHeadersonly=new CompoundName("streaming.headersonly"); private static final CompoundName streamingFromtimestamp=new CompoundName("streaming.fromtimestamp"); private static final CompoundName streamingTotimestamp=new CompoundName("streaming.totimestamp"); private static final CompoundName streamingLoadtype=new CompoundName("streaming.loadtype"); @@ -153,7 +152,6 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { params.setSessionTimeoutMs(query.getTimeout()); params.setVisitorLibrary("searchvisitor"); params.setLocalDataHandler(this); - params.setVisitHeadersOnly(query.properties().getBoolean(streamingHeadersonly)); if (query.properties().getDouble(streamingFromtimestamp) != null) { params.setFromTimestamp(query.properties().getDouble(streamingFromtimestamp).longValue()); } diff --git a/container-search/src/main/javacc/com/yahoo/search/grouping/request/parser/GroupingParser.jj b/container-search/src/main/javacc/com/yahoo/search/grouping/request/parser/GroupingParser.jj index 6a55a32eb8a..e1df35bb24a 100644 --- a/container-search/src/main/javacc/com/yahoo/search/grouping/request/parser/GroupingParser.jj +++ b/container-search/src/main/javacc/com/yahoo/search/grouping/request/parser/GroupingParser.jj @@ -169,7 +169,6 @@ TOKEN : <XOR: "xor"> | <XORBIT: "xorbit"> | <Y: "y"> | - <YMUM: "ymum"> | <ZCURVE: "zcurve"> | <IDENTIFIER: ["A"-"Z","a"-"z"](["A"-"Z","a"-"z","0"-"9","_","@"])*> } @@ -374,7 +373,6 @@ GroupingExpression value(GroupingOperation grp) : exp = ucaFunction(grp) | exp = xorExpression(grp) | exp = xorBitFunction(grp) | - exp = ymumValue() | exp = zcurveFunction(grp) ) | ( exp = attributeValue() ) ) { return exp; } @@ -912,12 +910,6 @@ XorBitFunction xorBitFunction(GroupingOperation grp) : { return new XorBitFunction(exp, num.intValue()); } } -YmumValue ymumValue() : { } -{ - ( <YMUM> lbrace() rbrace() ) - { return new YmumValue(); } -} - void bucket(GroupingOperation grp, BucketResolver resolver) : { ConstantValue from, to = null; @@ -1076,7 +1068,6 @@ String identifier() : <XOR> | <XORBIT> | <Y> | - <YMUM> | <ZCURVE> ) { ret = token.image; } space() ) { return ret; } } diff --git a/container-search/src/main/resources/configdefinitions/legacy-emulation.def b/container-search/src/main/resources/configdefinitions/legacy-emulation.def deleted file mode 100644 index 95469338bb7..00000000000 --- a/container-search/src/main/resources/configdefinitions/legacy-emulation.def +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=container.search - -## Keep emulating old (4.2, 5.0, early 5.1) string type fields for structured data -stringBackedStructuredData bool default=false - -## Keep emulating old (4.2, 5.0, early 5.1) string type fields for summaryfeatures / rankfeatures fields -stringBackedFeatureData bool default=false - -## This will populate even if they are empty/undefined. -forceFillEmptyFields bool default=false diff --git a/container-search/src/main/resources/configdefinitions/qr-binary-cache-region.def b/container-search/src/main/resources/configdefinitions/qr-binary-cache-region.def deleted file mode 100644 index ba2b6ed7802..00000000000 --- a/container-search/src/main/resources/configdefinitions/qr-binary-cache-region.def +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# TODO: Not in use - remove on Vespa 7 - -namespace=search.cache - -# size of the region (in MB). Cache size (in qr-binary-cache.cfg) -# and region size should be same -region_size int default=1024 - -# Number of threads that would concurrently access cache -concurrency_level int default=50 - -# Initial bucket count that would be created in a region -# (rehashing would grow it if necessary but is expensive) -bucket_count int default=12000 - -# Load factor of the region. -load_factor double default=0.75 - -#TTL in milli-seconds -time_to_live int default=-1 - -entry_size_range int default=500 - - -#Whether lazy invalidation is enabled or no -lazy_invalidation_enabled bool default=false - -# Time interval (in seconds) over which lazy invalidation parameters would be observed -tick_interval int default=1 -tick_count_to_average int default=10 - -# Maximum QPS that can be supported by the back-end. -# Refresh due to lazy invalidation and TTL would be disabled in the -# tick_interval in which total requests to backend have exceeded MAX_QPS -max_backend_qps int default=300 - -# Log file in which lazy invalidation stats would be -# recorded (keep it empty if no stats are required) -lazy_invalidation_stats_file string default="lstats.log" - -# Maximum stale time in seconds after a call to lazy invalidation is made. -# Once this time interval is expired, any request for an expired -# entry (due to TTL or lazy invalidation) would result in a cache miss. -lazy_invalidation_max_stale_time_in_sec int default=1000 diff --git a/container-search/src/main/resources/configdefinitions/qr-binary-cache.def b/container-search/src/main/resources/configdefinitions/qr-binary-cache.def deleted file mode 100644 index 917832e86fe..00000000000 --- a/container-search/src/main/resources/configdefinitions/qr-binary-cache.def +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# TODO: Not in use - remove on Vespa 7 - -namespace=search.cache - -# Size of a block in cache. A block is the smallest unit that would -# be allocated to a cache entry (key+value). Multiple blocks can be -# allocated to an entry if required and blocks can not be shared -# between multiple entries. IN BYTES -block_size int default=256 - -# Size of a buffer array that would hold the serialized content, -# maximum size here can be 2 GB. IN MB -buffer_size int default=64 - -# For debugging (for production it should be true) -assertions_disabled bool default=false - -# Whether to create the byte buffers in the NIO memory or in -# JVM heap itself (having these in NIO buffers gives lower GC pause times) -use_direct_buffers bool default=true - -# Size of the byte array pool that the cache uses to hold up -# the serialized content temporarily -pooled_byte_arrays int default=1000 - -# The size of the cahe in MB -# If the size is 0, the cache is disabled -cache_size int default=0 - -# If true, cache would write average time taken in various -# operations per 1000 requests in log -profiling_enabled bool default=false - -# For debugging purposes -lock_verification_enabled bool default=false diff --git a/container-search/src/main/resources/configdefinitions/qr-start.def b/container-search/src/main/resources/configdefinitions/qr-start.def index d29ebc8a826..8547ad08458 100644 --- a/container-search/src/main/resources/configdefinitions/qr-start.def +++ b/container-search/src/main/resources/configdefinitions/qr-start.def @@ -9,7 +9,7 @@ jvm.server bool default=true restart jvm.verbosegc bool default=true restart ## Garbage Collection tuning parameters -jvm.gcopts string default="-XX:+UseConcMarkSweepGC -XX:MaxTenuringThreshold=15 -XX:NewRatio=1" restart +jvm.gcopts string default="-XX:+UseG1GC -XX:MaxTenuringThreshold=15 -XX:NewRatio=1" restart ## Heap size (in megabytes) for the Java VM jvm.heapsize int default=1536 restart @@ -27,6 +27,10 @@ jvm.directMemorySizeCache int default=0 restart ## value above. Setting outside [1, 99] disables this setting. jvm.heapSizeAsPercentageOfPhysicalMemory int default=0 restart +## Number of processors available, can be used to set -XX:ActiveProcessorCount if set to non-zero. +## In that case will be the number returned by the JVM when calling Runtime.getRuntime().availableProcessors() +jvm.availableProcessors int default=0 restart + ## Extra environment variables qrs.env string default="YELL_MA_EURO=INXIGHT" restart diff --git a/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java index 9b1517454a9..54512e727c2 100644 --- a/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java +++ b/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java @@ -50,7 +50,7 @@ public class GetDocSumsPacketTestCase { @Test public void requireThatSessionIdIsEncodedAsPropertyWhenUsingSearchSession() throws BufferTooSmallException { Result result = new Result(new Query("?query=foo")); - SessionId sessionId = result.getQuery().getSessionId(true); // create session id. + SessionId sessionId = result.getQuery().getSessionId("node-0"); result.getQuery().getRanking().setQueryCache(true); FastHit hit = new FastHit(); result.hits().add(hit); diff --git a/container-search/src/test/java/com/yahoo/fs4/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/QueryTestCase.java index 78b2f65305a..e2ce26cb908 100644 --- a/container-search/src/test/java/com/yahoo/fs4/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/fs4/test/QueryTestCase.java @@ -12,6 +12,7 @@ import com.yahoo.prelude.query.PhraseSegmentItem; import com.yahoo.prelude.query.WeightedSetItem; import com.yahoo.prelude.query.WordItem; import com.yahoo.search.Query; +import com.yahoo.search.query.ranking.SoftTimeout; import org.junit.Test; import java.nio.ByteBuffer; @@ -52,6 +53,8 @@ public class QueryTestCase { @Test public void testEncodeQueryPacketWithSomeAdditionalFeatures() { Query query = new Query("/?query=chain&dataset=10&type=phrase&timeout=0"); + query.properties().set(SoftTimeout.enableProperty, false); + // Because the rank mapping now needs config and a searcher, // we do the sledgehammer dance: query.getRanking().setProfile("two"); @@ -79,7 +82,7 @@ public class QueryTestCase { "&ranking.features.query(foo)=30.3&ranking.features.query(bar)=0" + "&ranking.properties.property.p1=v1&ranking.properties.property.p2=v2" + "&pos.ll=S22.4532;W123.9887&pos.radius=3&pos.attribute=place&ranking.freshness=37" + - "&model.searchPath=7/3"); + "&model.searchPath=7/3&ranking.softtimeout.enable=false"); query.getRanking().setFreshness(new Freshness("123456")); query.getRanking().setSorting("+field1 -field2"); query.getRanking().setProfile("two"); @@ -105,7 +108,7 @@ public class QueryTestCase { "&ranking.features.query(foo)=30.3&ranking.features.query(bar)=0" + "&ranking.properties.property.p1=v1&ranking.properties.property.p2=v2" + "&pos.ll=S22.4532;W123.9887&pos.radius=3&pos.attribute=place&ranking.freshness=37" + - "&model.searchPath=7/3"); + "&model.searchPath=7/3&ranking.softtimeout.enable=false"); query.getRanking().setFreshness("123456"); query.getRanking().setSorting("+field1 -field2"); query.getRanking().setProfile("two"); @@ -127,6 +130,7 @@ public class QueryTestCase { @Test public void testEncodeQueryPacketWithLabelsConnectivityAndSignificance() { Query query = new Query(); + query.properties().set(SoftTimeout.enableProperty, false); AndItem and = new AndItem(); WeightedSetItem taggable1 = new WeightedSetItem("field1"); taggable1.setLabel("foo"); diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index 646c466eba9..25d6a6e9777 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -6,7 +6,6 @@ import com.yahoo.component.ComponentId; import com.yahoo.container.QrConfig; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.search.Fs4Config; -import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.fs4.QueryPacket; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; @@ -540,7 +539,6 @@ public class ClusterSearcherTestCase { qrSearchersConfig.build(), clusterConfig.build(), documentDbConfig.build(), - new LegacyEmulationConfig.Builder().build(), new QrMonitorConfig.Builder().build(), new DispatchConfig.Builder().build(), createClusterInfoConfig(), @@ -555,7 +553,7 @@ public class ClusterSearcherTestCase { clusterInfoConfigBuilder.nodeCount(1); return new ClusterInfoConfig(clusterInfoConfigBuilder); } - + private static class QueryTimeoutFixture { ClusterSearcher searcher; Execution exec; diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java index 81f16f7f261..ad4d0cf82e5 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.prelude.fastsearch; import com.google.common.collect.ImmutableSet; import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.prelude.hitfield.RawData; import com.yahoo.prelude.hitfield.XMLString; import com.yahoo.prelude.hitfield.JSONString; @@ -66,33 +65,6 @@ public class SlimeSummaryTestCase { } @Test - public void testDecodingEmptyWithLegacyEmulation() { - LegacyEmulationConfig emulationConfig = new LegacyEmulationConfig(new LegacyEmulationConfig.Builder().forceFillEmptyFields(true)); - DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf, emulationConfig); - FastHit hit = new FastHit(); - assertNull(docsum.lazyDecode("default", emptySummary(), hit)); - assertEquals(NanNumber.NaN, hit.getField("integer_field")); - assertEquals(NanNumber.NaN, hit.getField("short_field")); - assertEquals(NanNumber.NaN, hit.getField("byte_field")); - assertEquals(NanNumber.NaN, hit.getField("float_field")); - assertEquals(NanNumber.NaN, hit.getField("double_field")); - assertEquals(NanNumber.NaN, hit.getField("int64_field")); - assertEquals("", hit.getField("string_field")); - assertEquals(RawData.class, hit.getField("data_field").getClass()); - assertEquals("", hit.getField("data_field").toString()); - assertEquals("", hit.getField("longstring_field")); - assertEquals(RawData.class, hit.getField("longdata_field").getClass()); - assertEquals("", hit.getField("longdata_field").toString()); - assertEquals(XMLString.class, hit.getField("xmlstring_field").getClass()); - assertEquals("", hit.getField("xmlstring_field").toString()); - // assertEquals(hit.getField("jsonstring_field"), instanceOf(JSONString.class)); - assertEquals("", hit.getField("jsonstring_field").toString()); - // Empty tensors are represented by null because we don't have type information here to create the right empty tensor - assertNull(hit.getField("tensor_field1")); - assertNull(hit.getField("tensor_field2")); - } - - @Test public void testTimeout() { DocsumDefinitionSet docsum = createDocsumDefinitionSet(summary_cf); FastHit hit = new FastHit(); @@ -424,11 +396,6 @@ public class SlimeSummaryTestCase { return new DocsumDefinitionSet(config.documentdb(0)); } - private DocsumDefinitionSet createDocsumDefinitionSet(String configID, LegacyEmulationConfig legacyEmulationConfig) { - DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID); - return new DocsumDefinitionSet(config.documentdb(0), legacyEmulationConfig); - } - private static class Utf8FieldTraverser implements Hit.RawUtf8Consumer { private Map<String, Object> traversed; diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index c4d32111124..fd01fb9a740 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -153,7 +153,7 @@ public class FastSearcherTestCase { } { // direct.summaries due to query cache - String query = "?query=sddocname:a&ranking.queryCache"; + String query = "?query=sddocname:a&ranking.queryCache&timeout=5000ms"; Result result = doSearch(fastSearcher, new Query(query), 0, 10); doFill(fastSearcher, result); ErrorMessage error = result.hits().getError(); @@ -162,7 +162,7 @@ public class FastSearcherTestCase { } { // direct.summaries due to no summary features - String query = "?query=sddocname:a&dispatch.summaries&summary=simple&ranking=simpler"; + String query = "?query=sddocname:a&dispatch.summaries&summary=simple&ranking=simpler&timeout=5000ms"; Result result = doSearch(fastSearcher, new Query(query), 0, 10); doFill(fastSearcher, result); ErrorMessage error = result.hits().getError(); @@ -185,6 +185,7 @@ public class FastSearcherTestCase { documentdbConfigWithOneDb); Query query = new Query("?query=foo&model.restrict=testDb"); + query.getRanking().getSoftTimeout().setEnable(false); query.prepare(); doSearch(fastSearcher, query, 0, 10); @@ -330,7 +331,7 @@ public class FastSearcherTestCase { byte[] actual = new byte[buf.remaining()]; buf.get(actual); - SessionId sessionId = query.getSessionId(false); + SessionId sessionId = query.getSessionId(); byte IGNORE = 69; ByteBuffer answer = ByteBuffer.allocate(1024); answer.put(new byte[] { 0, 0, 0, (byte)(141+sessionId.asUtf8String().getByteLength()), 0, 0, 0, -37, 0, 0, 16, 17, 0, 0, 0, 0, diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/TestLinguistics.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/TestLinguistics.java index db5397e5292..b697841a969 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/TestLinguistics.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/TestLinguistics.java @@ -60,12 +60,6 @@ public class TestLinguistics implements Linguistics { return linguistics.getCharacterClasses(); } - @Override - @Deprecated - public Tuple2<String, Version> getVersion(Linguistics.Component component) { - return linguistics.getVersion(component); - } - } diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/indexcombinator.cfg b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/indexcombinator.cfg deleted file mode 100644 index a8790cc4051..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/indexcombinator.cfg +++ /dev/null @@ -1,29 +0,0 @@ -indexinfo[1] -indexinfo[0].name combinedattributeandindexsearch -indexinfo[0].command[12] -indexinfo[0].command[0].indexname sddocname -indexinfo[0].command[0].command index -indexinfo[0].command[1].indexname index1 -indexinfo[0].command[1].command index -indexinfo[0].command[2].indexname default -indexinfo[0].command[2].command index -indexinfo[0].command[3].indexname index1 -indexinfo[0].command[3].command stem -indexinfo[0].command[4].indexname default -indexinfo[0].command[4].command stem -indexinfo[0].command[5].indexname index1 -indexinfo[0].command[5].command index1 -indexinfo[0].command[5].command normalize -indexinfo[0].command[6].indexname default -indexinfo[0].command[6].command normalize -indexinfo[0].command[7].indexname attribute1 -indexinfo[0].command[7].command index -indexinfo[0].command[8].indexname attribute1 -indexinfo[0].command[8].command attribute -indexinfo[0].command[9].indexname attribute2 -indexinfo[0].command[9].command attribute -indexinfo[0].command[10].indexname default -indexinfo[0].command[10].command "match-group default attribute1 attribute2" -indexinfo[0].command[11].indexname ranklog -indexinfo[0].command[11].command xmlstring - diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java index fae869c5235..47009eb703b 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java @@ -51,20 +51,20 @@ public class BlendingSearcherTestCase { private final Map<String, Searcher> searchers = new HashMap<>(); private SearchChainRegistry chainRegistry; - private final String blendingDocumentId; + private final String blendingField; public BlendingSearcherWrapper() { - blendingDocumentId = null; + blendingField = null; } - public BlendingSearcherWrapper(String blendingDocumentId) { - this.blendingDocumentId = blendingDocumentId; + public BlendingSearcherWrapper(String blendingField) { + this.blendingField = blendingField; } @SuppressWarnings("serial") public BlendingSearcherWrapper(QrSearchersConfig cfg) { QrSearchersConfig.Com.Yahoo.Prelude.Searcher.BlendingSearcher s = cfg.com().yahoo().prelude().searcher().BlendingSearcher(); - blendingDocumentId = s.docid().length() > 0 ? s.docid() : null; + blendingField = s.docid().length() > 0 ? s.docid() : null; } public boolean addChained(Searcher searcher, String sourceName) { @@ -109,7 +109,7 @@ public class BlendingSearcherTestCase { FederationSearcher fedSearcher = new FederationSearcher(new FederationConfig(builder), contracts, new ComponentRegistry<>()); - BlendingSearcher blendingSearcher = new BlendingSearcher(blendingDocumentId); + BlendingSearcher blendingSearcher = new BlendingSearcher(blendingField); blendingChain = new SearchChain(ComponentId.createAnonymousComponentId("blendingChain"), blendingSearcher, fedSearcher); return true; } @@ -217,7 +217,7 @@ public class BlendingSearcherTestCase { r2.setTotalHitCount(1); chain2.addResult(q, r2); - BlendingSearcherWrapper blender = new BlendingSearcherWrapper("uri"); + BlendingSearcherWrapper blender = new BlendingSearcherWrapper("[id]"); blender.addChained(new FillSearcher(chain1), "a"); blender.addChained(new FillSearcher(chain2), "b"); blender.initialize(); diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java index bf86ca4544b..4875121a501 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java @@ -336,11 +336,6 @@ public class FieldCollapsingSearcherTestCase { assertHit("http://acme.org/h.html", 6,1,r.hits().get(1)); } - // This test depends on DocumentSourceSearcher filling the hits - // with whatever data it got, ignoring actual summary arguments - // in the fill call, then saying the hits are filled for the - // ignored argument. Rewrite to contain different summaries if - // DocumentSourceSearcher gets extended. @Test public void testFieldCollapsingTwoPhaseSelectSummary() { // Set up diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java index 9f4a12d24e6..24af91cb5c0 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java @@ -47,7 +47,7 @@ public class JuniperSearcherTestCase { DocumentSourceSearcher docsource = new DocumentSourceSearcher(); addResult(new Query("?query=12"), sdName, content, docsource); addResult(new Query("?query=12&bolding=false"), sdName, content, docsource); - return new Chain<Searcher>(searcher, docsource); + return new Chain<>(searcher, docsource); } private void addResult(Query query, String sdName, String content, DocumentSourceSearcher docsource) { diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuerySnapshotSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuerySnapshotSearcherTestCase.java deleted file mode 100644 index 7b92e2c1a7c..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuerySnapshotSearcherTestCase.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.searcher.test; - -import com.google.common.util.concurrent.MoreExecutors; -import com.yahoo.component.chain.Chain; -import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.rendering.RendererRegistry; -import com.yahoo.search.result.Hit; -import com.yahoo.prelude.searcher.QuerySnapshotSearcher; -import com.yahoo.search.searchchain.Execution; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertEquals; - -/** - * @author bratseth - */ -public class QuerySnapshotSearcherTestCase { - - @Test - public void test() { - Searcher searcher=new QuerySnapshotSearcher(); - Result result = doSearch(searcher, new Query(), 0,10); - Hit hit=result.hits().get(0); - assertEquals(String.valueOf(Double.POSITIVE_INFINITY), - hit.getRelevance().toString()); - } - - private Result doSearch(Searcher searcher, Query query, int offset, int hits) { - query.setOffset(offset); - query.setHits(hits); - return createExecution(searcher).search(query); - } - - private Execution createExecution(Searcher searcher) { - Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(MoreExecutors.directExecutor()), new SimpleLinguistics()); - return new Execution(chainedAsSearchChain(searcher), context); - } - - private Chain<Searcher> chainedAsSearchChain(Searcher topOfChain) { - List<Searcher> searchers = new ArrayList<>(); - searchers.add(topOfChain); - return new Chain<>(searchers); - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QueryValidatingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QueryValidatingSearcherTestCase.java deleted file mode 100644 index 3967ac63e36..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QueryValidatingSearcherTestCase.java +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.searcher.test; - -import com.google.common.util.concurrent.MoreExecutors; -import com.yahoo.component.chain.Chain; -import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.search.rendering.RendererRegistry; -import com.yahoo.search.result.Hit; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; -import com.yahoo.prelude.searcher.QueryValidatingSearcher; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -/** - * Tests correct denial of query. - * - * @author Steinar Knutsen - */ -public class QueryValidatingSearcherTestCase { - - @Test - public void testBasic() { - // Setup - Map<Searcher, Searcher> chained = new HashMap<>(); - Query query = new Query("?query=test"); - - Result result = new Result(query); - result.hits().add(new Hit("ymail://1111111111/AQAAAP7JgwEAj6XjQQAAAO/+ggA=",950)); - - Searcher validator = new QueryValidatingSearcher(); - DocumentSourceSearcher source = new DocumentSourceSearcher(); - chained.put(validator, source); - source.addResult(query, result); - - // Exercise - Result returnedResult = doSearch(validator, query, 0, 10, chained); - - // Validate - assertEquals(1, returnedResult.getHitCount()); - assertNull(returnedResult.hits().getError()); - - returnedResult = doSearch(validator, query, 0, 1001, chained); - assertEquals(0, returnedResult.getConcreteHitCount()); - assertEquals(4, returnedResult.hits().getError().getCode()); - - returnedResult = doSearch(validator, query, 1001, 10, chained); - assertEquals(0, returnedResult.getConcreteHitCount()); - assertEquals(4, returnedResult.hits().getError().getCode()); - } - - private Result doSearch(Searcher searcher, Query query, int offset, int hits, Map<Searcher, Searcher> chained) { - query.setOffset(offset); - query.setHits(hits); - return createExecution(searcher, chained).search(query); - } - - private Chain<Searcher> chainedAsSearchChain(Searcher topOfChain, Map<Searcher, Searcher> chained) { - List<Searcher> searchers = new ArrayList<>(); - for (Searcher current = topOfChain; current != null; current = chained.get(current)) { - searchers.add(current); - } - return new Chain<>(searchers); - } - - private Execution createExecution(Searcher searcher, Map<Searcher, Searcher> chained) { - Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(MoreExecutors.directExecutor()), new SimpleLinguistics()); - return new Execution(chainedAsSearchChain(searcher, chained), context); - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/BoomTemplate.java b/container-search/src/test/java/com/yahoo/prelude/templates/test/BoomTemplate.java deleted file mode 100644 index 0bbc5151b40..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/BoomTemplate.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates.test; - -import java.io.IOException; -import java.io.Writer; - -import com.yahoo.prelude.templates.Context; -import com.yahoo.prelude.templates.UserTemplate; - -/** - * Test template which throws a runtime exception in its footer. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -@SuppressWarnings("rawtypes") -public class BoomTemplate extends UserTemplate { - public BoomTemplate(String name, String mimeType, String encoding) { - super(name, mimeType, encoding); - } - - @Override - public void error(Context context, Writer writer) throws IOException { - // NOP - } - - @Override - public void footer(Context context, Writer writer) throws IOException { - throw new RuntimeException("Boom!"); - } - - @Override - public void header(Context context, Writer writer) throws IOException { - writer.write("header"); - } - - @Override - public void hit(Context context, Writer writer) throws IOException { - // NOP - } - - @Override - public void hitFooter(Context context, Writer writer) throws IOException { - // NOP - } - - @Override - public void noHits(Context context, Writer writer) throws IOException { - // NOP - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/GroupedResultTestCase.java b/container-search/src/test/java/com/yahoo/prelude/templates/test/GroupedResultTestCase.java deleted file mode 100644 index 07d5dcc9e2d..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/GroupedResultTestCase.java +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates.test; - -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * Tests composition of grouped results using the HitGroup class - * - * @author bratseth - */ -public class GroupedResultTestCase { - - @Test - public void testGroupedResult() { - Result result=new Result(new Query("?query=foo")); - HitGroup hitGroup1=new HitGroup("group1",300); - hitGroup1.add(new Hit("group1.1",200)); - HitGroup hitGroup2=new HitGroup("group2",600); - Hit topLevelHit1=new Hit("toplevel.1",500); - Hit topLevelHit2=new Hit("toplevel.2",700); - result.hits().add(hitGroup1); - result.hits().add(topLevelHit1); - result.hits().add(hitGroup2); - result.hits().add(topLevelHit2); - hitGroup1.add(new Hit("group1.2",800)); - hitGroup2.add(new Hit("group2.1",800)); - hitGroup2.add(new Hit("group2.2",300)); - hitGroup2.add(new Hit("group2.3",500)); - - // Should have 7 concrete hits, ordered as - // toplevel.2 - // group2 - // group2.1 - // group2.3 - // group2.2 - // toplevel.1 - // group1 - // group1.2 - // group1.1 - // Assert this: - - assertEquals(7,result.getConcreteHitCount()); - assertEquals(4,result.getHitCount()); - - Hit topLevel2=result.hits().get(0); - assertEquals("toplevel.2",topLevel2.getId().stringValue()); - - HitGroup returnedGroup2=(HitGroup)result.hits().get(1); - assertEquals(3,returnedGroup2.getConcreteSize()); - assertEquals(3,returnedGroup2.size()); - assertEquals("group2.1",returnedGroup2.get(0).getId().stringValue()); - assertEquals("group2.3",returnedGroup2.get(1).getId().stringValue()); - assertEquals("group2.2",returnedGroup2.get(2).getId().stringValue()); - - Hit topLevel1=result.hits().get(2); - assertEquals("toplevel.1",topLevel1.getId().stringValue()); - - HitGroup returnedGroup1=(HitGroup)result.hits().get(3); - assertEquals(2,returnedGroup1.getConcreteSize()); - assertEquals(2,returnedGroup1.size()); - assertEquals("group1.2",returnedGroup1.get(0).getId().stringValue()); - assertEquals("group1.1",returnedGroup1.get(1).getId().stringValue()); - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/HitContextTestCase.java b/container-search/src/test/java/com/yahoo/prelude/templates/test/HitContextTestCase.java deleted file mode 100644 index 6ff8c2f9d6c..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/HitContextTestCase.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates.test; - -import static org.junit.Assert.assertEquals; - -import java.lang.reflect.Method; -import java.util.List; - -import org.junit.Test; - -import com.yahoo.prelude.templates.HitContext; -import com.yahoo.protect.ClassValidator; - -/** - * Check the entire Context class is correctly masked. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class HitContextTestCase { - - @Test - public void checkMethods() { - List<Method> unmasked = ClassValidator.unmaskedMethodsFromSuperclass(HitContext.class); - assertEquals("Unmasked methods from superclass: " + unmasked, 0, unmasked.size()); - } -} diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/TemplateTestCase.java b/container-search/src/test/java/com/yahoo/prelude/templates/test/TemplateTestCase.java deleted file mode 100644 index 6f49ebbdee0..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/TemplateTestCase.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates.test; - -import java.io.ByteArrayOutputStream; -import java.nio.charset.Charset; -import java.nio.charset.CharsetEncoder; - -import com.yahoo.io.ByteWriter; -import com.yahoo.prelude.templates.UserTemplate; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author Steinar Knutsen - */ -public class TemplateTestCase { - - private CharsetEncoder encoder; - private ByteArrayOutputStream stream; - - public TemplateTestCase () { - Charset cs = Charset.forName("UTF-8"); - encoder = cs.newEncoder(); - stream = new ByteArrayOutputStream(); - } - - @Test - public void testASCIIQuoting() throws java.io.IOException { - stream.reset(); - byte[] c = new byte[] { 97, 98, 99, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 }; - ByteWriter bw = new ByteWriter(stream, encoder); - UserTemplate.dumpAndXMLQuoteUTF8(bw, c); - bw.close(); - String res = stream.toString("UTF-8"); - String correct = "abc\\u0000\\u0001\\u0002\\u0003\\u0004\\u0005\\u0006\\u0007\\u0008\t\n\\u000B\\u000C\r\\u000E\\u000F\\u0010\\u0011"; - assertEquals(correct, res); - - } - - @Test - public void testXMLQuoting() throws java.io.IOException { - stream.reset(); - // c = <s>> - byte[] c = new byte[] { 60, 115, 62, 38, 103, 116, 59 }; - ByteWriter bw = new ByteWriter(stream, encoder); - UserTemplate.dumpAndXMLQuoteUTF8(bw, c); - bw.close(); - String res = stream.toString("UTF-8"); - String correct = "<s>&gt;"; - assertEquals(correct, res); - - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/TestTemplate.java b/container-search/src/test/java/com/yahoo/prelude/templates/test/TestTemplate.java deleted file mode 100644 index 0f5e126dae8..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/TestTemplate.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates.test; - -import java.io.IOException; -import java.io.Writer; - -import com.yahoo.prelude.templates.Context; -import com.yahoo.prelude.templates.UserTemplate; - -/** - * Test basic UserTemplate functionality of detecting - * overridden group rendering methods. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -@SuppressWarnings("rawtypes") -public class TestTemplate extends UserTemplate { - - public TestTemplate(String name, String mimeType, String encoding) { - super(name, mimeType, encoding); - } - - @Override - public void error(Context context, Writer writer) throws IOException { - // NOP - } - - @Override - public void footer(Context context, Writer writer) throws IOException { - // NOP - } - - @Override - public void header(Context context, Writer writer) throws IOException { - // NOP - } - - @Override - public void hit(Context context, Writer writer) throws IOException { - // NOP - } - - @Override - public void hitFooter(Context context, Writer writer) throws IOException { - // NOP - } - - @Override - public void noHits(Context context, Writer writer) throws IOException { - // NOP - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/TilingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/templates/test/TilingTestCase.java deleted file mode 100644 index f86d76a12a4..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/TilingTestCase.java +++ /dev/null @@ -1,309 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates.test; - -import com.google.common.util.concurrent.MoreExecutors; -import com.yahoo.component.chain.Chain; -import com.yahoo.io.IOUtils; -import com.yahoo.language.Linguistics; -import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.prelude.hitfield.XMLString; -import com.yahoo.prelude.templates.SearchRendererAdaptor; -import com.yahoo.prelude.templates.TiledTemplateSet; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.federation.http.HTTPProviderSearcher; -import com.yahoo.search.rendering.RendererRegistry; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import com.yahoo.search.searchchain.Execution; -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.nio.charset.CharsetDecoder; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Tests representing a federated and grouped result as a Result object and - * rendering a tiled output of the result - * - * @author bratseth - */ -@SuppressWarnings("deprecation") -public class TilingTestCase { - - /** - * This result contains two blocks (center and right). - * The center block contains multiple subblocks while the right one contains a single block of ads. - */ - @Test - public void testTiling() throws IOException { - Chain<Searcher> chain=new Chain<>("tiling", new TiledResultProducer()); - - // Query it - Query query = new Query("/tiled?query=foo"); - Result result = callSearchAndSetRenderer(chain, query); - assertRendered(IOUtils.readFile(new File("src/test/java/com/yahoo/prelude/templates/test/tilingexample.xml")),result); - } - - /** - * This result contains center section and meta blocks. - */ - @Test - public void testTiling2() throws IOException { - Chain<Searcher> chain= new Chain<>("tiling", new TiledResultProducer2()); - - // Query it - Query query=new Query("/tiled?query=foo"); - Result result= callSearchAndSetRenderer(chain, query); - assertRendered(IOUtils.readFile(new File("src/test/java/com/yahoo/prelude/templates/test/tilingexample2.xml")),result); - } - - private Result callSearchAndSetRenderer(Chain<Searcher> chain, Query query) { - Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(MoreExecutors.directExecutor()), new SimpleLinguistics()); - Result result = new Execution(chain, context).search(query); - result.getTemplating().setRenderer(new SearchRendererAdaptor(new TiledTemplateSet())); - return result; - } - - public static void assertRenderedStartsWith(String expected,Result result) throws IOException { - assertRendered(expected,result,false); - } - - public static void assertRendered(String expected,Result result) throws IOException { - assertRendered(expected,result,true); - } - - public static void assertRendered(String expected, Result result,boolean checkFullEquality) throws IOException { - if (checkFullEquality) - assertEquals(filterComments(expected), getRendered(result)); - else - assertTrue(getRendered(result).startsWith(expected)); - } - - private static String filterComments(String s) { - StringBuilder b = new StringBuilder(); - for (String line : s.split("\\\n")) - if ( ! line.startsWith("<!--")) - b.append(line).append("\n"); - return b.toString(); - } - - public static String getRendered(Result result) throws IOException { - if (result.getTemplating().getRenderer() == null) - result.getTemplating().setTemplates(null); - - // Done in a roundabout way to simulate production code path - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - Charset cs = Charset.forName(result.getTemplating().getRenderer().getEncoding()); - CharsetDecoder decoder = cs.newDecoder(); - SearchRendererAdaptor.callRender(stream, result); - stream.flush(); - return decoder.decode(ByteBuffer.wrap(stream.toByteArray())).toString(); - } - - private static class TiledResultProducer extends Searcher { - - @Override - public Result search(Query query, Execution execution) { - Result result = new Result(query); - result.setTotalHitCount(2800000000l); - - // Blocks - HitGroup centerBlock=(HitGroup)result.hits().add(new HitGroup("section:center")); - centerBlock.types().add("section"); - centerBlock.setField("region","center"); - - HitGroup rightBlock=(HitGroup)result.hits().add(new HitGroup("section:right")); - rightBlock.types().add("section"); - rightBlock.setField("region","right"); - - // Center groups - HitGroup navigation=(HitGroup)centerBlock.add(new HitGroup("group:navigation",1.0)); - navigation.types().add("group"); - navigation.setField("type","navigation"); - - HitGroup adsNorth=(HitGroup)centerBlock.add(new HitGroup("group:ads:north",0.9)); - adsNorth.types().add("group"); - adsNorth.setField("type","ads"); - - HitGroup hits=(HitGroup)centerBlock.add(new HitGroup("group:navigation",0.8)); - hits.types().add("group"); - hits.setField("type","hits"); - - HitGroup adsSouth=(HitGroup)centerBlock.add(new HitGroup("group:ads:south",0.7)); - adsSouth.types().add("group"); - adsSouth.setField("type","ads"); - - // Right group - HitGroup adsRight=(HitGroup)rightBlock.add(new HitGroup("group:ads:right",0.7)); - adsRight.types().add("group"); - adsRight.setField("type","ads"); - - // Navigation content - /* - Hit alsoTry=navigation.add(new Hit("alsotry")); - alsoTry.types().add("alsotry"); - alsoTry.setMeta(true); - LinkList links=new LinkList(); - links.add(new Link("Hilton","?search=Hilton hotel")); - links.add(new Link("Habbo hotel","?search=Habbo hotel")); - links.add(new Link("Marriott","?search=Marriott hotel")); - alsoTry.setField("links",links); - */ - - // North ads content - Hit ad1=adsNorth.add(new Hit("http://www.hotels.com",0.7)); - ad1.types().add("ad"); - ad1.setAuxiliary(true); - ad1.setField("title",new XMLString("Cheap <hi>hotels</hi>")); - ad1.setField("body",new XMLString("Low Rates Guaranteed. Call a <hi>Hotel</hi> Expert.")); - - Hit ad2=adsNorth.add(new Hit("http://www.expedia.com",0.6)); - ad2.types().add("ad"); - ad2.setAuxiliary(true); - ad2.setField("title",new XMLString("Cheap <hi>hotels</hi> at Expedia")); - ad2.setField("body","Expedia Special Rates Means We Guarantee Our Low Rates on Rooms."); - -// // Hits content -// // - news hit -// HitGroup news1=(HitGroup)hits.add(new HitGroup("newsarticles",0.9)); -// news1.setMeta(false); -// news1.types().add("news"); -// news1.setField("title","Hotel - News results"); -// Hit article1=news1.add(new Hit("www.miamiherald.com/?article=jhsgd7323",0.5)); -// article1.setAuxiliary(true); -// article1.setField("title","Celebrity blackout: The Hilton of Paris changes name to regain search traffic"); -// article1.types().add("newsarticle"); -// article1.setField("age",23); -// article1.setField("source","Miami Herald"); -// Hit article2=news1.add(new Hit("www.sfgate.com/?article=8763khj7",0.4)); -// article2.setAuxiliary(true); -// article2.setField("title","Hotels - more expensive than staying at home"); -// article2.types().add("newsarticle"); -// article2.setField("age",3500); -// article2.setField("source","SF Gate"); - - // - collapsed hit - Hit hit1=hits.add(new Hit("www.hotels.com",0.8)); - hit1.types().add("hit.collapsed"); - hit1.setField("title","Hotels.com | Cheap Hotels | Discount Hotel Rooms | Motels | Lodging"); - hit1.setField("body",new XMLString("Hotels.com helps you find great rates on hotels and discount <hi>hotel</hi> packages.")); - /* - LinkList collapsed1=new LinkList(); - collapsed1.add(new Link("Last Minute Deals","www.hotels.com/lastminutedeals")); - collapsed1.add(new Link("Hotel Savings","www.hotels.com/deals")); - collapsed1.add(new Link("800-994-6835","www.hotels.com/?PSRC=OT2")); - hit1.setField("links",collapsed1); - */ - - // regular hit with links - Hit hit2=hits.add(new Hit("www.indigohotels.com",0.7)); - hit2.types().add("hit"); - hit2.setField("title","Hotel Indigo Hotels United States - Official Web Site"); - hit2.setField("body","Make Hotel Indigo online hotel reservations and book your hotel rooms today."); - /* - LinkList collapsed2=new LinkList(); - collapsed2.add(new Link("800-333-6835","www.indigohotels.com/order")); - collapsed2.add(new Link("Reservations","www.indigohotels.com/reservations")); - hit2.setField("links",collapsed2); - */ - - // boring old hit - Hit hit3=hits.add(new Hit("www.all-hotels.com",0.6)); - hit3.types().add("hit"); - hit3.setField("title","All hotels"); - hit3.setField("body","Online hotel directory and reservations."); - - // South ads - Hit southAd1=adsSouth.add(new Hit("www.daysinn.com",1.0)); - southAd1.types().add("ad"); - southAd1.setAuxiliary(true); - southAd1.setField("title","Days Inn Special Deal"); - southAd1.setField("body","Buy now and Save 15% Off Our Best Available Rate with Days Inn."); - Hit southAd2=adsSouth.add(new Hit("http://www.expedia.com",0.9)); - southAd2.types().add("ad"); - southAd2.setAuxiliary(true); - southAd2.setField("title",new XMLString("Cheap <hi>hotels</hi> at Expedia")); - southAd2.setField("body","Expedia Special Rates Means We Guarantee Our Low Rates on Rooms."); - - // Right ads - Hit rightAd1=adsRight.add(new Hit("www.daysinn.com",1.0)); - rightAd1.types().add("ad"); - rightAd1.setAuxiliary(true); - rightAd1.setField("title","Days Inn Special Deal"); - rightAd1.setField("body","Buy now and Save 15% Off Our Best Available Rate with Days Inn."); - Hit rightAd2=adsRight.add(new Hit("www.holidayinn.com",0.9)); - rightAd2.types().add("ad"); - rightAd2.setAuxiliary(true); - rightAd2.setField("title","Holiday Inn: Official Site"); - rightAd2.setField("body","Book with Holiday Inn. Free Internet. Kids eat free."); - - // Done creating result - must analyze because we add ads then later set them as auxiliary - result.analyzeHits(); - - return result; - } - - } - - private static class TiledResultProducer2 extends Searcher { - - @Override - public Result search(Query query,Execution execution) { - Result result=new Result(query); - result.setTotalHitCount(1); - - HitGroup section = new HitGroup("section:center"); - result.hits().add(section); - section.setMeta(false); - section.types().add("section"); - section.setField("region", "center"); - - HitGroup yst = new HitGroup("yst"); - section.add(yst); - yst.setMeta(false); - yst.setSource("sr"); - yst.types().add("sr"); - yst.setField("provider", "yst"); - - Hit theHit = new Hit("159"); - yst.add(theHit); - theHit.setSource("sr"); - theHit.types().add("sr"); - theHit.setField("provider", "yst"); - theHit.setField("title", "Yahoo"); - - HitGroup meta = new HitGroup("meta"); - result.hits().add(meta); - meta.types().add("meta"); - - Hit log = new Hit("com.yahoo.search.federation.yst.YSTBackendSearcherproxy-tw1cache.idp.inktomisearch.com55556/search"); - meta.add(log); - log.setMeta(true); - log.setSource("sr"); - log.setField("provider", "yst"); - log.types().add("logging"); - log.setField(HTTPProviderSearcher.LOG_URI, "http://proxy-tw1cache.idp.inktomisearch.com:55556/search?qp=yahootw-twp&Fields=url%2Credirecturl%2Cdate%2Csize%2Cformat%2Csms_product%2Ccacheurl%2Cnodename%2Cid%2Clanguage%2Crsslinks%2Crssvalidatedlinks%2Ccpc%2Cclustertype%2Cxml.active_abstract%2Cactive_abstract_type%2Cactive_abstract_source%2Ccontract_id%2Ctranslated%2Cxml.ydir_tw_hotlist_data%2Cxml.summary%2Cclustercollision%2Cxml.pi_info%2Cpage_adult_overridable%2Cpage_spam_overridable%2Ccategory_ydir%2Chate_edb&Unique=doc%2Chost+2&QueryEncoding=utf-8&Query=ALLWORDS%28yahoo%29&Database=dewownrm-zh-tw&FirstResult=0&srcpvid=&cacheecho=1&ResultsEncoding=utf-8&QueryLanguage=Chinese-traditional&Region=US&NumResults=10&Client=yahoous2"); - log.setField(HTTPProviderSearcher.LOG_SCHEME, "http"); - log.setField(HTTPProviderSearcher.LOG_HOST, "proxy-tw1cache.idp.inktomisearch.com"); - log.setField(HTTPProviderSearcher.LOG_PORT, "55556"); - log.setField(HTTPProviderSearcher.LOG_PATH, "/search"); - log.setField(HTTPProviderSearcher.LOG_STATUS, "200"); - log.setField(HTTPProviderSearcher.LOG_LATENCY_CONNECT, "757"); - log.setField(HTTPProviderSearcher.LOG_RESPONSE_HEADER_PREFIX + "content-length", "16217"); - - result.analyzeHits(); - - return result; - } - - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/error.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/error.templ deleted file mode 100644 index 4e7a9379b73..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/error.templ +++ /dev/null @@ -1 +0,0 @@ -### Result diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/footer.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/footer.templ deleted file mode 100644 index 4e7a9379b73..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/footer.templ +++ /dev/null @@ -1 +0,0 @@ -### Result diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/header.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/header.templ deleted file mode 100644 index 4e7a9379b73..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/header.templ +++ /dev/null @@ -1 +0,0 @@ -### Result diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/hit.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/hit.templ deleted file mode 100644 index 4e7a9379b73..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/hit.templ +++ /dev/null @@ -1 +0,0 @@ -### Result diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/nohits.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/nohits.templ deleted file mode 100644 index 4e7a9379b73..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/asearch/nohits.templ +++ /dev/null @@ -1 +0,0 @@ -### Result diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/error.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/error.templ deleted file mode 100644 index 4e7a9379b73..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/error.templ +++ /dev/null @@ -1 +0,0 @@ -### Result diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/footer.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/footer.templ deleted file mode 100644 index 4e7a9379b73..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/footer.templ +++ /dev/null @@ -1 +0,0 @@ -### Result diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/header.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/header.templ deleted file mode 100644 index 4e7a9379b73..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/header.templ +++ /dev/null @@ -1 +0,0 @@ -### Result diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/hit.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/hit.templ deleted file mode 100644 index 4e7a9379b73..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/hit.templ +++ /dev/null @@ -1 +0,0 @@ -### Result diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/nohits.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/nohits.templ deleted file mode 100644 index 4e7a9379b73..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/cgi-bin/asearch/nohits.templ +++ /dev/null @@ -1 +0,0 @@ -### Result diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/error.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/error.templ deleted file mode 100644 index ca186b86259..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/error.templ +++ /dev/null @@ -1 +0,0 @@ -<ERROR CODE="$result.error.code">$result.error.message</ERROR> diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/footer.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/footer.templ deleted file mode 100644 index 07a5dd6a881..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/footer.templ +++ /dev/null @@ -1 +0,0 @@ -</RESULTSET>
\ No newline at end of file diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/header.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/header.templ deleted file mode 100644 index 8e3a001545d..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/header.templ +++ /dev/null @@ -1,2 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<RESULTSET TOTALHITS="$result.totalHitCount"> diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/hit.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/hit.templ deleted file mode 100644 index 428a2f15ef5..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/hit.templ +++ /dev/null @@ -1,5 +0,0 @@ -<XTEMPLATEHIT RELEVANCY="$relevancy" SOURCE="$hit.source" TYPE="$hit.typeString" OFFSET="$hitno"> -<FIELD NAME="uri">$uri</FIELD> -<FIELD NAME="category">$category</FIELD> -<FIELD NAME="bsumtitle">$bsumtitle</FIELD> -</XTEMPLATEHIT> diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/nohits.templ b/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/nohits.templ deleted file mode 100644 index d466f0bb7d2..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/templates/xsearch/nohits.templ +++ /dev/null @@ -1 +0,0 @@ -<XTEMPLATENOHITS/>
\ No newline at end of file diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/tilingexample.xml b/container-search/src/test/java/com/yahoo/prelude/templates/test/tilingexample.xml deleted file mode 100644 index ea5d56d2b31..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/tilingexample.xml +++ /dev/null @@ -1,65 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<result version="1.0"> - - <section id="section:center" region="center"> - <group type="navigation" relevance="1.0"></group> - <group type="ads" relevance="0.9"> - <hit type="ad" relevance="0.7"> - <id>http://www.hotels.com/</id> - <title>Cheap <hi>hotels</hi></title> - <body>Low Rates Guaranteed. Call a <hi>Hotel</hi> Expert.</body> - </hit> - <hit type="ad" relevance="0.6"> - <id>http://www.expedia.com/</id> - <title>Cheap <hi>hotels</hi> at Expedia</title> - <body>Expedia Special Rates Means We Guarantee Our Low Rates on Rooms.</body> - </hit> - </group> - <group type="hits" relevance="0.8"> - <hit type="hit.collapsed" relevance="0.8"> - <id>www.hotels.com</id> - <title>Hotels.com | Cheap Hotels | Discount Hotel Rooms | Motels | Lodging</title> - <body>Hotels.com helps you find great rates on hotels and discount <hi>hotel</hi> packages.</body> - </hit> - <hit type="hit" relevance="0.7"> - <id>www.indigohotels.com</id> - <title>Hotel Indigo Hotels United States - Official Web Site</title> - <body>Make Hotel Indigo online hotel reservations and book your hotel rooms today.</body> - </hit> - <hit type="hit" relevance="0.6"> - <id>www.all-hotels.com</id> - <title>All hotels</title> - <body>Online hotel directory and reservations.</body> - </hit> - </group> - <group type="ads" relevance="0.7"> - <hit type="ad" relevance="1.0"> - <id>www.daysinn.com</id> - <title>Days Inn Special Deal</title> - <body>Buy now and Save 15% Off Our Best Available Rate with Days Inn.</body> - </hit> - <hit type="ad" relevance="0.9"> - <id>http://www.expedia.com/</id> - <title>Cheap <hi>hotels</hi> at Expedia</title> - <body>Expedia Special Rates Means We Guarantee Our Low Rates on Rooms.</body> - </hit> - </group> - </section> - - <section id="section:right" region="right"> - <group type="ads" relevance="0.7"> - <hit type="ad" relevance="1.0"> - <id>www.daysinn.com</id> - <title>Days Inn Special Deal</title> - <body>Buy now and Save 15% Off Our Best Available Rate with Days Inn.</body> - </hit> - <hit type="ad" relevance="0.9"> - <id>www.holidayinn.com</id> - <title>Holiday Inn: Official Site</title> - <body>Book with Holiday Inn. Free Internet. Kids eat free.</body> - </hit> - </group> - </section> - -</result> diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/tilingexample2.xml b/container-search/src/test/java/com/yahoo/prelude/templates/test/tilingexample2.xml deleted file mode 100644 index 2e936b0015a..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/tilingexample2.xml +++ /dev/null @@ -1,23 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<result version="1.0"> - - <section id="section:center" region="center"> - <group type="sr" relevance="1.0" source="sr" provider="yst"> - <hit type="sr" relevance="1.0" source="sr" provider="yst"> - <id>159</id> - <title>Yahoo</title> - </hit> - </group> - </section> - - <meta> - <provider name="yst" scheme="http" host="proxy-tw1cache.idp.inktomisearch.com" port="55556" path="/search" result="200"> - <id>com.yahoo.search.federation.yst.YSTBackendSearcherproxy-tw1cache.idp.inktomisearch.com55556/search</id> - <uri>http://proxy-tw1cache.idp.inktomisearch.com:55556/search?qp=yahootw-twp&Fields=url%2Credirecturl%2Cdate%2Csize%2Cformat%2Csms_product%2Ccacheurl%2Cnodename%2Cid%2Clanguage%2Crsslinks%2Crssvalidatedlinks%2Ccpc%2Cclustertype%2Cxml.active_abstract%2Cactive_abstract_type%2Cactive_abstract_source%2Ccontract_id%2Ctranslated%2Cxml.ydir_tw_hotlist_data%2Cxml.summary%2Cclustercollision%2Cxml.pi_info%2Cpage_adult_overridable%2Cpage_spam_overridable%2Ccategory_ydir%2Chate_edb&Unique=doc%2Chost+2&QueryEncoding=utf-8&Query=ALLWORDS%28yahoo%29&Database=dewownrm-zh-tw&FirstResult=0&srcpvid=&cacheecho=1&ResultsEncoding=utf-8&QueryLanguage=Chinese-traditional&Region=US&NumResults=10&Client=yahoous2</uri> - <latency type="connect">757</latency> - <response-header name="content-length">16217</response-header> - </provider> - </meta> - -</result> diff --git a/container-search/src/test/java/com/yahoo/prelude/test/RankFeatureDumpTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/RankFeatureDumpTestCase.java deleted file mode 100644 index 3e57c48e7f3..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/test/RankFeatureDumpTestCase.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import com.google.common.util.concurrent.MoreExecutors; -import com.yahoo.component.chain.Chain; -import com.yahoo.language.Linguistics; -import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.prelude.templates.test.TilingTestCase; -import com.yahoo.search.rendering.RendererRegistry; -import com.yahoo.search.result.Hit; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.search.Searcher; -import com.yahoo.search.searchchain.Execution; -import org.junit.Test; - -import static org.junit.Assert.assertTrue; - -/** - * Tests that rank features are rendered when requested in the query - * - * @author bratseth - */ -@SuppressWarnings("deprecation") -public class RankFeatureDumpTestCase { - - private static final String rankFeatureString= - "{\"match.weight.as1\":10,\"attribute(ai1)\":1.000000,\"proximity(as1, 1, 2)\":2.000000}"; - - @Test - public void test() throws IOException { - Query query=new Query("?query=five&rankfeatures"); - assertTrue(query.getRanking().getListFeatures()); // New api - Result result = doSearch(new MockBackend(), query, 0,10); - assertTrue(TilingTestCase.getRendered(result).contains( - "<field name=\"" + com.yahoo.search.result.Hit.RANKFEATURES_FIELD + "\">" + rankFeatureString + "</field>")); - } - - private static class MockBackend extends Searcher { - - @Override - public Result search(com.yahoo.search.Query query, Execution execution) { - Result result=new Result(query); - Hit hit=new FastHit("test",1000); - hit.setField(com.yahoo.search.result.Hit.RANKFEATURES_FIELD,rankFeatureString); - result.hits().add(hit); - return result; - } - - } - - private Result doSearch(Searcher searcher, Query query, int offset, int hits) { - query.setOffset(offset); - query.setHits(hits); - return createExecution(searcher).search(query); - } - - private Execution createExecution(Searcher searcher) { - Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(MoreExecutors.directExecutor()), new SimpleLinguistics()); - return new Execution(chainedAsSearchChain(searcher), context); - } - - private Chain<Searcher> chainedAsSearchChain(Searcher topOfChain) { - List<Searcher> searchers = new ArrayList<>(); - searchers.add(topOfChain); - return new Chain<>(searchers); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java deleted file mode 100644 index 6babba5a36a..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import static org.junit.Assert.*; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.UnknownHostException; -import java.util.Arrays; -import java.util.Random; -import java.util.zip.GZIPOutputStream; - -import org.apache.http.Header; -import org.apache.http.HttpEntity; -import org.apache.http.message.BasicHeader; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.yahoo.text.Utf8; - -/** - * Test GZip support for the HTTP integration introduced in 4.2. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class GzipDecompressingEntityTestCase { - private static final String STREAM_CONTENT = "00000000000000000000000000000000000000000000000000"; - private static final byte[] CONTENT_AS_BYTES = Utf8.toBytes(STREAM_CONTENT); - GzipDecompressingEntity testEntity; - - private static final class MockEntity implements HttpEntity { - - private final InputStream inStream; - - MockEntity(InputStream is) { - inStream = is; - } - - @Override - public boolean isRepeatable() { - return false; - } - - @Override - public boolean isChunked() { - return false; - } - - @Override - public long getContentLength() { - return -1; - } - - @Override - public Header getContentType() { - return new BasicHeader("Content-Type", "text/plain"); - } - - @Override - public Header getContentEncoding() { - return new BasicHeader("Content-Encoding", "gzip"); - } - - @Override - public InputStream getContent() throws IOException, - IllegalStateException { - return inStream; - } - - @Override - public void writeTo(OutputStream outstream) throws IOException { - } - - @Override - public boolean isStreaming() { - return false; - } - - @Override - public void consumeContent() throws IOException { - } - } - - @Before - public void setUp() throws Exception { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - GZIPOutputStream gzip = new GZIPOutputStream(out); - gzip.write(CONTENT_AS_BYTES); - gzip.finish(); - gzip.close(); - byte[] compressed = out.toByteArray(); - InputStream inStream = new ByteArrayInputStream(compressed); - testEntity = new GzipDecompressingEntity(new MockEntity(inStream)); - } - - @After - public void tearDown() throws Exception { - } - - @Test - public final void testGetContentLength() throws UnknownHostException { - assertEquals(STREAM_CONTENT.length(), testEntity.getContentLength()); - } - - @Test - public final void testGetContent() throws IllegalStateException, IOException { - InputStream in = testEntity.getContent(); - byte[] buffer = new byte[CONTENT_AS_BYTES.length]; - int read = in.read(buffer); - assertEquals(CONTENT_AS_BYTES.length, read); - assertArrayEquals(CONTENT_AS_BYTES, buffer); - } - - @Test - public final void testGetContentToBigArray() throws IllegalStateException, IOException { - InputStream in = testEntity.getContent(); - byte[] buffer = new byte[CONTENT_AS_BYTES.length * 2]; - in.read(buffer); - byte[] expected = Arrays.copyOf(CONTENT_AS_BYTES, CONTENT_AS_BYTES.length * 2); - assertArrayEquals(expected, buffer); - } - - @Test - public final void testGetContentAvailable() throws IllegalStateException, IOException { - InputStream in = testEntity.getContent(); - assertEquals(CONTENT_AS_BYTES.length, in.available()); - } - - @Test - public final void testLargeZip() throws IOException { - byte [] input = new byte [10000000]; - Random random = new Random(89); - for (int i = 0; i < input.length; i++) { - input[i] = (byte) random.nextInt(); - } - ByteArrayOutputStream out = new ByteArrayOutputStream(); - GZIPOutputStream gzip = new GZIPOutputStream(out); - gzip.write(input); - gzip.finish(); - gzip.close(); - byte[] compressed = out.toByteArray(); - assertEquals(10003073, compressed.length); - InputStream inStream = new ByteArrayInputStream(compressed); - GzipDecompressingEntity gunzipper = new GzipDecompressingEntity(new MockEntity(inStream)); - assertEquals(input.length, gunzipper.getContentLength()); - byte[] buffer = new byte[input.length]; - InputStream content = gunzipper.getContent(); - assertEquals(input.length, content.available()); - int read = content.read(buffer); - assertEquals(input.length, read); - assertArrayEquals(input, buffer); - } - - @Test - public final void testGetContentReadByte() throws IllegalStateException, IOException { - InputStream in = testEntity.getContent(); - byte[] buffer = new byte[CONTENT_AS_BYTES.length * 2]; - int i = 0; - while (i < buffer.length) { - int r = in.read(); - if (r == -1) { - break; - } else { - buffer[i++] = (byte) r; - } - } - byte[] expected = Arrays.copyOf(CONTENT_AS_BYTES, CONTENT_AS_BYTES.length * 2); - assertEquals(CONTENT_AS_BYTES.length, i); - assertArrayEquals(expected, buffer); - } - - @Test - public final void testGetContentReadWithOffset() throws IllegalStateException, IOException { - InputStream in = testEntity.getContent(); - byte[] buffer = new byte[CONTENT_AS_BYTES.length * 2]; - int read = in.read(buffer, CONTENT_AS_BYTES.length, CONTENT_AS_BYTES.length); - assertEquals(CONTENT_AS_BYTES.length, read); - byte[] expected = new byte[CONTENT_AS_BYTES.length * 2]; - for (int i = 0; i < CONTENT_AS_BYTES.length; ++i) { - expected[CONTENT_AS_BYTES.length + i] = CONTENT_AS_BYTES[i]; - } - assertArrayEquals(expected, buffer); - read = in.read(buffer, 0, CONTENT_AS_BYTES.length); - assertEquals(-1, read); - } - - @Test - public final void testGetContentSkip() throws IllegalStateException, IOException { - InputStream in = testEntity.getContent(); - final long n = 5L; - long skipped = in.skip(n); - assertEquals(n, skipped); - int read = in.read(); - assertEquals(CONTENT_AS_BYTES[(int) n], read); - skipped = in.skip(5000); - assertEquals(CONTENT_AS_BYTES.length - n - 1, skipped); - assertEquals(-1L, in.skip(1L)); - } - - - @Test - public final void testWriteToOutputStream() throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - testEntity.writeTo(out); - assertArrayEquals(CONTENT_AS_BYTES, out.toByteArray()); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java b/container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java deleted file mode 100644 index 11f5d7d14fd..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.search.federation.ProviderConfig; -import org.junit.Test; - -import static com.yahoo.search.federation.ProviderConfig.Yca; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -/** - * @author gjoranv - * @author Steinar Knutsen - */ -public class HttpParametersTest { - - @Test - public void create_from_config() throws Exception { - ProviderConfig config = new ProviderConfig(new ProviderConfig.Builder() - .connectionTimeout(1.0) - .maxConnectionPerRoute(2) - .maxConnections(3) - .path("myPath") - .readTimeout(4) - .socketBufferBytes(5) - .yca(new Yca.Builder() - .applicationId("myId") - .host("myYcaHost") - .port(7) - .retry(8) - .ttl(9) - .useProxy(true))); - - HTTPParameters httpParameters = new HTTPParameters(config); - - // Written to configuredConnectionTimeout, but it is not accessible!? - //assertThat(httpParameters.getConnectionTimeout(), is(1000)); - - - // This value is not set from config by the constructor!? - //assertThat(httpParameters.getMaxConnectionsPerRoute(), is(2)); - - // This value is not set from config by the constructor!? - //assertThat(httpParameters.getMaxTotalConnections(), is(3)); - - assertThat(httpParameters.getPath(), is("/myPath")); - - // This value is not set from config by the constructor!? - //assertThat(httpParameters.getReadTimeout(), is(4)); - - // This value is not set from config by the constructor!? - //assertThat(httpParameters.getSocketBufferSizeBytes(), is(5)); - - - assertThat(httpParameters.getYcaUseProxy(), is(true)); - assertThat(httpParameters.getYcaApplicationId(), is("myId")); - assertThat(httpParameters.getYcaProxy(), is("myYcaHost")); - assertThat(httpParameters.getYcaPort(), is(7)); - assertThat(httpParameters.getYcaRetry(), is(8000L)); - assertThat(httpParameters.getYcaTtl(), is(9000L)); - } - - @Test - public void requireFreezeWorksForAccessors() { - HTTPParameters p = new HTTPParameters(); - boolean caught = false; - final int expected = 37; - p.setConnectionTimeout(expected); - assertEquals(expected, p.getConnectionTimeout()); - p.freeze(); - try { - p.setConnectionTimeout(0); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setReadTimeout(expected); - assertEquals(expected, p.getReadTimeout()); - p.freeze(); - try { - p.setReadTimeout(0); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setPersistentConnections(true); - assertTrue(p.getPersistentConnections()); - p.freeze(); - try { - p.setPersistentConnections(false); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - assertEquals("http", p.getProxyType()); - - p = new HTTPParameters(); - caught = false; - p.setEnableProxy(true); - assertTrue(p.getEnableProxy()); - p.freeze(); - try { - p.setEnableProxy(false); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setProxyHost("nalle"); - assertEquals("nalle", p.getProxyHost()); - p.freeze(); - try { - p.setProxyHost("jappe"); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setProxyPort(expected); - assertEquals(expected, p.getProxyPort()); - p.freeze(); - try { - p.setProxyPort(0); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setMethod("POST"); - assertEquals("POST", p.getMethod()); - p.freeze(); - try { - p.setMethod("GET"); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setSchema("gopher"); - assertEquals("gopher", p.getSchema()); - p.freeze(); - try { - p.setSchema("http"); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setInputEncoding("iso-8859-15"); - assertEquals("iso-8859-15", p.getInputEncoding()); - p.freeze(); - try { - p.setInputEncoding("shift-jis"); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setOutputEncoding("iso-8859-15"); - assertEquals("iso-8859-15", p.getOutputEncoding()); - p.freeze(); - try { - p.setOutputEncoding("shift-jis"); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setMaxTotalConnections(expected); - assertEquals(expected, p.getMaxTotalConnections()); - p.freeze(); - try { - p.setMaxTotalConnections(0); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setMaxConnectionsPerRoute(expected); - assertEquals(expected, p.getMaxConnectionsPerRoute()); - p.freeze(); - try { - p.setMaxConnectionsPerRoute(0); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setSocketBufferSizeBytes(expected); - assertEquals(expected, p.getSocketBufferSizeBytes()); - p.freeze(); - try { - p.setSocketBufferSizeBytes(0); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setRetries(expected); - assertEquals(expected, p.getRetries()); - p.freeze(); - try { - p.setRetries(0); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - } -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java deleted file mode 100644 index e0b2afe4e1f..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.component.ComponentId; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.StupidSingleThreadedHttpServer; -import com.yahoo.search.federation.ProviderConfig.PingOption; -import com.yahoo.search.federation.http.Connection; -import com.yahoo.search.federation.http.HTTPProviderSearcher; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; -import org.apache.http.HttpEntity; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.entity.StringEntity; -import org.junit.Test; - -import java.io.InputStream; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URI; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.core.StringContains.containsString; -import static org.junit.Assert.assertThat; - -/** - * See bug #3234696. - * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - */ -public class HttpPostTestCase { - - @Test - public void testPostingSearcher() throws Exception { - StupidSingleThreadedHttpServer server = new StupidSingleThreadedHttpServer(); - server.start(); - - TestPostSearcher searcher = new TestPostSearcher(new ComponentId("foo:1"), - Arrays.asList(new Connection("localhost", server.getServerPort())), - "/"); - Query q = new Query(""); - q.setTimeout(10000000L); - Execution e = new Execution(searcher, Execution.Context.createContextStub()); - - searcher.search(q, e); - - assertThat(server.getRequest(), containsString("My POST body")); - server.stop(); - } - - private static class TestPostSearcher extends HTTPProviderSearcher { - public TestPostSearcher(ComponentId id, List<Connection> connections, String path) { - super(id, connections, httpParameters(path), Statistics.nullImplementation); - } - - private static HTTPParameters httpParameters(String path) { - HTTPParameters httpParameters = new HTTPParameters(path); - httpParameters.setPingOption(PingOption.Enum.DISABLE); - return httpParameters; - } - - @Override - protected HttpUriRequest createRequest(String method, URI uri, HttpEntity entity) { - HttpPost request = new HttpPost(uri); - request.setEntity(entity); - return request; - } - - @Override - protected HttpEntity getRequestEntity(Query query, Hit requestMeta) { - try { - return new StringEntity("My POST body"); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); - } - } - - @Override - public Map<String, String> getCacheKey(Query q) { - return new HashMap<>(0); - } - - @Override - public void unmarshal(final InputStream stream, long contentLength, final Result result) throws IOException { - // do nothing with the result - } - - @Override - protected void fill(Result result, String summaryClass, Execution execution, Connection connection) { - //Empty - } - } -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java deleted file mode 100644 index b4f3c13b8e2..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.component.ComponentId; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.StupidSingleThreadedHttpServer; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; -import com.yahoo.text.Utf8; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import javax.xml.bind.JAXBException; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; - -/** - * Rudimentary http searcher test. - * - * @author bratseth - */ -public class HttpTestCase { - - private StupidSingleThreadedHttpServer httpServer; - private TestHTTPClientSearcher searcher; - - @Test - public void testSearcher() throws JAXBException { - Result result = searchUsingLocalhost(); - - assertEquals("ok", result.getQuery().properties().get("gotResponse")); - assertEquals(0, result.getQuery().errors().size()); - } - - private Result searchUsingLocalhost() { - searcher = new TestHTTPClientSearcher("test","localhost",getPort()); - Query query = new Query("/?query=test"); - - query.setWindow(0,10); - return searcher.search(query, new Execution(searcher, Execution.Context.createContextStub())); - } - - @Test - public void test_that_ip_address_set_on_meta_hit() { - Result result = searchUsingLocalhost(); - Hit metaHit = getFirstMetaHit(result.hits()); - String ip = (String) metaHit.getField(HTTPSearcher.LOG_IP_ADDRESS); - - assertEquals(ip, "127.0.0.1"); - } - - private Hit getFirstMetaHit(HitGroup hits) { - for (Iterator<Hit> i = hits.unorderedDeepIterator(); i.hasNext();) { - Hit hit = i.next(); - if (hit.isMeta()) - return hit; - } - return null; - } - - @Before - public void setUp() throws Exception { - httpServer = new StupidSingleThreadedHttpServer(0, 0) { - @Override - protected byte[] getResponse(String request) { - return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + - "Content-Type: text/xml; charset=UTF-8\r\n" + - "Connection: close\r\n" + - "Content-Length: 5\r\n" + - "\r\n" + - "hello"); - } - }; - httpServer.start(); - } - - private int getPort() { - return httpServer.getServerPort(); - } - - @After - public void tearDown() throws Exception { - httpServer.stop(); - if (searcher != null) { - searcher.shutdownConnectionManagers(); - } - } - - private static class TestHTTPClientSearcher extends HTTPClientSearcher { - - public TestHTTPClientSearcher(String id, String hostName, int port) { - super(new ComponentId(id), toConnections(hostName,port), "", Statistics.nullImplementation); - } - - private static List<Connection> toConnections(String hostName,int port) { - List<Connection> connections=new ArrayList<>(); - connections.add(new Connection(hostName,port)); - return connections; - } - - @Override - public Query handleResponse(InputStream inputStream, long contentLength, Query query) throws IOException { - query.properties().set("gotResponse","ok"); - return query; - } - - @Override - public Map<String, String> getCacheKey(Query q) { - return null; - } - - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java deleted file mode 100644 index 8c1ff69666b..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java +++ /dev/null @@ -1,288 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.component.ComponentId; -import com.yahoo.prelude.Ping; -import com.yahoo.prelude.Pong; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.StupidSingleThreadedHttpServer; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; -import com.yahoo.text.Utf8; -import com.yahoo.yolean.Exceptions; -import org.apache.http.HttpEntity; -import org.junit.Test; - -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Check for different keep-alive scenarios. What we really want to test - * is the server does not hang. - * - * @author Steinar Knutsen - */ -public class PingTestCase { - - private static final int TIMEOUT_MS = 60000; - - @Test - public void testNiceCase() throws Exception { - NiceStupidServer server = new NiceStupidServer(); - server.start(); - checkSearchAndPing(true, true, true, server.getServerPort()); - server.stop(); - } - - private void checkSearchAndPing(boolean firstSearch, boolean pongCheck, boolean secondSearch, int port) { - String resultThing; - String comment; - TestHTTPClientSearcher searcher = new TestHTTPClientSearcher("test", - "localhost", port); - try { - - Query query = new Query("/?query=test"); - - query.setWindow(0, 10); - // high timeout to allow for overloaded test machine - query.setTimeout(TIMEOUT_MS); - Ping ping = new Ping(TIMEOUT_MS); - - long start = System.currentTimeMillis(); - Execution exe = new Execution(searcher, Execution.Context.createContextStub()); - exe.search(query); - - resultThing = firstSearch ? "ok" : null; - comment = firstSearch ? "First search should have succeeded." : "First search should fail."; - assertEquals(comment, resultThing, query.properties().get("gotResponse")); - Pong pong = searcher.ping(ping, searcher.getConnection()); - if (pongCheck) { - assertEquals("Ping should not have failed.", 0, pong.getErrorSize()); - } else { - assertEquals("Ping should have failed.", 1, pong.getErrorSize()); - } - exe = new Execution(searcher, Execution.Context.createContextStub()); - exe.search(query); - - resultThing = secondSearch ? "ok" : null; - comment = secondSearch ? "Second search should have succeeded." : "Second search should fail."; - - assertEquals(resultThing, query.properties().get("gotResponse")); - long duration = System.currentTimeMillis() - start; - // target for duration based on the timeout values + some slack - assertTrue("This test probably hanged.", duration < TIMEOUT_MS + 4000); - searcher.shutdownConnectionManagers(); - } finally { - searcher.deconstruct(); - } - } - - @Test - public void testUselessCase() throws Exception { - UselessStupidServer server = new UselessStupidServer(); - server.start(); - checkSearchAndPing(false, true, false, server.getServerPort()); - server.stop(); - } - - @Test - public void testGrumpyCase() throws Exception { - GrumpyStupidServer server = new GrumpyStupidServer(); - server.start(); - checkSearchAndPing(false, false, false, server.getServerPort()); - server.stop(); - } - - @Test - public void testPassiveAggressiveCase() throws Exception { - PassiveAggressiveStupidServer server = new PassiveAggressiveStupidServer(); - server.start(); - checkSearchAndPing(true, false, true, server.getServerPort()); - server.stop(); - } - - // OK on ping and search - private static class NiceStupidServer extends StupidSingleThreadedHttpServer { - private NiceStupidServer() throws IOException { - super(0, 0); - } - - @Override - protected byte[] getResponse(String request) { - return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + - "Content-Type: text/xml; charset=UTF-8\r\n" + - "Connection: close\r\n" + - "Content-Length: 6\r\n" + - "\r\n" + - "hello\n"); - } - } - - // rejects ping and accepts search - private static class PassiveAggressiveStupidServer extends StupidSingleThreadedHttpServer { - - private PassiveAggressiveStupidServer() throws IOException { - super(0, 0); - } - - @Override - protected byte[] getResponse(String request) { - if (request.contains("/ping")) { - return Utf8.toBytes("HTTP/1.1 404 Not found\r\n" + - "Content-Type: text/xml; charset=UTF-8\r\n" + - "Connection: close\r\n" + - "Content-Length: 8\r\n" + - "\r\n" + - "go away\n"); - } else { - return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + - "Content-Type: text/xml; charset=UTF-8\r\n" + - "Connection: close\r\n" + - "Content-Length: 6\r\n" + - "\r\n" + - "hello\n"); - } - } - } - - // accepts ping and rejects search - private static class UselessStupidServer extends StupidSingleThreadedHttpServer { - private UselessStupidServer() throws IOException { - super(0, 0); - } - - - @Override - protected byte[] getResponse(String request) { - if (request.contains("/ping")) { - return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + - "Content-Type: text/xml; charset=UTF-8\r\n" + - "Connection: close\r\n" + - "Content-Length: 6\r\n" + - "\r\n" + - "hello\n"); - } else { - return Utf8.toBytes("HTTP/1.1 404 Not found\r\n" + - "Content-Type: text/xml; charset=UTF-8\r\n" + - "Connection: close\r\n" + - "Content-Length: 8\r\n" + - "\r\n" + - "go away\n"); - } - } - } - - // rejects ping and search - private static class GrumpyStupidServer extends StupidSingleThreadedHttpServer { - private GrumpyStupidServer() throws IOException { - super(0, 0); - } - - @Override - protected byte[] getResponse(String request) { - return Utf8.toBytes("HTTP/1.1 404 Not found\r\n" + - "Content-Type: text/xml; charset=UTF-8\r\n" + - "Connection: close\r\n" + - "Content-Length: 8\r\n" + - "\r\n" + - "go away\n"); - } - } - - private static class TestHTTPClientSearcher extends HTTPClientSearcher { - - public TestHTTPClientSearcher(String id, String hostName, int port) { - super(new ComponentId(id), toConnections(hostName,port), "", Statistics.nullImplementation); - } - - private static List<Connection> toConnections(String hostName,int port) { - List<Connection> connections=new ArrayList<>(); - connections.add(new Connection(hostName,port)); - return connections; - } - - @Override - public Query handleResponse(InputStream inputStream, long contentLength, Query query) throws IOException { - query.properties().set("gotResponse","ok"); - return query; - } - - @Override - public Result search(Query query, Execution execution, - Connection connection) { - URI uri; - try { - uri = new URL("http", connection.getHost(), connection - .getPort(), "/search").toURI(); - } catch (MalformedURLException e) { - query.errors().add(createMalformedUrlError(query, e)); - return execution.search(query); - } catch (URISyntaxException e) { - query.errors().add(createMalformedUrlError(query, e)); - return execution.search(query); - } - - HttpEntity entity; - try { - entity = getEntity(uri, query); - } catch (IOException e) { - query.errors().add( - ErrorMessage.createBackendCommunicationError("Error when trying to connect to HTTP backend in " - + this + " using " + connection - + " for " + query + ": " - + Exceptions.toMessageString(e))); - return execution.search(query); - } catch (TimeoutException e) { - query.errors().add(ErrorMessage.createTimeout("No time left for HTTP traffic in " - + this - + " for " + query + ": " + e.getMessage())); - return execution.search(query); - } - if (entity == null) { - query.errors().add( - ErrorMessage.createBackendCommunicationError("No result from connecting to HTTP backend in " - + this + " using " + connection + " for " + query)); - return execution.search(query); - } - - try { - query = handleResponse(entity, query); - } catch (IOException e) { - query.errors().add( - ErrorMessage.createBackendCommunicationError("Error when trying to consume input in " - + this + ": " + Exceptions.toMessageString(e))); - } finally { - cleanupHttpEntity(entity); - } - return execution.search(query); - } - - @Override - public Map<String, String> getCacheKey(Query q) { - return null; - } - - @Override - protected URI getPingURI(Connection connection) - throws MalformedURLException, URISyntaxException { - return new URL("http", connection.getHost(), connection.getPort(), "/ping").toURI(); - } - - Connection getConnection() { - return getHasher().getNodes().select(0, 0); - } - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java deleted file mode 100644 index 19750cf84cc..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.component.ComponentId; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; -import com.yahoo.vespa.defaults.Defaults; -import org.junit.Test; - -import java.util.Collections; -import java.util.Map; - -import static org.junit.Assert.assertEquals; - -/** - * Tests that source and backend specific parameters from the query are added correctly to the backend requests - * - * @author bratseth - */ -public class QueryParametersTestCase { - - @Test - public void testQueryParameters() { - Query query=new Query(); - query.properties().set("a","a-value"); - query.properties().set("b.c","b.c-value"); - query.properties().set("source.otherSource.d","d-value"); - query.properties().set("source.testSource.e","e-value"); - query.properties().set("source.testSource.f.g","f.g-value"); - query.properties().set("provider.testProvider.h","h-value"); - query.properties().set("provider.testProvider.i.j","i.j-value"); - - query.properties().set("sourceName","testSource"); // Done by federation searcher - query.properties().set("providerName","testProvider"); // Done by federation searcher - - TestHttpProvider searcher=new TestHttpProvider(); - Map<String,String> parameters=searcher.getQueryMap(query); - searcher.deconstruct(); - - assertEquals(4,parameters.size()); // the appropriate 4 of the above - assertEquals(parameters.get("e"),"e-value"); - assertEquals(parameters.get("f.g"),"f.g-value"); - assertEquals(parameters.get("h"),"h-value"); - assertEquals(parameters.get("i.j"),"i.j-value"); - } - - public static class TestHttpProvider extends HTTPProviderSearcher { - - public TestHttpProvider() { - super(new ComponentId("test"), Collections.singletonList(new Connection("host", Defaults.getDefaults().vespaWebServicePort())), "path", Statistics.nullImplementation); - } - - @Override - public Map<String, String> getCacheKey(Query q) { - return Collections.singletonMap("nocaching", String.valueOf(Math.random())); - } - - @Override - protected void fill(Result result, String summaryClass, Execution execution, Connection connection) { - } - - } - -} - diff --git a/container-search/src/test/java/com/yahoo/search/federation/image/.gitignore b/container-search/src/test/java/com/yahoo/search/federation/image/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/image/.gitignore +++ /dev/null diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java deleted file mode 100644 index 022177bc42b..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa.test; - -import com.yahoo.language.Linguistics; -import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.*; -import com.yahoo.search.Query; -import com.yahoo.search.federation.vespa.QueryMarshaller; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.search.test.QueryTestCase; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -public class QueryMarshallerTestCase { - - private static final Linguistics linguistics = new SimpleLinguistics(); - - @Test - public void testCommonCommonCase() { - AndItem root = new AndItem(); - addThreeWords(root); - assertEquals("a AND b AND c", new QueryMarshaller().marshal(root)); - } - - @Test - public void testPhrase() { - PhraseItem root = new PhraseItem(); - root.setIndexName("habla"); - addThreeWords(root); - assertEquals("habla:\"a b c\"", new QueryMarshaller().marshal(root)); - } - - @Test - public void testPhraseDefaultIndex() { - PhraseItem root = new PhraseItem(); - addThreeWords(root); - assertEquals("\"a b c\"", new QueryMarshaller().marshal(root)); - } - - @Test - public void testLittleMoreComplex() { - AndItem root = new AndItem(); - addThreeWords(root); - OrItem ambig = new OrItem(); - root.addItem(ambig); - addThreeWords(ambig); - AndItem but = new AndItem(); - addThreeWords(but); - ambig.addItem(but); - assertEquals("a AND b AND c AND ( a OR b OR c OR ( a AND b AND c ) )", - new QueryMarshaller().marshal(root)); - } - - @Test - public void testRank() { - RankItem root = new RankItem(); - addThreeWords(root); - assertEquals("a RANK b RANK c", new QueryMarshaller().marshal(root)); - } - - @Test - public void testNear() { - NearItem near = new NearItem(3); - addThreeWords(near); - assertEquals("a NEAR(3) b NEAR(3) c", new QueryMarshaller().marshal(near)); - } - - @Test - public void testONear() { - ONearItem oNear = new ONearItem(3); - addThreeWords(oNear); - assertEquals("a ONEAR(3) b ONEAR(3) c", new QueryMarshaller().marshal(oNear)); - } - - private void addThreeWords(CompositeItem root) { - root.addItem(new WordItem("a")); - root.addItem(new WordItem("b")); - root.addItem(new WordItem("c")); - } - - @Test - public void testNegativeGroupedTerms() { - testQueryString(new QueryMarshaller(), "a -(b c) -(d e)", - "a ANDNOT ( b AND c ) ANDNOT ( d AND e )"); - } - - @Test - public void testPositiveGroupedTerms() { - testQueryString(new QueryMarshaller(), "a (b c)", "a AND ( b OR c )"); - } - - @Test - public void testInt() { - testQueryString(new QueryMarshaller(), "yahoo 123", "yahoo AND 123"); - } - - @Test - public void testCJKOneWord() { - testQueryString(new QueryMarshaller(), "天龍人"); - } - - @Test - public void testTwoWords() { - testQueryString(new QueryMarshaller(), "John Smith", "John AND Smith", null, new SimpleLinguistics()); - } - - @Test - public void testTwoWordsInPhrase() { - testQueryString(new QueryMarshaller(), "\"John Smith\"", "\"John Smith\"", null, new SimpleLinguistics()); - } - - @Test - public void testCJKTwoSentences() { - testQueryString(new QueryMarshaller(), "是不是這樣的夜晚 你才會這樣地想起我", "是不是這樣的夜晚 AND 你才會這樣地想起我"); - } - - @Test - public void testCJKTwoSentencesWithLanguage() { - testQueryString(new QueryMarshaller(), "助妳好孕 生1胎北市發2萬", "助妳好孕 AND 生1胎北市發2萬", "zh-Hant"); - } - - @Test - public void testCJKTwoSentencesInPhrase() { - QueryMarshaller marshaller = new QueryMarshaller(); - testQueryString(marshaller, "\"助妳好孕 生1胎北市發2萬\"", "\"助妳好孕 生1胎北市發2萬\"", "zh-Hant"); - testQueryString(marshaller, "\"是不是這樣的夜晚 你才會這樣地想起我\"", "\"是不是這樣的夜晚 你才會這樣地想起我\""); - } - - @Test - public void testCJKMultipleSentences() { - testQueryString(new QueryMarshaller(), "염부장님과 함께했던 좋은 추억들은", "염부장님과 AND 함께했던 AND 좋은 AND 추억들은"); - } - - @Test - public void testIndexRestriction() { - /** ticket 3707606, comment #29 */ - testQueryString(new QueryMarshaller(), "site:nytimes.com", "site:\"nytimes com\""); - } - - private void testQueryString(QueryMarshaller marshaller, String uq) { - testQueryString(marshaller, uq, uq, null); - } - - private void testQueryString(QueryMarshaller marshaller, String uq, String mq) { - testQueryString(marshaller, uq, mq, null); - } - - private void testQueryString(QueryMarshaller marshaller, String uq, String mq, String lang) { - testQueryString(marshaller, uq, mq, lang, linguistics); - } - - private void testQueryString(QueryMarshaller marshaller, String uq, String mq, String lang, Linguistics linguistics) { - Query query = new Query("/?query=" + QueryTestCase.httpEncode(uq) + ((lang != null) ? "&language=" + lang : "")); - query.getModel().setExecution(new Execution(new Execution.Context(null, new IndexFacts(), null, null, linguistics))); - assertEquals(mq, marshaller.marshal(query.getModel().getQueryTree().getRoot())); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java deleted file mode 100644 index 8d1ecf4085e..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa.test; - -import com.yahoo.search.Query; -import com.yahoo.search.federation.vespa.VespaSearcher; -import java.util.Map; - -import static org.junit.Assert.assertEquals; - -/** - * Tests that source and backend specific parameters from the query are added correctly to the backend requests - * - * @author bratseth - */ -public class QueryParametersTestCase { - - public void testQueryParameters() { - Query query=new Query(); - query.properties().set("a","a-value"); - query.properties().set("b.c","b.c-value"); - query.properties().set("source.otherSource.d","d-value"); - query.properties().set("source.testSource.e","e-value"); - query.properties().set("source.testSource.f.g","f.g-value"); - query.properties().set("provider.testProvider.h","h-value"); - query.properties().set("provider.testProvider.i.j","i.j-value"); - - query.properties().set("sourceName","testSource"); // Done by federation searcher - query.properties().set("providerName","testProvider"); // Done by federation searcher - - VespaSearcher searcher=new VespaSearcher("testProvider","",0,""); - Map<String,String> parameters=searcher.getQueryMap(query); - searcher.deconstruct(); - - assertEquals(9, parameters.size()); // 5 standard + the appropriate 4 of the above - assertEquals(parameters.get("e"),"e-value"); - assertEquals(parameters.get("f.g"),"f.g-value"); - assertEquals(parameters.get("h"),"h-value"); - assertEquals(parameters.get("i.j"),"i.j-value"); - } - -} - diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java deleted file mode 100644 index 50186e03ae8..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa.test; - -import java.util.Iterator; - -import com.yahoo.net.URI; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.federation.vespa.ResultBuilder; -import com.yahoo.search.result.ErrorHit; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.HitGroup; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -/** - * Test XML parsing of results. - * - * @author Steinar Knutsen - */ -@SuppressWarnings("deprecation") -public class ResultBuilderTestCase { - - private boolean quickCompare(double a, double b) { - double z = Math.min(Math.abs(a), Math.abs(b)); - if (Math.abs((a - b)) < (z / 1e14)) { - return true; - } else { - return false; - } - } - - @Test - public void testSimpleResult() { - boolean gotErrorDetails = false; - ResultBuilder r = new ResultBuilder(); - Result res = r.parse("file:src/test/java/com/yahoo/prelude/searcher/test/testhit.xml", new Query("?query=a")); - assertEquals(3, res.getConcreteHitCount()); - assertEquals(4, res.getHitCount()); - ErrorHit e = (ErrorHit) res.hits().get(0); - // known problem, if the same error is the main error is - // in details, it'll be added twice. Not sure how to fix that, - // because old Vespa systems give no error details, and there - // is no way of nuking an existing error if the details exist. - for (Iterator<?> i = e.errorIterator(); i.hasNext();) { - ErrorMessage err = (ErrorMessage) i.next(); - assertEquals(5, err.getCode()); - String details = err.getDetailedMessage(); - if (details != null) { - gotErrorDetails = true; - assertEquals("An error as ordered", details.trim()); - } - } - assertTrue("Error details are missing", gotErrorDetails); - assertEquals(new URI("http://def"), res.hits().get(1).getId()); - assertEquals("test/stuff\\tsome/other", res.hits().get(2).getField("category")); - assertEquals("<field>habla</field>" - + "<hi>blbl</hi><br /><>&fdlkkgj</field>;lk<a b=\"1\" c=\"2\" />" - + "<x><y><z /></y></x>", res.hits().get(3).getField("annoying").toString()); - } - - @Test - public void testNestedResult() { - ResultBuilder r = new ResultBuilder(); - Result res = r.parse("file:src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml", new Query("?query=a")); - assertNull(res.hits().getError()); - assertEquals(3, res.hits().size()); - assertEquals("ABCDEFGHIJKLMNOPQRSTUVWXYZ", res.hits().get(0).getField("guid").toString()); - HitGroup g1 = (HitGroup) res.hits().get(1); - HitGroup g2 = (HitGroup) res.hits().get(2); - assertEquals(15, g1.size()); - assertEquals("reward_for_thumb", g1.get(1).getField("id").toString()); - assertEquals(10, g2.size()); - HitGroup g3 = (HitGroup) g2.get(3); - assertEquals("badge", g3.types().iterator().next()); - assertEquals(2, g3.size()); - assertEquals("badge/Topic Explorer 5", g3.get(0).getField("name").toString()); - } - - @Test - public void testWeirdDocumentID() { - ResultBuilder r = new ResultBuilder(); - Result res = r.parse("file:src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml", new Query("?query=a")); - assertNull(res.hits().getError()); - assertEquals(3, res.hits().size()); - assertEquals(new URI("nalle"), res.hits().get(0).getId()); - assertEquals(new URI("tralle"), res.hits().get(1).getId()); - assertEquals(new URI("kalle"), res.hits().get(2).getId()); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java deleted file mode 100644 index 7fbe883bfb7..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa.test; - -import com.yahoo.component.chain.Chain; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.federation.vespa.VespaSearcher; -import com.yahoo.search.searchchain.Execution; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author bratseth - */ -public class VespaIntegrationTestCase { - - // TODO: Setup the answering vespa searcher from this test.... - @Test - public void testIt() { - if (System.currentTimeMillis() > 0) return; - Chain<Searcher> chain = new Chain<>(new VespaSearcher("test","example.yahoo.com",19010,"")); - Result result = new Execution(chain, Execution.Context.createContextStub()).search(new Query("?query=test")); - assertEquals(23, result.hits().size()); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java deleted file mode 100644 index cc00c6739fa..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa.test; - -import com.yahoo.prelude.query.*; -import com.yahoo.search.Query; -import com.yahoo.search.federation.vespa.VespaSearcher; -import com.yahoo.search.query.QueryTree; -import com.yahoo.search.query.parser.Parsable; -import com.yahoo.search.query.parser.Parser; -import com.yahoo.search.query.parser.ParserEnvironment; -import com.yahoo.search.query.parser.ParserFactory; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; -import org.apache.http.HttpEntity; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.net.URI; - -import static org.junit.Assert.assertEquals; - -/** - * Check query marshaling in VespaSearcher works. - * - * @author Steinar Knutsen - */ -public class VespaSearcherTestCase { - - private VespaSearcher searcher; - - @Before - public void setUp() { - searcher = new VespaSearcher("cache1","",0,""); - } - - @After - public void tearDown() { - searcher.deconstruct(); - } - - @Test - public void testMarshalQuery() { - RankItem root = new RankItem(); - QueryTree r = new QueryTree(root); - AndItem recall = new AndItem(); - PhraseItem usual = new PhraseItem(); - PhraseItem filterPhrase = new PhraseItem(new String[] {"bloody", "expensive"}); - WordItem filterWord = new WordItem("silly"); - - filterPhrase.setFilter(true); - filterWord.setFilter(true); - - root.addItem(recall); - usual.addItem(new WordItem("new")); - usual.addItem(new WordItem("york")); - recall.addItem(usual); - recall.addItem(new WordItem("shoes")); - root.addItem(new WordItem("nike")); - root.addItem(new WordItem("adidas")); - root.addItem(filterPhrase); - recall.addItem(filterWord); - - assertEquals("( \"new york\" AND shoes AND silly ) RANK nike RANK adidas RANK \"bloody expensive\"", searcher.marshalQuery(r)); - } - - @Test - public void testMarshalQuerySmallTree() { - RankItem root = new RankItem(); - QueryTree r = new QueryTree(root); - AndItem recall = new AndItem(); - PhraseItem usual = new PhraseItem(); - PhraseItem filterPhrase = new PhraseItem(new String[] {"bloody", "expensive"}); - WordItem filterWord = new WordItem("silly"); - - filterPhrase.setFilter(true); - filterWord.setFilter(true); - - root.addItem(recall); - usual.addItem(new WordItem("new")); - usual.addItem(new WordItem("york")); - recall.addItem(usual); - recall.addItem(new WordItem("shoes")); - root.addItem(filterPhrase); - recall.addItem(filterWord); - - assertEquals("( \"new york\" AND shoes AND silly ) RANK \"bloody expensive\"", searcher.marshalQuery(r)); - // TODO: Switch to this 2-way check rather than just 1-way and then also make this actually treat filter terms correctly - // assertMarshals(root) - } - - @Test - public void testWandMarshalling() { - WeakAndItem root = new WeakAndItem(); - root.setN(32); - root.addItem(new WordItem("a")); - root.addItem(new WordItem("b")); - root.addItem(new WordItem("c")); - assertMarshals(root); - } - - @Test - public void testWandMarshalling2() { - // AND (WAND(10) a!1 the!10) source:yahoonews - AndItem root = new AndItem(); - WeakAndItem wand = new WeakAndItem(10); - wand.addItem(newWeightedWordItem("a",1)); - wand.addItem(newWeightedWordItem("the",10)); - root.addItem(wand); - root.addItem(new WordItem("yahoonews","source")); - assertMarshals(root); - } - - private WordItem newWeightedWordItem(String word,int weight) { - WordItem wordItem=new WordItem(word); - wordItem.setWeight(weight); - return wordItem; - } - - private void assertMarshals(Item root) { - QueryTree r = new QueryTree(root); - String marshalledQuery=searcher.marshalQuery(r); - assertEquals("Marshalled form '" + marshalledQuery + "' recreates the original", - r,parseQuery(marshalledQuery,"")); - } - - private static Item parseQuery(String query, String filter) { - Parser parser = ParserFactory.newInstance(Query.Type.ADVANCED, new ParserEnvironment()); - return parser.parse(new Parsable().setQuery(query).setFilter(filter)); - } - - @Test - public void testSourceProviderProperties() throws Exception { - /* TODO: update test - Server httpServer = new Server(); - try { - SocketConnector listener = new SocketConnector(); - listener.setHost("0.0.0.0"); - httpServer.addConnector(listener); - httpServer.setHandler(new DummyHandler()); - httpServer.start(); - - int port=httpServer.getConnectors()[0].getLocalPort(); - - List<SourcesConfig.Source> sourcesConfig = new ArrayList<SourcesConfig.Source>(); - SourcesConfig.Source sourceConfig = new SourcesConfig.Source(); - sourceConfig.chain.setValue("news"); - sourceConfig.provider.setValue("news"); - sourceConfig.id.setValue("news"); - sourceConfig.timelimit.value = 10000; - sourcesConfig.add(sourceConfig); - FederationSearcher federator = - new FederationSearcher(ComponentId.createAnonymousComponentId(), - new ArrayList<SourcesConfig.Source>(sourcesConfig)); - SearchChain mainChain=new OrderedSearchChain(federator); - - SearchChainRegistry registry=new SearchChainRegistry(); - SearchChain sourceChain=new SearchChain(new ComponentId("news"),new VespaSearcher("test","localhost",port,"")); - registry.register(sourceChain); - Query query=new Query("?query=hans&hits=20&provider.news.a=a1&source.news.b=b1"); - Result result=new Execution(mainChain,registry).search(query); - assertNull(result.hits().getError()); - Hit testHit=result.hits().get("testHit"); - assertNotNull(testHit); - assertEquals("testValue",testHit.fields().get("testField")); - assertEquals("a1",testHit.fields().get("a")); - assertEquals("b1",testHit.fields().get("b")); - } - finally { - httpServer.stop(); - } - */ - } - - @Test - public void testVespaSearcher() { - VespaSearcher v=new VespaSearcherValidatingSubclass(); - new Execution(v, Execution.Context.createContextStub()).search(new Query(com.yahoo.search.test.QueryTestCase.httpEncode("?query=test&filter=myfilter"))); - } - - private class VespaSearcherValidatingSubclass extends VespaSearcher { - - public VespaSearcherValidatingSubclass() { - super("configId","host",80,"path"); - } - - @Override - protected HttpEntity getEntity(URI uri, Hit requestMeta, Query query) throws IOException { - assertEquals("http://host:80/path?query=test+RANK+myfilter&type=adv&offset=0&hits=10&presentation.format=xml",uri.toString()); - return super.getEntity(uri,requestMeta,query); - } - - } - - // used by the old testSourceProviderProperties() -// private class DummyHandler extends AbstractHandler { -// public void handle(String s, Request request, HttpServletRequest httpServletRequest, -// HttpServletResponse httpServletResponse) throws IOException, ServletException { -// -// try { -// Response httpResponse = httpServletResponse instanceof Response ? (Response) httpServletResponse : HttpConnection.getCurrentConnection().getResponse(); -// -// httpResponse.setStatus(HttpStatus.OK_200); -// httpResponse.setContentType("text/xml"); -// httpResponse.setCharacterEncoding("UTF-8"); -// Result r=new Result(new Query()); -// Hit testHit=new Hit("testHit"); -// testHit.setField("uri","testHit"); // That this is necessary is quite unfortunate... -// testHit.setField("testField","testValue"); -// // Write back all incoming properties: -// for (Object e : httpServletRequest.getParameterMap().entrySet()) { -// Map.Entry entry=(Map.Entry)e; -// testHit.setField(entry.getKey().toString(),getFirstValue(entry.getValue())); -// } -// -// r.hits().add(testHit); -// -// //StringWriter sw=new StringWriter(); -// //r.render(sw); -// //System.out.println(sw.toString()); -// -// SearchRendererAdaptor.callRender(httpResponse.getWriter(), r); -// httpResponse.complete(); -// } -// catch (Exception e) { -// System.out.println("WARNING: Could not respond to request: " + Exceptions.toMessageString(e)); -// e.printStackTrace(); -// } -// } -// -// private String getFirstValue(Object entry) { -// if (entry instanceof String[]) -// return ((String[])entry)[0].toString(); -// else -// return entry.toString(); -// } -// } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml deleted file mode 100644 index c17bbf8474d..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml +++ /dev/null @@ -1,23 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<result total-hit-count="3"> - <hit relevancy="75" source="test" type="summary"> - <field name="uri">nalle</field> - <field name="relevancy">75</field> - <field name="collapseId">0</field> - </hit> - <hit relevancy="73" source="test" type="summary test other"> - <field name="documentId">tralle</field> - <field name="relevancy">73</field> - <field name="collapseId">0</field> - <field name="category">test/stuff\tsome/other</field> - <field name="bsumtitle">dklf øæå sdf > & < -Ipsum, etc.</field> - </hit> - <hit relevancy="70" source="test" type="summary"> - <field name="DOCUMENTID">kalle</field> - <field name="relevancy">75</field> - <field name="collapseId">0</field> - <field name="annoying"><field>habla</field><hi>blbl</hi><br /><![CDATA[<>&fdlkkgj</field>]]>;lk<a b="1" c="2" /><x><y><z /></y></x></field> - </hit> -</result> diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml deleted file mode 100644 index 8b5ab710378..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml +++ /dev/null @@ -1,318 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<result total-hit-count="36"> -<hit type="user_reputation"> -<field name="guid">ABCDEFGHIJKLMNOPQRSTUVWXYZ</field> -<field name="level">zero</field> -<field name="points">0</field> -<field name="created">1287600988</field> -<field name="updated">1287600988</field> -</hit> -<group type="actions"> -<hit type="action"> -<field name="id">thumb</field> -<field name="created">1287600992</field> -<field name="updated">1287600992</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> -<hit type="action"> -<field name="id">reward_for_thumb</field> -<field name="created">1287600992</field> -<field name="updated">1287600992</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> -<hit type="action"> -<field name="id">undo_thumb</field> -<field name="created">1287600992</field> -<field name="updated">1287600992</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">buzz</field> -<field name="created">1287600989</field> -<field name="updated">1287600989</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">undo_reward_for_thumb</field> -<field name="created">1287600992</field> -<field name="updated">1287600992</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">vote</field> -<field name="created">1287600989</field> -<field name="updated">1287600989</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">report_abuse</field> -<field name="created">1287600992</field> -<field name="updated">1287600992</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">reward_for_vote</field> -<field name="created">1287600989</field> -<field name="updated">1287600989</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">signup</field> -<field name="created">1287600993</field> -<field name="updated">1287600993</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">registered</field> -<field name="created">1287600989</field> -<field name="updated">1287600989</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">get_points</field> -<field name="created">1287600989</field> -<field name="updated">1287600989</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">contrib_SignedUp</field> -<field name="created">1287600993</field> -<field name="updated">1287600993</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">contrib_AgreedToTos</field> -<field name="created">1287600993</field> -<field name="updated">1287600993</field> -<field name="points">500</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">Create Feature</field> -<field name="created"/> -<field name="updated"/> -<field name="points">0</field> -<field name="level"/> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">add_theme</field> -<field name="created"/> -<field name="updated"/> -<field name="points">0</field> -<field name="level"/> -<field name="isEnabled">1</field> -</hit> -</group> - -<group type="awards"> - -<group type="badge"> - -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/First Feature</field> -<field name="description">You’ve created your First Feature!</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/1stfeature.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> - -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> - -<group type="badge"> - -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/25th Feature</field> -<field name="description">You’ve created your 25th Feature!</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/25thfeature.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> - -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> - -<group type="badge"> - -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/50th Feature</field> -<field name="description">You’ve created your 50th Feature!</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/10thfeature.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> - -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> - -<group type="badge"> - -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/Topic Explorer 5</field> -<field name="description">You’ve added a Feature to your 5th Topic Page!</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/5thtopic.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> - -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> - -<group type="badge"> - -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/Topic Explorer 15</field> -<field name="description">You’ve added a Feature to your 15th Topic Page!</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/15thtopic.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> - -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> - -<group type="badge"> - -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/Topic Explorer 30</field> -<field name="description">You’ve added a Feature to your 30th Topic Page!</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/30thtopic.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> - -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> - -<group type="badge"> - -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/Pollster</field> -<field name="description">You’ve created your 5th Poll Feature.</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/pollster.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> -<group type="badge"> -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/Reporter</field> -<field name="description">You’ve created your 5th Article Feature.</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/newsreporter.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> -<group type="badge"> -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/Paparazzi</field> -<field name="description">You’ve created your 5th Image Feature.</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/paparazzi.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> -<group type="badge"> -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/Video Reporter</field> -<field name="description">You’ve created your 5th Video Feature.</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/director.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> -</group> -</result> diff --git a/container-search/src/test/java/com/yahoo/search/federation/ysm/.gitignore b/container-search/src/test/java/com/yahoo/search/federation/ysm/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/ysm/.gitignore +++ /dev/null diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java index 60ada5124a0..326e37ede38 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java @@ -203,7 +203,6 @@ public class GroupingParserBenchmarkTest { "all(group(artist) max(2) each(each(output(summary()))))", "all(group(artist) max(2) each(each(output(summary(simple)))))", "all(group(artist) max(5) each(output(count()) each(output(summary()))))", - "all(group(ymum()))", "all(group(strlen(attr)))", "all(group(normalizesubject(attr)))", "all(group(strcat(attr, attr2)))", diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java index fcf1c3bcdd0..5d311361edf 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java @@ -150,7 +150,6 @@ public class GroupingParserTestCase { "xor", "xorbit", "y", - "ymum", "zcurve"); for (String image : images) { assertParse("all(group(" + image + "))", "all(group(" + image + "))"); @@ -512,7 +511,6 @@ public class GroupingParserTestCase { assertParse("all(group(artist) max(2) each(each(output(summary()))))"); assertParse("all(group(artist) max(2) each(each(output(summary(simple)))))"); assertParse("all(group(artist) max(5) each(output(count()) each(output(summary()))))"); - assertParse("all(group(ymum()))"); assertParse("all(group(strlen(attr)))"); assertParse("all(group(normalizesubject(attr)))"); assertParse("all(group(strcat(attr, attr2)))"); diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java index c64c4d624f2..f61299bd317 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java @@ -160,7 +160,6 @@ public class RequestBuilderTestCase { assertLayout("all(group(time.year(a)) each(output(count())))", "[[{ TimeStamp, result = [Count] }]]"); assertLayout("all(group(xor(a,b)) each(output(count())))", "[[{ Xor, result = [Count] }]]"); assertLayout("all(group(xorbit(a,1)) each(output(count())))", "[[{ XorBit, result = [Count] }]]"); - assertLayout("all(group(ymum()) each(output(count())))", "[[{ GetYMUMChecksum, result = [Count] }]]"); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java index fa398efd293..9617e0ceb25 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java @@ -232,7 +232,7 @@ public class JSONSearchHandlerTestCase { public void testNormalResultExplicitDefaultRenderingFullRendererName1() throws Exception { JSONObject json = new JSONObject(); json.put("query", "abc"); - json.put("format", "DefaultRenderer"); + json.put("format", "XmlRenderer"); assertXmlResult(json, driver); } @@ -244,22 +244,6 @@ public class JSONSearchHandlerTestCase { assertJsonResult(json, driver); } - @Test - public void testResultLegacyTiledFormat() throws Exception { - JSONObject json = new JSONObject(); - json.put("query", "abc"); - json.put("format", "tiled"); - assertTiledResult(json, driver); - } - - @Test - public void testResultLegacyPageFormat() throws Exception { - JSONObject json = new JSONObject(); - json.put("query", "abc"); - json.put("format", "page"); - assertPageResult(json, driver); - } - private static final String xmlResult = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + "<result total-hit-count=\"0\">\n" + @@ -284,21 +268,6 @@ public class JSONSearchHandlerTestCase { } - private static final String tiledResult = - "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + - "<result version=\"1.0\">\n" + - "\n" + - " <hit relevance=\"1.0\">\n" + - " <id>testHit</id>\n" + - " <uri>testHit</uri>\n" + - " </hit>\n" + - "\n" + - "</result>\n"; - - private void assertTiledResult(JSONObject json, RequestHandlerTestDriver driver) { - assertOkResult(driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE), tiledResult); - } - private static final String pageResult = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + "<page version=\"1.0\">\n" + @@ -312,10 +281,6 @@ public class JSONSearchHandlerTestCase { "\n" + "</page>\n"; - private void assertPageResult(JSONObject json, RequestHandlerTestDriver driver) { - assertOkResult(driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE), pageResult); - } - private void assertOkResult(RequestHandlerTestDriver.MockResponseHandler response, String expected) { assertEquals(expected, response.readAll()); assertEquals(200, response.getStatus()); @@ -508,7 +473,6 @@ public class JSONSearchHandlerTestCase { assertEquals("Should have same mapping for properties", map, propertyMap); } - @Test public void testContentTypeParsing() throws Exception { JSONObject json = new JSONObject(); diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java index 5ef13eba2ed..20b18ba6723 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java @@ -15,7 +15,7 @@ import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.handler.HttpSearchResponse; import com.yahoo.search.handler.SearchHandler; -import com.yahoo.search.rendering.DefaultRenderer; +import com.yahoo.search.rendering.XmlRenderer; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; @@ -105,12 +105,12 @@ public class SearchHandlerTestCase { } @Test - public void testFailing() throws Exception { + public void testFailing() { assertTrue(driver.sendRequest("http://localhost?query=test&searchChain=classLoadingError").readAll().contains("NoClassDefFoundError")); } @Test - public synchronized void testPluginError() throws Exception { + public synchronized void testPluginError() { assertTrue(driver.sendRequest("http://localhost?query=test&searchChain=exceptionInPlugin").readAll().contains("NullPointerException")); } @@ -158,11 +158,11 @@ public class SearchHandlerTestCase { Result r = new Result(q); r.hits().addError(ErrorMessage.createUnspecifiedError("bamse")); r.hits().add(new Hit("http://localhost/dummy", 0.5)); - HttpSearchResponse s = new HttpSearchResponse(200, r, q, new DefaultRenderer()); + HttpSearchResponse s = new HttpSearchResponse(200, r, q, new XmlRenderer()); assertEquals("text/xml", s.getContentType()); assertNull(s.getCoverage()); assertEquals("query 'dummy'", s.getParsedQuery()); - assertEquals(5000, s.getTiming().getTimeout()); + assertEquals(500, s.getTiming().getTimeout()); } @Test @@ -191,7 +191,7 @@ public class SearchHandlerTestCase { } } @Test - public void testInvalidQueryParamWithoutQueryProfile() throws Exception { + public void testInvalidQueryParamWithoutQueryProfile() { testInvalidQueryParam(driver); } private void testInvalidQueryParam(final RequestHandlerTestDriver testDriver) { @@ -213,45 +213,35 @@ public class SearchHandlerTestCase { } @Test - public void testNormalResultImplicitDefaultRendering() throws Exception { + public void testNormalResultImplicitDefaultRendering() { assertJsonResult("http://localhost?query=abc", driver); } @Test - public void testNormalResultExplicitDefaultRendering() throws Exception { + public void testNormalResultExplicitDefaultRendering() { assertJsonResult("http://localhost?query=abc&format=default", driver); } @Test - public void testNormalResultXmlAliasRendering() throws Exception { + public void testNormalResultXmlAliasRendering() { assertXmlResult("http://localhost?query=abc&format=xml", driver); } @Test - public void testNormalResultJsonAliasRendering() throws Exception { + public void testNormalResultJsonAliasRendering() { assertJsonResult("http://localhost?query=abc&format=json", driver); } @Test - public void testNormalResultExplicitDefaultRenderingFullRendererName1() throws Exception { - assertXmlResult("http://localhost?query=abc&format=DefaultRenderer", driver); + public void testNormalResultExplicitDefaultRenderingFullRendererName1() { + assertXmlResult("http://localhost?query=abc&format=XmlRenderer", driver); } @Test - public void testNormalResultExplicitDefaultRenderingFullRendererName2() throws Exception { + public void testNormalResultExplicitDefaultRenderingFullRendererName2() { assertJsonResult("http://localhost?query=abc&format=JsonRenderer", driver); } - @Test - public void testResultLegacyTiledFormat() throws Exception { - assertTiledResult("http://localhost?query=abc&format=tiled", driver); - } - - @Test - public void testResultLegacyPageFormat() throws Exception { - assertPageResult("http://localhost?query=abc&format=page", driver); - } - private static final String xmlResult = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + "<result total-hit-count=\"0\">\n" + @@ -260,10 +250,12 @@ public class SearchHandlerTestCase { " <field name=\"uri\">testHit</field>\n" + " </hit>\n" + "</result>\n"; - private void assertXmlResult(String request, RequestHandlerTestDriver driver) throws Exception { + + private void assertXmlResult(String request, RequestHandlerTestDriver driver) { assertOkResult(driver.sendRequest(request), xmlResult); } - private void assertXmlResult(RequestHandlerTestDriver driver) throws Exception { + + private void assertXmlResult(RequestHandlerTestDriver driver) { assertXmlResult("http://localhost?query=abc", driver); } @@ -272,38 +264,9 @@ public class SearchHandlerTestCase { + "\"children\":[" + "{\"id\":\"testHit\",\"relevance\":1.0,\"fields\":{\"uri\":\"testHit\"}}" + "]}}"; - private void assertJsonResult(String request, RequestHandlerTestDriver driver) throws Exception { - assertOkResult(driver.sendRequest(request), jsonResult); - } - - private static final String tiledResult = - "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + - "<result version=\"1.0\">\n" + - "\n" + - " <hit relevance=\"1.0\">\n" + - " <id>testHit</id>\n" + - " <uri>testHit</uri>\n" + - " </hit>\n" + - "\n" + - "</result>\n"; - private void assertTiledResult(String request, RequestHandlerTestDriver driver) throws Exception { - assertOkResult(driver.sendRequest(request), tiledResult); - } - private static final String pageResult = - "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + - "<page version=\"1.0\">\n" + - "\n" + - " <content>\n" + - " <hit relevance=\"1.0\">\n" + - " <id>testHit</id>\n" + - " <uri>testHit</uri>\n" + - " </hit>\n" + - " </content>\n" + - "\n" + - "</page>\n"; - private void assertPageResult(String request, RequestHandlerTestDriver driver) throws Exception { - assertOkResult(driver.sendRequest(request), pageResult); + private void assertJsonResult(String request, RequestHandlerTestDriver driver) { + assertOkResult(driver.sendRequest(request), jsonResult); } private void assertOkResult(RequestHandlerTestDriver.MockResponseHandler response, String expected) { diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java index 999a6d32ac6..569f219f635 100644 --- a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java @@ -3,9 +3,6 @@ package com.yahoo.search.pagetemplates.engine.test; import com.google.common.util.concurrent.ListenableFuture; import com.yahoo.io.IOUtils; -import com.yahoo.prelude.templates.TiledTemplateSet; -import com.yahoo.prelude.templates.UserTemplate; -import com.yahoo.prelude.templates.test.TilingTestCase; import com.yahoo.search.Result; import com.yahoo.search.pagetemplates.PageTemplate; import com.yahoo.search.pagetemplates.config.PageTemplateXMLReader; diff --git a/container-search/src/test/java/com/yahoo/search/query/SoftTimeoutTestCase.java b/container-search/src/test/java/com/yahoo/search/query/SoftTimeoutTestCase.java index 83ef955a6d9..5026250ab91 100644 --- a/container-search/src/test/java/com/yahoo/search/query/SoftTimeoutTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/SoftTimeoutTestCase.java @@ -13,14 +13,14 @@ public class SoftTimeoutTestCase { @Test public void testDefaultsInQuery() { Query query=new Query("?query=test"); - assertNull(query.getRanking().getSoftTimeout().getEnable()); + assertTrue(query.getRanking().getSoftTimeout().getEnable()); assertNull(query.getRanking().getSoftTimeout().getFactor()); assertNull(query.getRanking().getSoftTimeout().getTailcost()); } @Test public void testQueryOverride() { - Query query=new Query("?query=test&ranking.softtimeout.enable&ranking.softtimeout.factor=0.7&ranking.softtimeout.tailcost=0.3"); + Query query=new Query("?query=test&ranking.softtimeout.factor=0.7&ranking.softtimeout.tailcost=0.3"); assertTrue(query.getRanking().getSoftTimeout().getEnable()); assertEquals(Double.valueOf(0.7), query.getRanking().getSoftTimeout().getFactor()); assertEquals(Double.valueOf(0.3), query.getRanking().getSoftTimeout().getTailcost()); @@ -30,6 +30,14 @@ public class SoftTimeoutTestCase { assertEquals("0.3", query.getRanking().getProperties().get("vespa.softtimeout.tailcost").get(0)); } + @Test + public void testDisable() { + Query query=new Query("?query=test&ranking.softtimeout.enable=false"); + assertFalse(query.getRanking().getSoftTimeout().getEnable()); + query.prepare(); + assertTrue(query.getRanking().getProperties().isEmpty()); + } + private void verifyException(String key, String value) { try { new Query("?query=test&ranking.softtimeout."+key+"="+value); diff --git a/container-search/src/test/java/com/yahoo/search/query/test/ModelTestCase.java b/container-search/src/test/java/com/yahoo/search/query/test/ModelTestCase.java index 1794f8989b4..276ff107769 100644 --- a/container-search/src/test/java/com/yahoo/search/query/test/ModelTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/test/ModelTestCase.java @@ -41,7 +41,7 @@ public class ModelTestCase { @Test public void testCopyParameters() { - Query q1 = new Query("?query=test1&filter=test2&defidx=content&default-index=lala&encoding=iso8859-1"); + Query q1 = new Query("?query=test1&filter=test2&default-index=content&default-index=lala&encoding=iso8859-1"); Query q2 = q1.clone(); Model r1 = q1.getModel(); Model r2 = q2.getModel(); diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/LegacyCombinatorTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/LegacyCombinatorTestCase.java deleted file mode 100644 index 4796937eb65..00000000000 --- a/container-search/src/test/java/com/yahoo/search/querytransform/LegacyCombinatorTestCase.java +++ /dev/null @@ -1,263 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.querytransform; - -import java.util.HashSet; -import java.util.Iterator; -import java.util.Set; - -import com.yahoo.container.protect.Error; -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.AndItem; -import com.yahoo.prelude.query.WordItem; -import com.yahoo.search.Query; -import com.yahoo.search.Searcher; -import com.yahoo.search.searchchain.Execution; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * Unit testing of the searcher com.yahoo.search.querytransform.LegacyCombinator. - * - * @author Steinar Knutsen - */ -public class LegacyCombinatorTestCase { - - Searcher searcher; - - @Before - public void setUp() throws Exception { - searcher = new LegacyCombinator(); - } - - @Test - public void testStraightForwardSearch() { - Query q = new Query("?query=a&query.juhu=b"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("AND a b", q.getModel().getQueryTree().toString()); - q = new Query("?query=a&query.juhu=b&defidx.juhu=juhu.22[gnuff]"); - e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("AND a juhu.22[gnuff]:b", q.getModel().getQueryTree().toString()); - q = new Query("?query=a&query.juhu="); - e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("a", q.getModel().getQueryTree().toString()); - q = new Query("?query=a+c&query.juhu=b"); - e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("AND a c b", q.getModel().getQueryTree().toString()); - } - - @Test - public void testNoBaseQuery() { - Query q = new Query("?query.juhu=b"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("b", q.getModel().getQueryTree().toString()); - } - - @Test - public void testIncompatibleNewAndOldQuery() { - Query q = new Query("?query.juhu=b&defidx.juhu=a&query.juhu.defidx=c"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("NULL", q.getModel().getQueryTree().toString()); - assertTrue("No expected error found.", q.errors().size() > 0); - assertEquals("Did not get invalid query parameter error as expected.", - Error.INVALID_QUERY_PARAMETER.code, q.errors().get(0).getCode()); - } - - @Test - public void testNotCombinatorWithoutRoot() { - Query q = new Query("?query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=not"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("NULL", q.getModel().getQueryTree().toString()); - assertTrue("No expected error found.", q.errors().size() > 0); - System.out.println(q.errors()); - assertEquals("Did not get invalid query parameter error as expected.", - Error.INVALID_QUERY_PARAMETER.code, q.errors().get(0).getCode()); - } - - @Test - public void testRankCombinator() { - Query q = new Query("?query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=rank"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("nalle:b", q.getModel().getQueryTree().toString()); - } - - @Test - public void testRankAndNot() { - Query q = new Query("?query.yahoo=2&query.yahoo.defidx=1&query.yahoo.operator=not&query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=rank"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("+nalle:b -1:2", q.getModel().getQueryTree().toString()); - } - - @Test - public void testReqAndRankAndNot() { - Query q = new Query("?query.yahoo=2&query.yahoo.defidx=1&query.yahoo.operator=not&query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=rank&query.bamse=z&query.bamse.defidx=y"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("+(RANK y:z nalle:b) -1:2", q.getModel().getQueryTree().toString()); - } - - @Test - public void testReqAndRank() { - Query q = new Query("?query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=rank&query.bamse=z&query.bamse.defidx=y"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("RANK y:z nalle:b", q.getModel().getQueryTree().toString()); - } - - @Test - public void testReqAndNot() { - Query q = new Query("?query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=not&query.bamse=z&query.bamse.defidx=y"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("+y:z -nalle:b", q.getModel().getQueryTree().toString()); - } - - @Test - public void testNewAndOld() { - Query q = new Query("?query.juhu=b&defidx.juhu=nalle&query.bamse=z&query.bamse.defidx=y"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - Set<StringPair> nastierItems = new HashSet<>(); - nastierItems.add(new StringPair("nalle", "b")); - nastierItems.add(new StringPair("y", "z")); - e.search(q); - AndItem root = (AndItem) q.getModel().getQueryTree().getRoot(); - Iterator<?> iterator = root.getItemIterator(); - while (iterator.hasNext()) { - WordItem word = (WordItem) iterator.next(); - StringPair asPair = new StringPair(word.getIndexName(), word.stringValue()); - if (nastierItems.contains(asPair)) { - nastierItems.remove(asPair); - } else { - assertFalse("Got unexpected item in query tree: (" - + word.getIndexName() + ", " + word.stringValue() + ")", - true); - } - } - assertEquals("Not all expected items found in query.", 0, nastierItems.size()); - } - - @Test - public void testReqAndNotWithQuerySyntaxAll() { - Query q = new Query("?query.juhu=b+c&query.juhu.defidx=nalle&query.juhu.operator=not&query.juhu.type=any&query.bamse=z&query.bamse.defidx=y"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("+y:z -(OR nalle:b nalle:c)", q.getModel().getQueryTree().toString()); - } - - @Test - public void testDefaultIndexWithoutQuery() { - Query q = new Query("?defidx.juhu=b"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("NULL", q.getModel().getQueryTree().toString()); - q = new Query("?query=a&defidx.juhu=b"); - e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("a", q.getModel().getQueryTree().toString()); - } - - private static class StringPair { - - public final String index; - public final String value; - - StringPair(String index, String value) { - super(); - this.index = index; - this.value = value; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((index == null) ? 0 : index.hashCode()); - result = prime * result + ((value == null) ? 0 : value.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - final StringPair other = (StringPair) obj; - if (index == null) { - if (other.index != null) - return false; - } else if (!index.equals(other.index)) - return false; - if (value == null) { - if (other.value != null) - return false; - } else if (!value.equals(other.value)) - return false; - return true; - } - - } - - @Test - public void testMultiPart() { - Query q = new Query("?query=a&query.juhu=b&query.nalle=c"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - Set<String> items = new HashSet<>(); - items.add("a"); - items.add("b"); - items.add("c"); - e.search(q); - // OK, the problem here is we have no way of knowing whether nalle or - // juhu was added first, since we have passed through HashMap instances - // inside the implementation - - AndItem root = (AndItem) q.getModel().getQueryTree().getRoot(); - Iterator<?> iterator = root.getItemIterator(); - while (iterator.hasNext()) { - WordItem word = (WordItem) iterator.next(); - if (items.contains(word.stringValue())) { - items.remove(word.stringValue()); - } else { - assertFalse("Got unexpected item in query tree: " + word.stringValue(), true); - } - } - assertEquals("Not all expected items found in query.", 0, items.size()); - - Set<StringPair> nastierItems = new HashSet<>(); - nastierItems.add(new StringPair("", "a")); - nastierItems.add(new StringPair("juhu.22[gnuff]", "b")); - nastierItems.add(new StringPair("gnuff[8].name(\"tralala\")", "c")); - q = new Query("?query=a&query.juhu=b&defidx.juhu=juhu.22[gnuff]&query.nalle=c&defidx.nalle=gnuff[8].name(%22tralala%22)"); - e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - root = (AndItem) q.getModel().getQueryTree().getRoot(); - iterator = root.getItemIterator(); - while (iterator.hasNext()) { - WordItem word = (WordItem) iterator.next(); - StringPair asPair = new StringPair(word.getIndexName(), word.stringValue()); - if (nastierItems.contains(asPair)) { - nastierItems.remove(asPair); - } else { - assertFalse("Got unexpected item in query tree: (" - + word.getIndexName() + ", " + word.stringValue() + ")", - true); - } - } - assertEquals("Not all expected items found in query.", 0, nastierItems.size()); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/test/QueryCombinatorTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/test/QueryCombinatorTestCase.java deleted file mode 100644 index 995c1cc7a72..00000000000 --- a/container-search/src/test/java/com/yahoo/search/querytransform/test/QueryCombinatorTestCase.java +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.querytransform.test; - -import java.util.HashSet; -import java.util.Iterator; -import java.util.Set; - -import com.yahoo.component.ComponentId; -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.AndItem; -import com.yahoo.prelude.query.WordItem; -import com.yahoo.search.Query; -import com.yahoo.search.Searcher; -import com.yahoo.search.querytransform.QueryCombinator; -import com.yahoo.search.searchchain.Execution; - -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; - -/** - * Unit testing of the searcher com.yahoo.search.querytransform.QueryCombinator. - * - * @author Steinar Knutsen - */ -public class QueryCombinatorTestCase { - - Searcher searcher; - - @Before - @SuppressWarnings("deprecation") - public void setUp() throws Exception { - searcher = new QueryCombinator(new ComponentId("combinationTest")); - } - - @Test - public void testStraightForwardSearch() { - Query q = new Query("?query=a&query.juhu=b"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("AND a b", q.getModel().getQueryTree().toString()); - q = new Query("?query=a&query.juhu=b&defidx.juhu=juhu.22[gnuff]"); - e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("AND a juhu.22[gnuff]:b", q.getModel().getQueryTree().toString()); - q = new Query("?query=a&query.juhu="); - e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("a", q.getModel().getQueryTree().toString()); - q = new Query("?query=a+c&query.juhu=b"); - e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("AND a c b", q.getModel().getQueryTree().toString()); - } - - @Test - public void testNoBaseQuery() { - Query q = new Query("?query.juhu=b"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("b", q.getModel().getQueryTree().toString()); - } - - @Test - public void testDefaultIndexWithoutQuery() { - Query q = new Query("?defidx.juhu=b"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("NULL", q.getModel().getQueryTree().toString()); - q = new Query("?query=a&defidx.juhu=b"); - e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - assertEquals("a", q.getModel().getQueryTree().toString()); - } - - private static class StringPair { - - public final String index; - public final String value; - - StringPair(String index, String value) { - super(); - this.index = index; - this.value = value; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((index == null) ? 0 : index.hashCode()); - result = prime * result + ((value == null) ? 0 : value.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - final StringPair other = (StringPair) obj; - if (index == null) { - if (other.index != null) - return false; - } else if (!index.equals(other.index)) - return false; - if (value == null) { - if (other.value != null) - return false; - } else if (!value.equals(other.value)) - return false; - return true; - } - - } - - @Test - public void testMultiPart() { - Query q = new Query("?query=a&query.juhu=b&query.nalle=c"); - Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - Set<String> items = new HashSet<>(); - items.add("a"); - items.add("b"); - items.add("c"); - e.search(q); - // OK, the problem here is we have no way of knowing whether nalle or - // juhu was added first, since we have passed through HashMap instances - // inside the implementation - - AndItem root = (AndItem) q.getModel().getQueryTree().getRoot(); - Iterator<?> iterator = root.getItemIterator(); - while (iterator.hasNext()) { - WordItem word = (WordItem) iterator.next(); - if (items.contains(word.stringValue())) { - items.remove(word.stringValue()); - } else { - assertFalse("Got unexpected item in query tree: " + word.stringValue(), true); - } - } - assertEquals("Not all expected items found in query.", 0, items.size()); - - Set<StringPair> nastierItems = new HashSet<>(); - nastierItems.add(new StringPair("", "a")); - nastierItems.add(new StringPair("juhu.22[gnuff]", "b")); - nastierItems.add(new StringPair("gnuff[8].name(\"tralala\")", "c")); - q = new Query("?query=a&query.juhu=b&defidx.juhu=juhu.22[gnuff]&query.nalle=c&defidx.nalle=gnuff[8].name(%22tralala%22)"); - e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); - e.search(q); - root = (AndItem) q.getModel().getQueryTree().getRoot(); - iterator = root.getItemIterator(); - while (iterator.hasNext()) { - WordItem word = (WordItem) iterator.next(); - StringPair asPair = new StringPair(word.getIndexName(), word.stringValue()); - if (nastierItems.contains(asPair)) { - nastierItems.remove(asPair); - } else { - assertFalse("Got unexpected item in query tree: (" - + word.getIndexName() + ", " + word.stringValue() + ")", - true); - } - } - assertEquals("Not all expected items found in query.", 0, nastierItems.size()); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index 5b3b5ca6d73..10a3b695f64 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -1,14 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.rendering; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.ListenableFuture; +import com.yahoo.component.ComponentId; import com.yahoo.component.chain.Chain; +import com.yahoo.container.QrSearchersConfig; import com.yahoo.data.access.simple.Value; import com.yahoo.data.access.slime.SlimeAdapter; import com.yahoo.document.DataType; @@ -19,8 +18,13 @@ import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.Struct; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.predicate.Predicate; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.prelude.SearchDefinition; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.hitfield.JSONString; +import com.yahoo.prelude.searcher.JuniperSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -38,6 +42,7 @@ import com.yahoo.search.result.NanNumber; import com.yahoo.search.result.Relevance; import com.yahoo.search.result.StructuredData; import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; import com.yahoo.search.statistics.ElapsedTimeTestCase; import com.yahoo.search.statistics.ElapsedTimeTestCase.CreativeTimeSource; import com.yahoo.search.statistics.ElapsedTimeTestCase.UselessSearcher; @@ -51,32 +56,31 @@ import com.yahoo.yolean.trace.TraceNode; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; -import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.mockito.Mockito; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.times; /** * Functional testing of {@link JsonRenderer}. * * @author Steinar Knutsen + * @author bratseth */ public class JsonRendererTestCase { - JsonRenderer originalRenderer; - JsonRenderer renderer; + private JsonRenderer originalRenderer; + private JsonRenderer renderer; public JsonRendererTestCase() { originalRenderer = new JsonRenderer(); @@ -84,23 +88,11 @@ public class JsonRendererTestCase { @Before public void setUp() throws Exception { - // Do the same dance as in production + // Use the shared renderer as a prototype object, as specified in the API contract renderer = (JsonRenderer) originalRenderer.clone(); renderer.init(); } - @After - public void tearDown() throws Exception { - renderer = null; - } - - private static final class Thingie { - @Override - public String toString() { - return "thingie"; - } - } - @Test public void testDocumentId() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" @@ -130,14 +122,6 @@ public class JsonRendererTestCase { assertEqualJson(expected, summary); } - private Result newEmptyResult(String[] args) { - return new Result(new Query("/?" + String.join("&", args))); - } - - private Result newEmptyResult() { - return newEmptyResult(new String[] {"query=a"}); - } - @Test public void testDataTypes() throws IOException, InterruptedException, ExecutionException, JSONException { String expected = "{\n" @@ -188,7 +172,7 @@ public class JsonRendererTestCase { @Test - public final void testTracing() throws IOException, InterruptedException, ExecutionException { + public void testTracing() throws IOException, InterruptedException, ExecutionException { // which clearly shows a trace child is created once too often... String expected = "{\n" + " \"root\": {\n" @@ -243,7 +227,7 @@ public class JsonRendererTestCase { } @Test - public final void testEmptyTracing() throws IOException, InterruptedException, ExecutionException { + public void testEmptyTracing() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"fields\": {\n" @@ -272,7 +256,7 @@ public class JsonRendererTestCase { @SuppressWarnings("unchecked") @Test - public final void testTracingWithEmptySubtree() throws IOException, InterruptedException, ExecutionException { + public void testTracingWithEmptySubtree() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"fields\": {\n" @@ -346,15 +330,8 @@ public class JsonRendererTestCase { assertEqualJson(expected, summary); } - private void subExecution(Execution execution, String color, int traceLevel) { - Execution e2 = new Execution(new Chain<Searcher>(), execution.context()); - Query subQuery = new Query("/?query=b&tracelevel=" + traceLevel); - e2.search(subQuery); - subQuery.trace(color, 1); - } - @Test - public final void testTracingOfNodesWithBothChildrenAndData() throws IOException, InterruptedException, ExecutionException { + public void testTracingOfNodesWithBothChildrenAndData() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"fields\": {\n" @@ -400,7 +377,7 @@ public class JsonRendererTestCase { @Test - public final void testTracingOfNodesWithBothChildrenAndDataAndEmptySubnode() throws IOException, InterruptedException, ExecutionException { + public void testTracingOfNodesWithBothChildrenAndDataAndEmptySubnode() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"fields\": {\n" @@ -441,7 +418,7 @@ public class JsonRendererTestCase { } @Test - public final void testTracingOfNestedNodesWithDataAndSubnodes() throws IOException, InterruptedException, ExecutionException { + public void testTracingOfNestedNodesWithDataAndSubnodes() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"fields\": {\n" @@ -490,7 +467,7 @@ public class JsonRendererTestCase { @Test - public final void test() throws IOException, InterruptedException, ExecutionException { + public void test() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"children\": [\n" @@ -498,8 +475,7 @@ public class JsonRendererTestCase { + " \"children\": [\n" + " {\n" + " \"fields\": {\n" - + " \"c\": \"d\",\n" - + " \"uri\": \"http://localhost/1\"\n" + + " \"c\": \"d\"\n" + " },\n" + " \"id\": \"http://localhost/1\",\n" + " \"relevance\": 0.9,\n" @@ -523,8 +499,7 @@ public class JsonRendererTestCase { + " },\n" + " {\n" + " \"fields\": {\n" - + " \"b\": \"foo\",\n" - + " \"uri\": \"http://localhost/\"\n" + + " \"b\": \"foo\"\n" + " },\n" + " \"id\": \"http://localhost/\",\n" + " \"relevance\": 0.95,\n" @@ -555,7 +530,7 @@ public class JsonRendererTestCase { + " \"relevance\": 1.0\n" + " }\n" + "}"; - Query q = new Query("/?query=a&tracelevel=5&reportCoverage=true"); + Query q = new Query("/?query=a&tracelevel=5"); Execution execution = new Execution(Execution.Context.createContextStub()); Result r = new Result(q); r.setCoverage(new Coverage(500, 500,1,1)); @@ -603,7 +578,7 @@ public class JsonRendererTestCase { + " \"relevance\": 1.0\n" + " }\n" + "}"; - Query q = new Query("/?query=a&tracelevel=5&reportCoverage=true"); + Query q = new Query("/?query=a&tracelevel=5"); Execution execution = new Execution(Execution.Context.createContextStub()); Result r = new Result(q); r.setCoverage(new Coverage(500, 600).setDegradedReason(5)); @@ -725,7 +700,7 @@ public class JsonRendererTestCase { + " \"relevance\": 1.0\n" + " }\n" + "}\n"; - Query q = new Query("/?query=a&tracelevel=5&reportCoverage=true"); + Query q = new Query("/?query=a&tracelevel=5"); Result r = new Result(q); Throwable t = new Throwable(); StackTraceElement[] stack = new StackTraceElement[1]; @@ -885,7 +860,7 @@ public class JsonRendererTestCase { }); GroupList gl = new GroupList("customer"); Group g = new Group(new DoubleBucketId(1.0, 2.0), new Relevance(1.0)); - g.setField("something()", Integer.valueOf(7)); + g.setField("something()", 7); gl.add(g); rg.add(gl); r.hits().add(rg); @@ -958,7 +933,7 @@ public class JsonRendererTestCase { } @Test - public final void testFieldValueInHit() throws IOException, InterruptedException, ExecutionException, JSONException { + public void testFieldValueInHit() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"children\": [\n" @@ -993,7 +968,7 @@ public class JsonRendererTestCase { } @Test - public final void testHiddenFields() throws IOException, InterruptedException, ExecutionException, JSONException { + public void testHiddenFields() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"children\": [\n" @@ -1017,17 +992,8 @@ public class JsonRendererTestCase { assertEqualJson(expected, summary); } - private Hit createHitWithOnlyHiddenFields() { - Hit h = new Hit("hiddenFields"); - h.setField("NaN", NanNumber.NaN); - h.setField("emptyString", ""); - h.setField("emptyStringFieldValue", new StringFieldValue("")); - h.setField("$vespaImplementationDetail", "Hello, World!"); - return h; - } - @Test - public final void testDebugRendering() throws IOException, InterruptedException, ExecutionException, JSONException { + public void testDebugRendering() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"children\": [\n" @@ -1058,7 +1024,7 @@ public class JsonRendererTestCase { } @Test - public final void testTimingRendering() throws InterruptedException, ExecutionException, JsonParseException, JsonMappingException, IOException { + public void testTimingRendering() throws InterruptedException, ExecutionException, IOException { String expected = "{" + " \"root\": {" + " \"fields\": {" @@ -1093,7 +1059,7 @@ public class JsonRendererTestCase { } @Test - public final void testJsonCallback() throws IOException, InterruptedException, ExecutionException, JSONException { + public void testJsonCallback() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"children\": [\n" @@ -1131,7 +1097,7 @@ public class JsonRendererTestCase { } @Test - public final void testMapInField() throws IOException, InterruptedException, ExecutionException, JSONException { + public void testMapInField() throws IOException, InterruptedException, ExecutionException { String expected = "{\n" + " \"root\": {\n" + " \"children\": [\n" @@ -1187,6 +1153,54 @@ public class JsonRendererTestCase { + "}"; assertEquals("Duplicate key \"duplicate\"", validateJSON(json)); } + + @Test + public void testDynamicSummary() throws Exception { + String content = "\uFFF9Feeding\uFFFAfeed\uFFFB \u001F\uFFF9documents\uFFFAdocument\uFFFB\u001F into Vespa \uFFF9is\uFFFAbe\u001Eincrement of a set of \u001F\uFFF9documents\uFFFAdocument\uFFFB\u001F fed into Vespa \uFFF9is\u001Efloat in XML when \u001Fdocument\u001F attribute \uFFF9is\uFFFAbe\uFFFB int\u001E"; + Result result = createResult("one", content, true); + + String summary = render(result); + + String expected = + "{ \n" + + " \"root\":{ " + + " \"id\":\"toplevel\"," + + " \"relevance\":1.0," + + " \"fields\":{ " + + " \"totalCount\":0" + + " }," + + " \"children\":[ " + + " { " + + " \"id\":\"http://abc.html/\"," + + " \"relevance\":1.0," + + " \"fields\":{ " + + " \"sddocname\":\"one\",\n" + + " \"dynteaser\":\"Feeding <hi>documents</hi> into Vespa is<sep />increment of a set of <hi>documents</hi> fed into Vespa <sep />float in XML when <hi>document</hi> attribute is int<sep />\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + "}\n"; + assertEqualJson(expected, summary); + } + + private Result newEmptyResult(String[] args) { + return new Result(new Query("/?" + String.join("&", args))); + } + + private Result newEmptyResult() { + return newEmptyResult(new String[] {"query=a"}); + } + + private Hit createHitWithOnlyHiddenFields() { + Hit h = new Hit("hiddenFields"); + h.setField("NaN", NanNumber.NaN); + h.setField("emptyString", ""); + h.setField("emptyStringFieldValue", new StringFieldValue("")); + h.setField("$vespaImplementationDetail", "Hello, World!"); + return h; + } + private String render(Result r) throws InterruptedException, ExecutionException { Execution execution = new Execution(Execution.Context.createContextStub()); return render(execution, r); @@ -1209,6 +1223,7 @@ public class JsonRendererTestCase { assertEquals("", validateJSON(expected)); assertEquals("", validateJSON(generated)); } + private String validateJSON(String presumablyValidJson) { try { new JSONObject(presumablyValidJson); @@ -1218,4 +1233,76 @@ public class JsonRendererTestCase { } } + private static final class Thingie { + @Override + public String toString() { + return "thingie"; + } + } + + private Result createResult(String sdName, String content, boolean bolding) { + Chain<Searcher> chain = createSearchChain(sdName, content); + Query query = new Query("?query=12"); + if ( ! bolding) + query = new Query("?query=12&bolding=false"); + Execution execution = createExecution(chain); + Result result = execution.search(query); + execution.fill(result); + return result; + } + + /** + * Creates a search chain which always returns a result with one hit containing information given in this + * + * @param sdName the search definition type of the returned hit + * @param content the content of the "dynteaser" field of the returned hit + */ + private Chain<Searcher> createSearchChain(String sdName, String content) { + JuniperSearcher searcher = new JuniperSearcher(new ComponentId("test"), + new QrSearchersConfig(new QrSearchersConfig.Builder())); + + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + addResult(new Query("?query=12"), sdName, content, docsource); + addResult(new Query("?query=12&bolding=false"), sdName, content, docsource); + return new Chain<>(searcher, docsource); + } + + private void addResult(Query query, String sdName, String content, DocumentSourceSearcher docsource) { + Result r = new Result(query); + FastHit hit = new FastHit(); + hit.setId("http://abc.html"); + hit.setRelevance(new Relevance(1)); + hit.setField(Hit.SDDOCNAME_FIELD, sdName); + hit.setField("dynteaser", content); + r.hits().add(hit); + docsource.addResult(query, r); + } + + private Execution createExecution(Chain<Searcher> chain) { + Map<String, List<String>> clusters = new LinkedHashMap<>(); + Map<String, SearchDefinition> searchDefs = new LinkedHashMap<>(); + searchDefs.put("one", createSearchDefinitionOne()); + SearchDefinition union = new SearchDefinition("union"); + IndexModel indexModel = new IndexModel(clusters, searchDefs, union); + return new Execution(chain, Execution.Context.createContextStub(new IndexFacts(indexModel))); + } + + private SearchDefinition createSearchDefinitionOne() { + SearchDefinition one = new SearchDefinition("one"); + + Index dynteaser = new Index("dynteaser"); + dynteaser.setDynamicSummary(true); + one.addIndex(dynteaser); + + Index bigteaser = new Index("bigteaser"); + dynteaser.setHighlightSummary(true); + one.addIndex(bigteaser); + + Index otherteaser = new Index("otherteaser"); + otherteaser.setDynamicSummary(true); + one.addIndex(otherteaser); + + return one; + } + } diff --git a/container-search/src/test/java/com/yahoo/search/rendering/SyncDefaultRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/SyncDefaultRendererTestCase.java index 1d2187376a4..acb69670b14 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/SyncDefaultRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/SyncDefaultRendererTestCase.java @@ -58,8 +58,8 @@ public class SyncDefaultRendererTestCase { @SuppressWarnings("deprecation") @Test - public final void testRenderWriterResult() throws InterruptedException, ExecutionException { - Query q = new Query("/?query=a&tracelevel=5&reportCoverage=true"); + public void testRenderWriterResult() throws InterruptedException, ExecutionException { + Query q = new Query("/?query=a&tracelevel=5"); q.getPresentation().setTiming(true); Result r = new Result(q); r.setCoverage(new Coverage(500, 1)); @@ -96,7 +96,7 @@ public class SyncDefaultRendererTestCase { assertTrue(f.get()); String summary = Utf8.toString(bs.toByteArray()); // TODO figure out a reasonably strict and reasonably flexible way to test - assertTrue(summary.length() > 1000); + assertTrue(summary.length() > 900); } } diff --git a/container-search/src/test/java/com/yahoo/search/rendering/XMLRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/XMLRendererTestCase.java index c95019d37d6..184a8c1aa43 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/XMLRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/XMLRendererTestCase.java @@ -4,13 +4,21 @@ package com.yahoo.search.rendering; import static org.junit.Assert.*; import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; - -import com.yahoo.search.handler.SearchHandler; -import org.junit.After; -import org.junit.Before; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.yahoo.component.ComponentId; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.prelude.SearchDefinition; +import com.yahoo.prelude.searcher.JuniperSearcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.Relevance; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; import org.junit.Test; import com.google.common.util.concurrent.ListenableFuture; @@ -25,50 +33,112 @@ import com.yahoo.search.result.HitGroup; import com.yahoo.search.statistics.ElapsedTimeTestCase; import com.yahoo.search.statistics.TimeTracker; import com.yahoo.search.statistics.ElapsedTimeTestCase.CreativeTimeSource; -import com.yahoo.search.statistics.ElapsedTimeTestCase.UselessSearcher; import com.yahoo.text.Utf8; /** * Test the XML renderer * * @author Steinar Knutsen + * @author bratseth */ public class XMLRendererTestCase { - DefaultRenderer d; - - @Before - public void setUp() throws Exception { - d = new DefaultRenderer(); - d.init(); + @Test + public void testGetEncoding() { + XmlRenderer renderer = new XmlRenderer(); + renderer.init(); + assertEquals("utf-8", renderer.getEncoding()); } - @After - public void tearDown() throws Exception { + @Test + public void testGetMimeType() { + XmlRenderer renderer = new XmlRenderer(); + renderer.init(); + assertEquals("text/xml", renderer.getMimeType()); } @Test - public final void testGetEncoding() { - assertEquals("utf-8", d.getEncoding()); + public void testXmlRendering() throws Exception { + Query q = new Query("/?query=a"); + + Result result = new Result(q); + result.setCoverage(new Coverage(500, 1)); + + FastHit h = new FastHit("http://localhost/", .95); + h.setField("$a", "Hello, world."); + h.setField("b", "foo"); + result.hits().add(h); + + HitGroup g = new HitGroup("usual"); + h = new FastHit("http://localhost/1", .90); + h.setField("c", "d"); + g.add(h); + result.hits().add(g); + + HitGroup gg = new HitGroup("type grouphit"); + gg.types().add("grouphit"); + gg.setField("e", "f"); + result.hits().add(gg); + result.hits().addError(ErrorMessage.createInternalServerError("message")); + + String summary = render(result); + + String expected = + "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<result total-hit-count=\"0\" coverage-docs=\"500\" coverage-nodes=\"1\" coverage-full=\"false\" coverage=\"0\" results-full=\"0\" results=\"1\">\n" + + " <error code=\"18\">Internal server error.</error>\n" + + " <errordetails>\n" + + " <error error=\"Internal server error.\" code=\"18\">message</error>\n" + + " </errordetails>\n" + + " <group relevancy=\"1.0\">\n" + + " <hit type=\"summary\" relevancy=\"0.9\">\n" + + " <field name=\"relevancy\">0.9</field>\n" + + " <field name=\"c\">d</field>\n" + + " </hit>\n" + + " </group>\n" + + " <hit type=\"grouphit\" relevancy=\"1.0\">\n" + + " <id>type grouphit</id>\n" + + " </hit>\n" + + " <hit type=\"summary\" relevancy=\"0.95\">\n" + + " <field name=\"relevancy\">0.95</field>\n" + + " <field name=\"b\">foo</field>\n" + + " </hit>\n" + + "</result>\n"; + + assertEquals(expected, summary); } @Test - public final void testGetMimeType() { - assertEquals("text/xml", d.getMimeType()); + public void testXmlRenderingOfDynamicSummary() throws Exception { + String content = "\uFFF9Feeding\uFFFAfeed\uFFFB \u001F\uFFF9documents\uFFFAdocument\uFFFB\u001F into Vespa \uFFF9is\uFFFAbe\u001Eincrement of a set of \u001F\uFFF9documents\uFFFAdocument\uFFFB\u001F fed into Vespa \uFFF9is\u001Efloat in XML when \u001Fdocument\u001F attribute \uFFF9is\uFFFAbe\uFFFB int\u001E"; + Result result = createResult("one", content, true); + + String summary = render(result); + + String expected = + "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<result total-hit-count=\"0\">\n" + + " <hit relevancy=\"1.0\">\n" + + " <field name=\"relevancy\">1.0</field>\n" + + " <field name=\"sddocname\">one</field>\n" + + " <field name=\"dynteaser\">Feeding <hi>documents</hi> into Vespa is<sep />increment of a set of <hi>documents</hi> fed into Vespa <sep />float in XML when <hi>document</hi> attribute is int<sep /></field>\n" + + " </hit>\n" + + "</result>\n"; + assertEquals(expected, summary); } @Test - public final void testImplicitDefaultRender() throws Exception { - Query q = new Query("/?query=a&tracelevel=5&reportCoverage=true"); + public void testXmlRenderingWithTimeTracking() throws Exception { + Query q = new Query("/?query=a&tracelevel=5"); q.getPresentation().setTiming(true); - Result r = new Result(q); - r.setCoverage(new Coverage(500, 1)); - - TimeTracker t = new TimeTracker(new Chain<Searcher>( - new UselessSearcher("first"), new UselessSearcher("second"), - new UselessSearcher("third"))); - ElapsedTimeTestCase.doInjectTimeSource(t, new CreativeTimeSource( - new long[] { 1L, 2L, 3L, 4L, 5L, 6L, 7L })); + + Result result = new Result(q); + result.setCoverage(new Coverage(500, 1)); + + TimeTracker t = new TimeTracker(new Chain<Searcher>(new NoopSearcher("first"), + new NoopSearcher("second"), + new NoopSearcher("third"))); + ElapsedTimeTestCase.doInjectTimeSource(t, new CreativeTimeSource(new long[] { 1L, 2L, 3L, 4L, 5L, 6L, 7L })); t.sampleSearch(0, true); t.sampleSearch(1, true); t.sampleSearch(2, true); @@ -76,47 +146,103 @@ public class XMLRendererTestCase { t.sampleSearchReturn(2, true, null); t.sampleSearchReturn(1, true, null); t.sampleSearchReturn(0, true, null); - r.getElapsedTime().add(t); - FastHit h = new FastHit("http://localhost/", .95); - h.setField("$a", "Hello, world."); - h.setField("b", "foo"); - r.hits().add(h); - HitGroup g = new HitGroup("usual"); - h = new FastHit("http://localhost/1", .90); - h.setField("c", "d"); - g.add(h); - r.hits().add(g); - HitGroup gg = new HitGroup("type grouphit"); - gg.types().add("grouphit"); - gg.setField("e", "f"); - r.hits().add(gg); - r.hits().addError(ErrorMessage.createInternalServerError("boom")); + result.getElapsedTime().add(t); + String summary = render(result); + + assertEquals("<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n<result total-hit-count=\"0\"", + summary.substring(0, 67)); + assertTrue(summary.contains("querytime=")); + assertTrue(summary.contains("summaryfetchtime=")); + assertTrue(summary.contains("searchtime=")); + assertTrue(summary.contains("<meta type=\"context\">")); + } + + private String render(Result result) throws Exception { + XmlRenderer renderer = new XmlRenderer(); + renderer.init(); ByteArrayOutputStream bs = new ByteArrayOutputStream(); - ListenableFuture<Boolean> f = d.render(bs, r, null, null); + ListenableFuture<Boolean> f = renderer.render(bs, result, null, null); assertTrue(f.get()); - String summary = Utf8.toString(bs.toByteArray()); + return Utf8.toString(bs.toByteArray()); + } - assertEquals("<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + - "<result total-hit-count=\"0\"", - summary.substring(0, 67) - ); - assertTrue(summary.contains("<meta type=\"context\">")); - assertTrue(summary.contains("<error code=\"18\">Internal server error.</error>")); - assertTrue(summary.contains("<hit type=\"grouphit\" relevancy=\"1.0\">")); - assertTrue(summary.contains("<hit type=\"summary\" relevancy=\"0.95\">")); - assertEquals(2, occurrences("<error ", summary)); - assertTrue(summary.length() > 1000); + private Result createResult(String sdName, String content, boolean bolding) { + Chain<Searcher> chain = createSearchChain(sdName, content); + Query query = new Query("?query=12"); + if ( ! bolding) + query = new Query("?query=12&bolding=false"); + Execution execution = createExecution(chain); + Result result = execution.search(query); + execution.fill(result); + return result; + } + + /** + * Creates a search chain which always returns a result with one hit containing information given in this + * + * @param sdName the search definition type of the returned hit + * @param content the content of the "dynteaser" field of the returned hit + */ + private Chain<Searcher> createSearchChain(String sdName, String content) { + JuniperSearcher searcher = new JuniperSearcher(new ComponentId("test"), + new QrSearchersConfig(new QrSearchersConfig.Builder())); + + DocumentSourceSearcher docsource = new DocumentSourceSearcher(); + addResult(new Query("?query=12"), sdName, content, docsource); + addResult(new Query("?query=12&bolding=false"), sdName, content, docsource); + return new Chain<>(searcher, docsource); + } + + private void addResult(Query query, String sdName, String content, DocumentSourceSearcher docsource) { + Result r = new Result(query); + FastHit hit = new FastHit(); + hit.setId("http://abc.html"); + hit.setRelevance(new Relevance(1)); + hit.setField(Hit.SDDOCNAME_FIELD, sdName); + hit.setField("dynteaser", content); + r.hits().add(hit); + docsource.addResult(query, r); + } + + private Execution createExecution(Chain<Searcher> chain) { + Map<String, List<String>> clusters = new LinkedHashMap<>(); + Map<String, SearchDefinition> searchDefs = new LinkedHashMap<>(); + searchDefs.put("one", createSearchDefinitionOne()); + SearchDefinition union = new SearchDefinition("union"); + IndexModel indexModel = new IndexModel(clusters, searchDefs, union); + return new Execution(chain, Execution.Context.createContextStub(new IndexFacts(indexModel))); } - private int occurrences(String fragment, String string) { - int occurrences = 0; - int cursor = 0; - while ( -1 != (cursor = string.indexOf(fragment, cursor))) { - occurrences++; - cursor += fragment.length(); + private SearchDefinition createSearchDefinitionOne() { + SearchDefinition one = new SearchDefinition("one"); + + Index dynteaser = new Index("dynteaser"); + dynteaser.setDynamicSummary(true); + one.addIndex(dynteaser); + + Index bigteaser = new Index("bigteaser"); + dynteaser.setHighlightSummary(true); + one.addIndex(bigteaser); + + Index otherteaser = new Index("otherteaser"); + otherteaser.setDynamicSummary(true); + one.addIndex(otherteaser); + + return one; + } + + public static class NoopSearcher extends Searcher { + + public NoopSearcher(String name) { + super(new ComponentId(name)); } - return occurrences; + + @Override + public Result search(Query query, Execution execution) { + return execution.search(query); + } + } } diff --git a/container-search/src/test/java/com/yahoo/search/result/PositionsDataTestCase.java b/container-search/src/test/java/com/yahoo/search/result/PositionsDataTestCase.java index c77ff2eca3c..91c8acbab12 100644 --- a/container-search/src/test/java/com/yahoo/search/result/PositionsDataTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/result/PositionsDataTestCase.java @@ -5,8 +5,6 @@ import com.yahoo.data.access.simple.Value; import static org.junit.Assert.*; -import org.junit.After; -import org.junit.Before; import org.junit.Test; /** @@ -15,41 +13,43 @@ import org.junit.Test; public class PositionsDataTestCase { @Test - public void testRendering() { - Value.ArrayValue arr = new Value.ArrayValue(); - Value.ObjectValue p1 = new Value.ObjectValue(); - p1.put("x", new Value.LongValue(-122057174)); - p1.put("y", new Value.LongValue(37374821)); - p1.put("latlong", new Value.StringValue("N37.374821;W122.057174")); - arr.add(p1); + public void testRenderingOfSinglePosition() { + Value.ObjectValue pos = createPosition(-122057174, 37374821, "N37.374821;W122.057174"); - PositionsData pd = new PositionsData(arr.inspect()); + PositionsData pd = new PositionsData(pos.inspect()); - String rendered = pd.toXML(); - String correct = "<position x=\"-122057174\" y=\"37374821\" latlong=\"N37.374821;W122.057174\" />"; - assertEquals(correct, rendered); + assertXml("<position x=\"-122057174\" y=\"37374821\" latlong=\"N37.374821;W122.057174\" />", pd); + assertJson("{\"x\":-122057174,\"y\":37374821,\"latlong\":\"N37.374821;W122.057174\"}", pd); + } + + @Test + public void testRenderingOfMultiplePositions() { + Value.ArrayValue arr = new Value.ArrayValue(); + arr.add(createPosition(-122057174, 37374821, "N37.374821;W122.057174")); + arr.add(createPosition(3, -7, "S0.000007;E0.000003")); - rendered = pd.toJson(); - correct = "[{\"x\":-122057174,\"y\":37374821,\"latlong\":\"N37.374821;W122.057174\"}]"; - assertEquals(correct, rendered); + PositionsData pd = new PositionsData(arr.inspect()); - Value.ObjectValue p2 = new Value.ObjectValue(); - p2.put("x", new Value.LongValue(3)); - p2.put("y", new Value.LongValue(-7)); - p2.put("latlong", new Value.StringValue("S0.000007;E0.000003")); - arr.add(p2); + assertXml("<position x=\"-122057174\" y=\"37374821\" latlong=\"N37.374821;W122.057174\" />" + + "<position x=\"3\" y=\"-7\" latlong=\"S0.000007;E0.000003\" />", pd); + assertJson("[{\"x\":-122057174,\"y\":37374821,\"latlong\":\"N37.374821;W122.057174\"}," + + "{\"x\":3,\"y\":-7,\"latlong\":\"S0.000007;E0.000003\"}]", pd); + } - pd = new PositionsData(arr.inspect()); + private Value.ObjectValue createPosition(long x, long y, String latlong) { + Value.ObjectValue result = new Value.ObjectValue(); + result.put("x", new Value.LongValue(x)); + result.put("y", new Value.LongValue(y)); + result.put("latlong", new Value.StringValue(latlong)); + return result; + } - rendered = pd.toXML(); - correct = "<position x=\"-122057174\" y=\"37374821\" latlong=\"N37.374821;W122.057174\" />" + - "<position x=\"3\" y=\"-7\" latlong=\"S0.000007;E0.000003\" />"; - assertEquals(correct, rendered); + private void assertXml(String expected, PositionsData pd) { + assertEquals(expected, pd.toXML()); + } - rendered = pd.toJson(); - correct = "[{\"x\":-122057174,\"y\":37374821,\"latlong\":\"N37.374821;W122.057174\"}," + - "{\"x\":3,\"y\":-7,\"latlong\":\"S0.000007;E0.000003\"}]"; - assertEquals(correct, rendered); + private void assertJson(String expected, PositionsData pd) { + assertEquals(expected, pd.toJson()); } } diff --git a/container-search/src/test/java/com/yahoo/search/result/TemplatingTestCase.java b/container-search/src/test/java/com/yahoo/search/result/TemplatingTestCase.java deleted file mode 100644 index 13d0c64a0c2..00000000000 --- a/container-search/src/test/java/com/yahoo/search/result/TemplatingTestCase.java +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.result; - -import static org.junit.Assert.*; - -import java.io.IOException; -import java.io.Writer; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; - -import com.yahoo.search.rendering.Renderer; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.google.common.base.Splitter; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.templates.UserTemplate; -import com.yahoo.prelude.templates.test.BoomTemplate; -import com.yahoo.search.Query; -import com.yahoo.search.Result; - -/** - * Control helper method for result rendering/result templates. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class TemplatingTestCase { - Result result; - - @Before - public void setUp() throws Exception { - Query q = new Query("/?query=a&presentation.format=nalle&offset=1&hits=5"); - result = new Result(q); - result.setTotalHitCount(1000L); - result.hits().add(new FastHit("http://localhost/1", .95)); - result.hits().add(new FastHit("http://localhost/2", .90)); - result.hits().add(new FastHit("http://localhost/3", .85)); - result.hits().add(new FastHit("http://localhost/4", .80)); - result.hits().add(new FastHit("http://localhost/5", .75)); - } - - @After - public void tearDown() throws Exception { - } - - @Test - public final void testGetFirstHitNo() { - assertEquals(2, result.getTemplating().getFirstHitNo()); - } - - @Test - public final void testGetNextFirstHitNo() { - assertEquals(7, result.getTemplating().getNextFirstHitNo()); - result.getQuery().setHits(6); - assertEquals(0, result.getTemplating().getNextFirstHitNo()); - } - - @Test - public final void testGetNextLastHitNo() { - assertEquals(11, result.getTemplating().getNextLastHitNo()); - result.getQuery().setHits(6); - assertEquals(0, result.getTemplating().getNextLastHitNo()); - } - - @Test - public final void testGetLastHitNo() { - assertEquals(6, result.getTemplating().getLastHitNo()); - } - - @Test - public final void testGetPrevFirstHitNo() { - assertEquals(1, result.getTemplating().getPrevFirstHitNo()); - } - - @Test - public final void testGetPrevLastHitNo() { - assertEquals(1, result.getTemplating().getPrevLastHitNo()); - } - - @Test - public final void testGetNextResultURL() { - String next = result.getTemplating().getNextResultURL(); - Set<String> expectedParameters = new HashSet<>(Arrays.asList(new String[] { - "hits=5", - "query=a", - "presentation.format=nalle", - "offset=6" - })); - Set<String> actualParameters = new HashSet<>(); - Splitter s = Splitter.on("&"); - for (String parameter : s.split(next.substring(next.indexOf('?') + 1))) { - actualParameters.add(parameter); - } - assertEquals(expectedParameters, actualParameters); - } - - @Test - public final void testGetPreviousResultURL() { - String previous = result.getTemplating().getPreviousResultURL(); - Set<String> expectedParameters = new HashSet<>(Arrays.asList(new String[] { - "hits=5", - "query=a", - "presentation.format=nalle", - "offset=0" - })); - Set<String> actualParameters = new HashSet<>(); - Splitter s = Splitter.on("&"); - for (String parameter : s.split(previous.substring(previous.indexOf('?') + 1))) { - actualParameters.add(parameter); - } - assertEquals(expectedParameters, actualParameters); - } - - @Test - public final void testGetCurrentResultURL() { - String previous = result.getTemplating().getCurrentResultURL(); - Set<String> expectedParameters = new HashSet<>(Arrays.asList(new String[] { - "hits=5", - "query=a", - "presentation.format=nalle", - "offset=1" - })); - Set<String> actualParameters = new HashSet<>(); - Splitter s = Splitter.on("&"); - for (String parameter : s.split(previous.substring(previous.indexOf('?') + 1))) { - actualParameters.add(parameter); - } - assertEquals(expectedParameters, actualParameters); - } - - @Test - public final void testGetTemplates() { - @SuppressWarnings({ "unchecked", "deprecation" }) - UserTemplate<Writer> t = result.getTemplating().getTemplates(); - assertEquals("default", t.getName()); - } - - @SuppressWarnings("deprecation") - @Test - public final void testSetTemplates() { - result.getTemplating().setTemplates(new BoomTemplate("gnuff", "text/plain", "ISO-8859-15")); - @SuppressWarnings("unchecked") - UserTemplate<Writer> t = result.getTemplating().getTemplates(); - assertEquals("gnuff", t.getName()); - } - - private static class TestRenderer extends Renderer { - - @Override - public void render(Writer writer, Result result) throws IOException { - } - - @Override - public String getEncoding() { - return null; - } - - @Override - public String getMimeType() { - return null; - } - } - - @SuppressWarnings("deprecation") - @Test - public final void testUsesDefaultTemplate() { - assertTrue(result.getTemplating().usesDefaultTemplate()); - result.getTemplating().setRenderer(new TestRenderer()); - assertFalse(result.getTemplating().usesDefaultTemplate()); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/result/test/ArrayOutputTestCase.java b/container-search/src/test/java/com/yahoo/search/result/test/ArrayOutputTestCase.java deleted file mode 100644 index 11d163373ac..00000000000 --- a/container-search/src/test/java/com/yahoo/search/result/test/ArrayOutputTestCase.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.result.test; - -import java.io.IOException; - -import com.yahoo.prelude.hitfield.XMLString; -import com.yahoo.prelude.templates.test.TilingTestCase; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.result.Hit; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author bratseth - */ -public class ArrayOutputTestCase { - - @Test - public void testArrayOutput() throws IOException { - Result r=new Result(new Query("?query=ignored")); - Hit hit=new Hit("test"); - hit.setField("phone",new XMLString("\n <item>408-555-1234</item>" + "\n <item>408-555-5678</item>\n ")); - r.hits().add(hit); - - String rendered = TilingTestCase.getRendered(r); - String[] lines= rendered.split("\n"); - assertEquals(" <field name=\"phone\">",lines[4]); - assertEquals(" <item>408-555-1234</item>",lines[5]); - assertEquals(" <item>408-555-5678</item>",lines[6]); - assertEquals(" </field>",lines[7]); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/statistics/ElapsedTimeTestCase.java b/container-search/src/test/java/com/yahoo/search/statistics/ElapsedTimeTestCase.java index 15937e77899..7491e970b23 100644 --- a/container-search/src/test/java/com/yahoo/search/statistics/ElapsedTimeTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/statistics/ElapsedTimeTestCase.java @@ -50,6 +50,7 @@ public class ElapsedTimeTestCase { } public static class UselessSearcher extends Searcher { + public UselessSearcher(String name) { super(new ComponentId(name)); } @@ -58,9 +59,11 @@ public class ElapsedTimeTestCase { public Result search(Query query, Execution execution) { return execution.search(query); } + } private static class AlmostUselessSearcher extends Searcher { + AlmostUselessSearcher(String name) { super(new ComponentId(name)); } @@ -73,9 +76,11 @@ public class ElapsedTimeTestCase { r.hits().add(h); return r; } + } private static class NoForwardSearcher extends Searcher { + @Override public Result search(Query query, Execution execution) { Result r = new Result(query); @@ -84,6 +89,7 @@ public class ElapsedTimeTestCase { r.hits().add(h); return r; } + } private class TestingSearcher extends Searcher { diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index ed80c0bf256..a5c46fc7887 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -684,34 +684,34 @@ public class QueryTestCase { @Test public void testThatSessionIdIsUniquePerQuery() { Query q = new Query(); - assertNull(q.getSessionId(false)); - assertNull(q.getSessionId(false)); - SessionId s1 = q.getSessionId(true); + assertNull(q.getSessionId()); + assertNull(q.getSessionId()); + SessionId s1 = q.getSessionId("node-0"); assertNotNull(s1); - SessionId s2 = q.getSessionId(true); + SessionId s2 = q.getSessionId("node-0"); assertNotSame(s1, s2); assertEquals(s1, s2); assertEquals(s1.toString(), s2.toString()); Query q2 = new Query(); - assertNotEquals(q.getSessionId(false), q2.getSessionId(true)); - assertNotEquals(q.getSessionId(false).toString(), q2.getSessionId(true).toString()); + assertNotEquals(q.getSessionId(), q2.getSessionId("node-0")); + assertNotEquals(q.getSessionId().toString(), q2.getSessionId("node-0").toString()); } @Test public void testThatCloneGetANewSessionId() { Query q = new Query(); - q.getSessionId(true); + q.getSessionId("node-0"); Query clonedQ = q.clone(); - assertNull(clonedQ.getSessionId(false)); - assertNotEquals(q.getSessionId(false), clonedQ.getSessionId(true)); + assertNull(clonedQ.getSessionId()); + assertNotEquals(q.getSessionId(), clonedQ.getSessionId("node-0")); } @Test public void testThatSessionIdIsUniquePerRankProfilePerQuery() { Query q = new Query(); - SessionId s1 = q.getSessionId(true); + SessionId s1 = q.getSessionId("node-0"); q.getRanking().setProfile("my-profile"); - SessionId s2 = q.getSessionId(false); + SessionId s2 = q.getSessionId(); assertNotEquals(s1, s2); } @@ -719,14 +719,14 @@ public class QueryTestCase { public void testThatSessionIdIsNotSharedIfCreatedAfterClone() { Query q = new Query(); Query q2 = q.clone(); - assertNull(q.getSessionId(false)); - assertNull(q2.getSessionId(false)); + assertNull(q.getSessionId()); + assertNull(q2.getSessionId()); - assertNotNull(q.getSessionId(true)); - assertNull(q2.getSessionId(false)); + assertNotNull(q.getSessionId("node-0")); + assertNull(q2.getSessionId()); - assertNotNull(q2.getSessionId(true)); - assertNotEquals(q.getSessionId(false), q2.getSessionId(false)); + assertNotNull(q2.getSessionId("node-0")); + assertNotEquals(q.getSessionId(), q2.getSessionId()); } @Test diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java index f75b78fd36f..d7fc53a6c00 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java @@ -100,7 +100,7 @@ public class VdsVisitorTestCase { private class QueryArguments { // General query parameters String query = "test"; - long timeout = 5; + double timeout = 0.5; int offset = 0; int hits = 10; int traceLevel = 0; @@ -114,7 +114,6 @@ public class VdsVisitorTestCase { String userId = null; String groupName = null; String selection = null; - boolean headersOnly = false; long from = 0; long to = 0; String loadTypeName = null; @@ -140,7 +139,6 @@ public class VdsVisitorTestCase { userId = "1234"; groupName = null; selection = null; - headersOnly = true; from = 123; to = 456; loadTypeName = "low"; @@ -155,7 +153,7 @@ public class VdsVisitorTestCase { private Query buildQuery(QueryArguments qa) throws Exception { StringBuilder queryString = new StringBuilder(); queryString.append("/?query=").append(qa.query); - if (qa.timeout != 5) { + if (qa.timeout != 0.5) { queryString.append("&timeout=").append(qa.timeout); } if (qa.offset != 0) { @@ -192,9 +190,6 @@ public class VdsVisitorTestCase { if (qa.selection != null) { queryString.append("&streaming.selection=").append(URLEncoder.encode(qa.selection, "UTF-8")); } - if (qa.headersOnly) { - queryString.append("&streaming.headersonly=").append(qa.headersOnly); - } if (qa.from != 0) { queryString.append("&streaming.fromtimestamp=").append(qa.from); } @@ -236,7 +231,6 @@ public class VdsVisitorTestCase { } else { assertEquals(docType + " and ( " + qa.selection + " )", params.getDocumentSelection()); } - assertEquals(qa.headersOnly, params.getVisitHeadersOnly()); assertEquals(qa.from, params.getFromTimestamp()); assertEquals(qa.to, params.getToTimestamp()); if (qa.loadTypeName != null && loadTypeSet.getNameMap().get(qa.loadTypeName) != null) { @@ -276,8 +270,8 @@ public class VdsVisitorTestCase { } // Verify parameters based only on query - assertEquals(qa.timeout*1000, params.getTimeoutMs()); - assertEquals(qa.timeout*1000, params.getSessionTimeoutMs()); + assertEquals(qa.timeout*1000, params.getTimeoutMs(),0.0000001); + assertEquals(qa.timeout*1000, params.getSessionTimeoutMs(), 0.0000001); assertEquals("searchvisitor", params.getVisitorLibrary()); assertEquals(Integer.MAX_VALUE, params.getMaxPending()); assertEquals(qa.traceLevel, params.getTraceLevel()); diff --git a/container-test-jars/bundle-with-provided-bundle/pom.xml b/container-test-jars/bundle-with-provided-bundle/pom.xml index e133ee955dd..d9d97ddd2b0 100644 --- a/container-test-jars/bundle-with-provided-bundle/pom.xml +++ b/container-test-jars/bundle-with-provided-bundle/pom.xml @@ -8,7 +8,7 @@ <parent> <groupId>com.yahoo.vespa.container-test-jars</groupId> <artifactId>container-test-jars</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>bundle-with-provided-bundle</artifactId> diff --git a/container-test-jars/jersey-resources/pom.xml b/container-test-jars/jersey-resources/pom.xml index 2b6761e6411..413f06784e8 100644 --- a/container-test-jars/jersey-resources/pom.xml +++ b/container-test-jars/jersey-resources/pom.xml @@ -8,7 +8,7 @@ <parent> <groupId>com.yahoo.vespa.container-test-jars</groupId> <artifactId>container-test-jars</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>jersey-resources</artifactId> diff --git a/container-test-jars/pom.xml b/container-test-jars/pom.xml index 804ca5eb879..979b98f9803 100644 --- a/container-test-jars/pom.xml +++ b/container-test-jars/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <groupId>com.yahoo.vespa.container-test-jars</groupId> <artifactId>container-test-jars</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>pom</packaging> <modules> <module>jersey-resources</module> diff --git a/container-test/pom.xml b/container-test/pom.xml index 845660f638b..49ac89be0e9 100644 --- a/container-test/pom.xml +++ b/container-test/pom.xml @@ -10,11 +10,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container-test</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>jar</packaging> <dependencies> <dependency> diff --git a/container/pom.xml b/container/pom.xml index 529f01e0a40..6007c066770 100644 --- a/container/pom.xml +++ b/container/pom.xml @@ -10,11 +10,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>container</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>jar</packaging> <dependencies> <dependency> diff --git a/controller-api/pom.xml b/controller-api/pom.xml index 7d06e2facbd..15098a04787 100644 --- a/controller-api/pom.xml +++ b/controller-api/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>controller-api</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> diff --git a/controller-server/pom.xml b/controller-server/pom.xml index 7e27a727bfd..23d2bfc33d4 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>controller-server</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> diff --git a/defaults/pom.xml b/defaults/pom.xml index dd924e65cfe..35d8c55f20e 100644 --- a/defaults/pom.xml +++ b/defaults/pom.xml @@ -7,13 +7,13 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <groupId>com.yahoo.vespa</groupId> <artifactId>defaults</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>defaults</name> <description> Library with the defaults for Vespa. diff --git a/dist/vespa.spec b/dist/vespa.spec index 6aeb099e8fb..b2d9499085f 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -67,7 +67,7 @@ BuildRequires: lz4-devel BuildRequires: libzstd-devel BuildRequires: zlib-devel BuildRequires: libicu-devel -BuildRequires: java-1.8.0-openjdk-devel +BuildRequires: java-11-openjdk-devel BuildRequires: openssl-devel BuildRequires: rpm-build BuildRequires: make @@ -132,7 +132,7 @@ Requires: llvm-libs >= 7.0.0 %define _extra_link_directory /opt/vespa-cppunit/lib%{?_vespa_llvm_link_directory:;%{_vespa_llvm_link_directory}}%{?_vespa_gtest_link_directory:;%{_vespa_gtest_link_directory}} %define _extra_include_directory /opt/vespa-cppunit/include%{?_vespa_llvm_include_directory:;%{_vespa_llvm_include_directory}}%{?_vespa_gtest_include_directory:;%{_vespa_gtest_include_directory}} %endif -Requires: java-1.8.0-openjdk +Requires: java-11-openjdk Requires: openssl Requires: vespa-cppunit >= 1.12.1-6 Requires(pre): shadow-utils @@ -155,14 +155,16 @@ source %{_devtoolset_enable} || true %if 0%{?_rhmaven35_enable:1} source %{_rhmaven35_enable} || true %endif +alternatives --set java java-11-openjdk.x86_64 +alternatives --set javac java-11-openjdk.x86_64 export FACTORY_VESPA_VERSION=%{version} sh bootstrap.sh java mvn --batch-mode -nsu -T 1 install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true cmake3 -DCMAKE_INSTALL_PREFIX=%{_prefix} \ - -DJAVA_HOME=/usr/lib/jvm/java-openjdk \ + -DJAVA_HOME=/usr/lib/jvm/java-11-openjdk \ -DEXTRA_LINK_DIRECTORY="%{_extra_link_directory}" \ -DEXTRA_INCLUDE_DIRECTORY="%{_extra_include_directory}" \ - -DCMAKE_INSTALL_RPATH="%{_prefix}/lib64%{?_extra_link_directory:;%{_extra_link_directory}};/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server" \ + -DCMAKE_INSTALL_RPATH="%{_prefix}/lib64%{?_extra_link_directory:;%{_extra_link_directory}};/usr/lib/jvm/jre-11-openjdk/lib" \ %{?_vespa_llvm_version:-DVESPA_LLVM_VERSION="%{_vespa_llvm_version}"} \ . diff --git a/docker-api/pom.xml b/docker-api/pom.xml index ae06378043b..ce36c4fd127 100644 --- a/docker-api/pom.xml +++ b/docker-api/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>docker-api</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> diff --git a/docproc/abi-spec.json b/docproc/abi-spec.json index bc76aefb958..65ca886efaf 100644 --- a/docproc/abi-spec.json +++ b/docproc/abi-spec.json @@ -11,7 +11,7 @@ "public abstract java.util.Map documentTypes()", "public abstract java.util.Map structTypes()", "public abstract java.util.Map annotationTypes()", - "public com.yahoo.document.Document getDocumentCopy(java.lang.String, com.yahoo.document.datatypes.StructuredFieldValue, com.yahoo.document.DocumentId)" + "public abstract com.yahoo.document.Document getDocumentCopy(java.lang.String, com.yahoo.document.datatypes.StructuredFieldValue, com.yahoo.document.DocumentId)" ], "fields": [] }, @@ -269,7 +269,6 @@ "methods": [ "public void <init>()", "public static com.yahoo.docproc.Processing of(com.yahoo.document.DocumentOperation)", - "public static com.yahoo.docproc.Processing fromDocumentOperation(com.yahoo.document.DocumentOperation)", "public void <init>(java.lang.String, com.yahoo.document.DocumentOperation, com.yahoo.docproc.CallStack)", "public static com.yahoo.docproc.Processing createProcessingFromDocumentOperations(java.lang.String, java.util.List, com.yahoo.docproc.CallStack)", "public com.yahoo.component.provider.ComponentRegistry getDocprocServiceRegistry()", @@ -312,7 +311,6 @@ ], "methods": [ "public void <init>()", - "public void process(com.yahoo.document.Document)", "public void process(com.yahoo.document.DocumentPut)", "public void process(com.yahoo.document.DocumentUpdate)", "public void process(com.yahoo.document.DocumentRemove)", diff --git a/docproc/pom.xml b/docproc/pom.xml index 6fe2458ce95..3def1a65a44 100644 --- a/docproc/pom.xml +++ b/docproc/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>docproc</artifactId> <packaging>jar</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>junit</groupId> diff --git a/docproc/src/main/java/com/yahoo/docproc/AbstractConcreteDocumentFactory.java b/docproc/src/main/java/com/yahoo/docproc/AbstractConcreteDocumentFactory.java index 36686caf7ac..8f771418959 100644 --- a/docproc/src/main/java/com/yahoo/docproc/AbstractConcreteDocumentFactory.java +++ b/docproc/src/main/java/com/yahoo/docproc/AbstractConcreteDocumentFactory.java @@ -28,18 +28,5 @@ public abstract class AbstractConcreteDocumentFactory extends com.yahoo.componen * * @return A concrete document instance */ - public com.yahoo.document.Document getDocumentCopy(java.lang.String type, com.yahoo.document.datatypes.StructuredFieldValue src, com.yahoo.document.DocumentId id) { - // Note: This method can't be abstract because it must work with older bundles where the ConcreteDocumentFactory may not implement it. - // It is overridden to not use reflection by newer bundles. - // The implementation here is not so good in bundles, since it instantiates the doc using reflection. - // TODO: for 6.0: make this method abstract and throw away the code below. - Class<? extends Document> concreteClass = documentTypes().get(type); - try { - Constructor<? extends Document> copyCon = concreteClass.getConstructor(StructuredFieldValue.class, DocumentId.class); - return copyCon.newInstance(src, id); - } catch (InvocationTargetException | NoSuchMethodException | InstantiationException | IllegalAccessException e) { - throw new RuntimeException(Exceptions.toMessageString(e), e); - } - } - + public abstract com.yahoo.document.Document getDocumentCopy(java.lang.String type, com.yahoo.document.datatypes.StructuredFieldValue src, com.yahoo.document.DocumentId id); } diff --git a/docproc/src/main/java/com/yahoo/docproc/Processing.java b/docproc/src/main/java/com/yahoo/docproc/Processing.java index eacbf6ef132..e157ad0b09b 100644 --- a/docproc/src/main/java/com/yahoo/docproc/Processing.java +++ b/docproc/src/main/java/com/yahoo/docproc/Processing.java @@ -61,16 +61,6 @@ public class Processing { return new Processing(documentOperation); } - /** - * Create a Processing from the given document operation - * @deprecated Use {@link #of(DocumentOperation)} instead - */ - @Deprecated - @SuppressWarnings("unused") - public static Processing fromDocumentOperation(DocumentOperation documentOperation) { - return Processing.of(documentOperation); - } - private Processing(DocumentOperation documentOperation) { this(); addDocumentOperation(documentOperation); diff --git a/docproc/src/main/java/com/yahoo/docproc/SimpleDocumentProcessor.java b/docproc/src/main/java/com/yahoo/docproc/SimpleDocumentProcessor.java index bcd1ee521bc..bb9ef8ff636 100644 --- a/docproc/src/main/java/com/yahoo/docproc/SimpleDocumentProcessor.java +++ b/docproc/src/main/java/com/yahoo/docproc/SimpleDocumentProcessor.java @@ -28,29 +28,16 @@ import com.yahoo.log.LogLevel; public class SimpleDocumentProcessor extends DocumentProcessor { /** - * Override this to process the Document inside a DocumentPut. - * @deprecated use process(DocumentPut) - * - * @param document the Document to process. - */ - @Deprecated - // TODO: Remove on Vespa 7 - public void process(Document document) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Ignored " + document); - } - } - - /** * Override this to process DocumentPuts. If this method is not overridden, the implementation in this class * will ignore DocumentPuts (passing them through un-processed). If processing of this DocumentPut fails, the * implementation must throw a {@link RuntimeException}. * * @param put the DocumentPut to process. */ - @SuppressWarnings("deprecation") public void process(DocumentPut put) { - process(put.getDocument()); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "Ignored " + put); + } } /** diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java index 7b6c96ffa4a..89f3782da52 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java @@ -7,8 +7,6 @@ import com.yahoo.documentapi.messagebus.loadtypes.LoadType; import com.yahoo.documentapi.messagebus.protocol.*; import com.yahoo.log.LogLevel; import com.yahoo.messagebus.Message; -import com.yahoo.vdslib.DocumentList; -import com.yahoo.vdslib.Entry; import java.util.logging.Logger; diff --git a/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocument.java b/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocument.java index ac9310784f2..b540f1c204d 100644 --- a/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocument.java +++ b/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocument.java @@ -290,18 +290,6 @@ public class ProxyDocument extends Document implements DocumentOperationWrapper } @Override - @SuppressWarnings("deprecation") - public void serializeHeader(Serializer target) throws SerializationException { - doc.serializeHeader(target); - } - - @Override - @SuppressWarnings("deprecation") - public void serializeBody(Serializer target) throws SerializationException { - doc.serializeBody(target); - } - - @Override public DocumentType getDataType() { return doc.getDataType(); } diff --git a/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocumentUpdate.java b/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocumentUpdate.java index 517f44cb983..51f9afa7a2e 100644 --- a/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocumentUpdate.java +++ b/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocumentUpdate.java @@ -11,7 +11,6 @@ import com.yahoo.document.serialization.DocumentUpdateWriter; import com.yahoo.document.update.FieldUpdate; import java.util.Collection; -import java.util.List; import java.util.Map; /** @@ -19,7 +18,6 @@ import java.util.Map; * * @author vegardh */ -// TODO Vespa 7 Remove all deprecated methods public class ProxyDocumentUpdate extends DocumentUpdate implements DocumentOperationWrapper { private DocumentUpdate docU; @@ -47,13 +45,6 @@ public class ProxyDocumentUpdate extends DocumentUpdate implements DocumentOpera } @Override - @Deprecated - @SuppressWarnings( "deprecation" ) - public FieldUpdate getFieldUpdate(int index) { - return docU.getFieldUpdate(index); - } - - @Override public FieldUpdate getFieldUpdate(String fieldName) { String mapped = fieldMap.get(fieldName); if (mapped==null) { @@ -64,12 +55,6 @@ public class ProxyDocumentUpdate extends DocumentUpdate implements DocumentOpera } @Override - @Deprecated - @SuppressWarnings( "deprecation" ) - public List<FieldUpdate> getFieldUpdates() { - return docU.getFieldUpdates(); - } - @Override public Collection<FieldUpdate> fieldUpdates() { return docU.fieldUpdates(); } diff --git a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java index e0ca630889d..375c0e0552a 100644 --- a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java +++ b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java @@ -16,8 +16,6 @@ import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.Reply; -import com.yahoo.vdslib.DocumentList; -import com.yahoo.vdslib.Entry; import org.junit.Test; import java.util.ArrayList; diff --git a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerForkTestCase.java b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerForkTestCase.java index e7f750dc8dd..2158901b009 100644 --- a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerForkTestCase.java +++ b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerForkTestCase.java @@ -12,8 +12,6 @@ import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.WriteDocumentReply; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.Reply; -import com.yahoo.vdslib.DocumentList; -import com.yahoo.vdslib.Entry; import org.junit.Test; import java.util.*; diff --git a/docproc/src/test/java/com/yahoo/docproc/util/docindoc.sd b/docproc/src/test/java/com/yahoo/docproc/util/docindoc.sd index 8b6acd71f47..b05f8d71c89 100644 --- a/docproc/src/test/java/com/yahoo/docproc/util/docindoc.sd +++ b/docproc/src/test/java/com/yahoo/docproc/util/docindoc.sd @@ -1,7 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. search docindoc { document docindoc { - field name type string { header } - field content type string { body } + field name type string { } + field content type string { } } } diff --git a/docproc/src/test/java/com/yahoo/docproc/util/outerdoc.sd b/docproc/src/test/java/com/yahoo/docproc/util/outerdoc.sd index 6fdbd402ffe..57a1bda323a 100644 --- a/docproc/src/test/java/com/yahoo/docproc/util/outerdoc.sd +++ b/docproc/src/test/java/com/yahoo/docproc/util/outerdoc.sd @@ -1,6 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. search outerdoc { document outerdoc { - field innerdocuments type array<docindoc> { body } + field innerdocuments type array<docindoc> { } } } diff --git a/docprocs/pom.xml b/docprocs/pom.xml index 16259a861c7..b7272de2253 100644 --- a/docprocs/pom.xml +++ b/docprocs/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>docprocs</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/document/abi-spec.json b/document/abi-spec.json index 79e0cdc34d0..a17bc298289 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -295,8 +295,6 @@ "public java.lang.Long getLastModified()", "public void setLastModified(java.lang.Long)", "public void onSerialize(com.yahoo.vespa.objects.Serializer)", - "public void serializeHeader(com.yahoo.vespa.objects.Serializer)", - "public void serializeBody(com.yahoo.vespa.objects.Serializer)", "public com.yahoo.document.DocumentType getDataType()", "public int getFieldCount()", "public void serialize(com.yahoo.document.serialization.DocumentWriter)", @@ -508,7 +506,6 @@ "public com.yahoo.document.DocumentType getDocumentType(java.lang.String)", "public final com.yahoo.document.Document createDocument(com.yahoo.io.GrowableByteBuffer)", "public com.yahoo.document.Document createDocument(com.yahoo.document.serialization.DocumentDeserializer)", - "public com.yahoo.document.Document createDocument(com.yahoo.io.GrowableByteBuffer, com.yahoo.io.GrowableByteBuffer)", "public java.util.Collection getDataTypes()", "public java.util.Map getDocumentTypes()", "public java.util.Iterator documentTypeIterator()", @@ -553,14 +550,10 @@ "public void setId(com.yahoo.document.DocumentId)", "public com.yahoo.document.DocumentUpdate applyTo(com.yahoo.document.Document)", "public com.yahoo.document.DocumentUpdate prune(com.yahoo.document.Document)", - "public java.util.List getFieldUpdates()", "public java.util.Collection fieldUpdates()", - "public java.util.List getFieldPathUpdates()", "public java.util.Collection fieldPathUpdates()", "public com.yahoo.document.DocumentType getDocumentType()", "public void setDocumentType(com.yahoo.document.DocumentType)", - "public com.yahoo.document.update.FieldUpdate getFieldUpdate(int)", - "public com.yahoo.document.update.FieldUpdate setFieldUpdate(int, com.yahoo.document.update.FieldUpdate)", "public com.yahoo.document.update.FieldUpdate getFieldUpdate(com.yahoo.document.Field)", "public void clearFieldUpdates()", "public com.yahoo.document.update.FieldUpdate getFieldUpdate(java.lang.String)", @@ -571,7 +564,6 @@ "public com.yahoo.document.DocumentUpdate addFieldUpdate(com.yahoo.document.update.FieldUpdate)", "public com.yahoo.document.DocumentUpdate addFieldPathUpdate(com.yahoo.document.fieldpathupdate.FieldPathUpdate)", "public void addAll(com.yahoo.document.DocumentUpdate)", - "public com.yahoo.document.update.FieldUpdate removeFieldUpdate(int)", "public com.yahoo.document.update.FieldUpdate removeFieldUpdate(com.yahoo.document.Field)", "public com.yahoo.document.update.FieldUpdate removeFieldUpdate(java.lang.String)", "public com.yahoo.document.DocumentType getType()", @@ -3227,7 +3219,7 @@ "public void setNewValues(com.yahoo.document.datatypes.Array)", "public com.yahoo.document.datatypes.Array getNewValues()", "public com.yahoo.document.datatypes.FieldPathIteratorHandler getIteratorHandler(com.yahoo.document.Document)", - "public void serialize(com.yahoo.document.serialization.VespaDocumentSerializerHead)", + "public void serialize(com.yahoo.document.serialization.VespaDocumentSerializer6)", "public boolean equals(java.lang.Object)", "public int hashCode()", "public java.lang.String toString()" @@ -3253,7 +3245,7 @@ "public void setRemoveIfZero(boolean)", "public void setCreateMissingPath(boolean)", "public boolean isArithmetic()", - "public void serialize(com.yahoo.document.serialization.VespaDocumentSerializerHead)", + "public void serialize(com.yahoo.document.serialization.VespaDocumentSerializer6)", "public boolean equals(java.lang.Object)", "public int hashCode()", "public java.lang.String toString()", @@ -3307,7 +3299,7 @@ "public com.yahoo.document.select.DocumentSelector getWhereClause()", "public java.lang.String getOriginalWhereClause()", "public void applyTo(com.yahoo.document.Document)", - "public void serialize(com.yahoo.document.serialization.VespaDocumentSerializerHead)", + "public void serialize(com.yahoo.document.serialization.VespaDocumentSerializer6)", "public static com.yahoo.document.fieldpathupdate.FieldPathUpdate create(com.yahoo.document.fieldpathupdate.FieldPathUpdate$Type, com.yahoo.document.DocumentType, com.yahoo.document.serialization.DocumentUpdateReader)", "public boolean equals(java.lang.Object)", "public int hashCode()", @@ -3608,211 +3600,6 @@ ], "fields": [] }, - "com.yahoo.document.select.BucketSelector": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.document.BucketIdFactory)", - "public com.yahoo.document.select.BucketSet getBucketList(java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.document.select.BucketSet": { - "superClass": "java.util.HashSet", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.document.BucketId)", - "public void <init>(com.yahoo.document.select.BucketSet)", - "public com.yahoo.document.select.BucketSet intersection(com.yahoo.document.select.BucketSet)", - "public com.yahoo.document.select.BucketSet union(com.yahoo.document.select.BucketSet)" - ], - "fields": [] - }, - "com.yahoo.document.select.Context": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.document.DocumentOperation)", - "public com.yahoo.document.DocumentOperation getDocumentOperation()", - "public void setDocumentOperation(com.yahoo.document.DocumentOperation)", - "public java.util.Map getVariables()", - "public void setVariables(java.util.Map)" - ], - "fields": [] - }, - "com.yahoo.document.select.DocumentSelector": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String)", - "public com.yahoo.document.select.Result accepts(com.yahoo.document.DocumentOperation)", - "public com.yahoo.document.select.Result accepts(com.yahoo.document.select.Context)", - "public com.yahoo.document.select.ResultList getMatchingResultList(com.yahoo.document.DocumentOperation)", - "public com.yahoo.document.select.ResultList getMatchingResultList(com.yahoo.document.select.Context)", - "public java.lang.String toString()", - "public com.yahoo.document.select.OrderingSpecification getOrdering(int)", - "public void visit(com.yahoo.document.select.Visitor)" - ], - "fields": [] - }, - "com.yahoo.document.select.NowCheckVisitor": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.document.select.Visitor" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public boolean requiresConversion()", - "public void visit(com.yahoo.document.select.rule.ArithmeticNode)", - "public void visit(com.yahoo.document.select.rule.AttributeNode)", - "public void visit(com.yahoo.document.select.rule.ComparisonNode)", - "public void visit(com.yahoo.document.select.rule.DocumentNode)", - "public void visit(com.yahoo.document.select.rule.EmbracedNode)", - "public void visit(com.yahoo.document.select.rule.IdNode)", - "public void visit(com.yahoo.document.select.rule.LiteralNode)", - "public void visit(com.yahoo.document.select.rule.LogicNode)", - "public void visit(com.yahoo.document.select.rule.NegationNode)", - "public void visit(com.yahoo.document.select.rule.NowNode)", - "public void visit(com.yahoo.document.select.rule.SearchColumnNode)", - "public void visit(com.yahoo.document.select.rule.VariableNode)" - ], - "fields": [] - }, - "com.yahoo.document.select.OrderingSpecification": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(int)", - "public void <init>(int, long, short, short)", - "public int getOrder()", - "public long getOrderingStart()", - "public short getWidthBits()", - "public short getDivisionBits()", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public java.lang.String toString()" - ], - "fields": [ - "public static int ASCENDING", - "public static int DESCENDING", - "public final int order", - "public final long orderingStart", - "public final short widthBits", - "public final short divisionBits" - ] - }, - "com.yahoo.document.select.Result": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.document.select.Result[] values()", - "public static com.yahoo.document.select.Result valueOf(java.lang.String)", - "public java.lang.String toString()", - "public static com.yahoo.document.select.Result invert(com.yahoo.document.select.Result)", - "public static com.yahoo.document.select.Result toResult(java.lang.Object)" - ], - "fields": [ - "public static final enum com.yahoo.document.select.Result TRUE", - "public static final enum com.yahoo.document.select.Result FALSE", - "public static final enum com.yahoo.document.select.Result INVALID" - ] - }, - "com.yahoo.document.select.ResultList$ResultPair": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public com.yahoo.document.datatypes.FieldPathIteratorHandler$VariableMap getVariables()", - "public com.yahoo.document.select.Result getResult()", - "public java.lang.String toString()" - ], - "fields": [] - }, - "com.yahoo.document.select.ResultList$VariableValue": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.document.datatypes.FieldPathIteratorHandler$VariableMap, java.lang.Object)", - "public com.yahoo.document.datatypes.FieldPathIteratorHandler$VariableMap getVariables()", - "public java.lang.Object getValue()", - "public java.lang.String toString()" - ], - "fields": [] - }, - "com.yahoo.document.select.ResultList": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.document.select.Result)", - "public void add(com.yahoo.document.datatypes.FieldPathIteratorHandler$VariableMap, com.yahoo.document.select.Result)", - "public java.util.List getResults()", - "public static com.yahoo.document.select.ResultList fromBoolean(boolean)", - "public com.yahoo.document.select.Result toResult()", - "public com.yahoo.document.select.ResultList combineAND(com.yahoo.document.select.ResultList)", - "public com.yahoo.document.select.ResultList combineOR(com.yahoo.document.select.ResultList)", - "public static com.yahoo.document.select.ResultList toResultList(java.lang.Object)", - "public java.lang.String toString()" - ], - "fields": [] - }, - "com.yahoo.document.select.Visitor": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void visit(com.yahoo.document.select.rule.ArithmeticNode)", - "public abstract void visit(com.yahoo.document.select.rule.AttributeNode)", - "public abstract void visit(com.yahoo.document.select.rule.ComparisonNode)", - "public abstract void visit(com.yahoo.document.select.rule.DocumentNode)", - "public abstract void visit(com.yahoo.document.select.rule.EmbracedNode)", - "public abstract void visit(com.yahoo.document.select.rule.IdNode)", - "public abstract void visit(com.yahoo.document.select.rule.LiteralNode)", - "public abstract void visit(com.yahoo.document.select.rule.LogicNode)", - "public abstract void visit(com.yahoo.document.select.rule.NegationNode)", - "public abstract void visit(com.yahoo.document.select.rule.NowNode)", - "public abstract void visit(com.yahoo.document.select.rule.SearchColumnNode)", - "public abstract void visit(com.yahoo.document.select.rule.VariableNode)" - ], - "fields": [] - }, "com.yahoo.document.select.convert.NowQueryExpression": { "superClass": "java.lang.Object", "interfaces": [], @@ -4598,8 +4385,8 @@ "methods": [ "public void <init>()", "public static com.yahoo.document.serialization.DocumentDeserializer createHead(com.yahoo.document.DocumentTypeManager, com.yahoo.io.GrowableByteBuffer)", - "public static com.yahoo.document.serialization.DocumentDeserializer create42(com.yahoo.document.DocumentTypeManager, com.yahoo.io.GrowableByteBuffer)", - "public static com.yahoo.document.serialization.DocumentDeserializer create42(com.yahoo.document.DocumentTypeManager, com.yahoo.io.GrowableByteBuffer, com.yahoo.io.GrowableByteBuffer)" + "public static com.yahoo.document.serialization.DocumentDeserializer create6(com.yahoo.document.DocumentTypeManager, com.yahoo.io.GrowableByteBuffer)", + "public static com.yahoo.document.serialization.DocumentDeserializer create42(com.yahoo.document.DocumentTypeManager, com.yahoo.io.GrowableByteBuffer)" ], "fields": [] }, @@ -4646,8 +4433,9 @@ "methods": [ "public void <init>()", "public static com.yahoo.document.serialization.DocumentSerializer createHead(com.yahoo.io.GrowableByteBuffer)", + "public static com.yahoo.document.serialization.DocumentSerializer create6(com.yahoo.io.GrowableByteBuffer)", + "public static com.yahoo.document.serialization.DocumentSerializer create6()", "public static com.yahoo.document.serialization.DocumentSerializer create42(com.yahoo.io.GrowableByteBuffer)", - "public static com.yahoo.document.serialization.DocumentSerializer create42(com.yahoo.io.GrowableByteBuffer, boolean)", "public static com.yahoo.document.serialization.DocumentSerializer create42()" ], "fields": [] @@ -4866,7 +4654,7 @@ "fields": [] }, "com.yahoo.document.serialization.VespaDocumentDeserializer42": { - "superClass": "com.yahoo.document.serialization.VespaDocumentSerializer42", + "superClass": "com.yahoo.vespa.objects.BufferSerializer", "interfaces": [ "com.yahoo.document.serialization.DocumentDeserializer" ], @@ -4913,16 +4701,62 @@ ], "fields": [] }, + "com.yahoo.document.serialization.VespaDocumentDeserializer6": { + "superClass": "com.yahoo.vespa.objects.BufferSerializer", + "interfaces": [ + "com.yahoo.document.serialization.DocumentDeserializer" + ], + "attributes": [ + "public" + ], + "methods": [ + "public final com.yahoo.document.DocumentTypeManager getDocumentTypeManager()", + "public void read(com.yahoo.document.Document)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.Document)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.FieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.Array)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.MapFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.CollectionFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.ByteFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.BoolFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.DoubleFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.FloatFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.IntegerFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.LongFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.Raw)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.PredicateFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.StringFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.TensorFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.ReferenceFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.Struct)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.StructuredFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.WeightedSet)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.annotation.AnnotationReference)", + "public void read(com.yahoo.document.DocumentUpdate)", + "public void read(com.yahoo.document.fieldpathupdate.FieldPathUpdate)", + "public void read(com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate)", + "public void read(com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate)", + "public void read(com.yahoo.document.fieldpathupdate.AddFieldPathUpdate)", + "public com.yahoo.document.update.ValueUpdate getValueUpdate(com.yahoo.document.DataType, com.yahoo.document.DataType)", + "public void read(com.yahoo.document.update.FieldUpdate)", + "public com.yahoo.document.DocumentId readDocumentId()", + "public com.yahoo.document.DocumentType readDocumentType()", + "public void read(com.yahoo.document.annotation.SpanTree)", + "public void read(com.yahoo.document.annotation.Annotation)", + "public void read(com.yahoo.document.annotation.Span)", + "public void read(com.yahoo.document.annotation.SpanList)", + "public void read(com.yahoo.document.annotation.AlternateSpanList)" + ], + "fields": [] + }, "com.yahoo.document.serialization.VespaDocumentDeserializerHead": { - "superClass": "com.yahoo.document.serialization.VespaDocumentDeserializer42", + "superClass": "com.yahoo.document.serialization.VespaDocumentDeserializer6", "interfaces": [], "attributes": [ "public" ], "methods": [ - "public void <init>(com.yahoo.document.DocumentTypeManager, com.yahoo.io.GrowableByteBuffer)", - "public void read(com.yahoo.document.DocumentUpdate)", - "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.BoolFieldValue)" + "public void <init>(com.yahoo.document.DocumentTypeManager, com.yahoo.io.GrowableByteBuffer)" ], "fields": [] }, @@ -4935,7 +4769,6 @@ "public" ], "methods": [ - "public void setHeaderOnly(boolean)", "public void write(com.yahoo.document.Document)", "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.Document)", "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.FieldValue)", @@ -4977,19 +4810,67 @@ ], "fields": [] }, - "com.yahoo.document.serialization.VespaDocumentSerializerHead": { - "superClass": "com.yahoo.document.serialization.VespaDocumentSerializer42", - "interfaces": [], + "com.yahoo.document.serialization.VespaDocumentSerializer6": { + "superClass": "com.yahoo.vespa.objects.BufferSerializer", + "interfaces": [ + "com.yahoo.document.serialization.DocumentSerializer" + ], "attributes": [ "public" ], "methods": [ - "public void <init>(com.yahoo.io.GrowableByteBuffer)", + "public void write(com.yahoo.document.Document)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.Document)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.FieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.Array)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.MapFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.ByteFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.BoolFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.CollectionFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.DoubleFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.FloatFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.IntegerFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.LongFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.Raw)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.PredicateFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.StringFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.TensorFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.ReferenceFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.Struct)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.StructuredFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.WeightedSet)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.annotation.AnnotationReference)", + "public void write(com.yahoo.document.DocumentId)", + "public void write(com.yahoo.document.DocumentType)", + "public void write(com.yahoo.document.annotation.Annotation)", + "public void write(com.yahoo.document.annotation.SpanTree)", + "public void write(com.yahoo.document.annotation.SpanNode)", + "public void write(com.yahoo.document.annotation.Span)", + "public void write(com.yahoo.document.annotation.SpanList)", + "public void write(com.yahoo.document.annotation.AlternateSpanList)", "public void write(com.yahoo.document.DocumentUpdate)", "public void write(com.yahoo.document.fieldpathupdate.FieldPathUpdate)", "public void write(com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate)", "public void write(com.yahoo.document.fieldpathupdate.AddFieldPathUpdate)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.ByteFieldValue)" + "public void write(com.yahoo.document.update.FieldUpdate)", + "public void write(com.yahoo.document.update.AddValueUpdate, com.yahoo.document.DataType)", + "public void write(com.yahoo.document.update.MapValueUpdate, com.yahoo.document.DataType)", + "public void write(com.yahoo.document.update.ArithmeticValueUpdate)", + "public void write(com.yahoo.document.update.AssignValueUpdate, com.yahoo.document.DataType)", + "public void write(com.yahoo.document.update.RemoveValueUpdate, com.yahoo.document.DataType)", + "public void write(com.yahoo.document.update.ClearValueUpdate, com.yahoo.document.DataType)", + "public static long getSerializedSize(com.yahoo.document.Document)" + ], + "fields": [] + }, + "com.yahoo.document.serialization.VespaDocumentSerializerHead": { + "superClass": "com.yahoo.document.serialization.VespaDocumentSerializer6", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.io.GrowableByteBuffer)" ], "fields": [] }, @@ -5359,4 +5240,4 @@ "protected com.yahoo.document.update.ValueUpdate$ValueUpdateClassID valueUpdateClassID" ] } -}
\ No newline at end of file +} diff --git a/document/pom.xml b/document/pom.xml index b8344391631..19f1a9470fc 100644 --- a/document/pom.xml +++ b/document/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>document</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/document/src/main/java/com/yahoo/document/Document.java b/document/src/main/java/com/yahoo/document/Document.java index 222ebe29c6d..cf0951fb035 100644 --- a/document/src/main/java/com/yahoo/document/Document.java +++ b/document/src/main/java/com/yahoo/document/Document.java @@ -137,7 +137,7 @@ public class Document extends StructuredFieldValue { } public int getSerializedSize() throws SerializationException { - DocumentSerializer data = DocumentSerializerFactory.create42(new GrowableByteBuffer(64 * 1024, 2.0f)); + DocumentSerializer data = DocumentSerializerFactory.create6(new GrowableByteBuffer(64 * 1024, 2.0f)); data.write(this); return data.getBuf().position(); } @@ -149,7 +149,7 @@ public class Document extends StructuredFieldValue { public final int getApproxSize() { return 4096; } public void serialize(OutputStream out) throws SerializationException { - DocumentSerializer writer = DocumentSerializerFactory.create42(new GrowableByteBuffer(64 * 1024, 2.0f)); + DocumentSerializer writer = DocumentSerializerFactory.create6(new GrowableByteBuffer(64 * 1024, 2.0f)); writer.write(this); GrowableByteBuffer data = writer.getBuf(); byte[] array; @@ -345,38 +345,6 @@ public class Document extends StructuredFieldValue { serialize((DocumentWriter)data); } - @SuppressWarnings("deprecation") - @Deprecated - public void serializeHeader(Serializer data) throws SerializationException { - if (data instanceof DocumentWriter) { - if (data instanceof com.yahoo.document.serialization.VespaDocumentSerializer42) { - ((com.yahoo.document.serialization.VespaDocumentSerializer42)data).setHeaderOnly(true); - } - serialize((DocumentWriter)data); - } else if (data instanceof BufferSerializer) { - serialize(DocumentSerializerFactory.create42(((BufferSerializer) data).getBuf(), true)); - } else { - DocumentSerializer fw = DocumentSerializerFactory.create42(new GrowableByteBuffer(), true); - serialize(fw); - data.put(null, fw.getBuf().getByteBuffer()); - } - } - - @Deprecated - public void serializeBody(Serializer data) throws SerializationException { - if (getBody().getFieldCount() > 0) { - if (data instanceof FieldWriter) { - getBody().serialize(new Field("body", getBody().getDataType()), (FieldWriter) data); - } else if (data instanceof BufferSerializer) { - getBody().serialize(new Field("body", getBody().getDataType()), DocumentSerializerFactory.create42(((BufferSerializer) data).getBuf())); - } else { - DocumentSerializer fw = DocumentSerializerFactory.create42(new GrowableByteBuffer()); - getBody().serialize(new Field("body", getBody().getDataType()), fw); - data.put(null, fw.getBuf().getByteBuffer()); - } - } - } - @Override public DocumentType getDataType() { return (DocumentType)super.getDataType(); diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManager.java b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java index 5fad35a2287..a3ba27b640f 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManager.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java @@ -266,18 +266,13 @@ public class DocumentTypeManager { } final public Document createDocument(GrowableByteBuffer buf) { - DocumentDeserializer data = DocumentDeserializerFactory.create42(this, buf); + DocumentDeserializer data = DocumentDeserializerFactory.create6(this, buf); return new Document(data); } public Document createDocument(DocumentDeserializer data) { return new Document(data); } - public Document createDocument(GrowableByteBuffer header, GrowableByteBuffer body) { - DocumentDeserializer data = DocumentDeserializerFactory.create42(this, header, body); - return new Document(data); - } - /** * Returns a read only view of the registered data types * diff --git a/document/src/main/java/com/yahoo/document/DocumentUpdate.java b/document/src/main/java/com/yahoo/document/DocumentUpdate.java index 0bbb57ec60b..ef075662ee7 100644 --- a/document/src/main/java/com/yahoo/document/DocumentUpdate.java +++ b/document/src/main/java/com/yahoo/document/DocumentUpdate.java @@ -15,8 +15,10 @@ import com.yahoo.io.GrowableByteBuffer; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Optional; /** @@ -38,7 +40,6 @@ import java.util.Optional; * @see com.yahoo.document.update.FieldUpdate * @see com.yahoo.document.update.ValueUpdate */ -//TODO Vespa 7 Remove all deprecated methods and use a map to avoid quadratic scaling on insert/update/remove public class DocumentUpdate extends DocumentOperation implements Iterable<FieldPathUpdate> { @@ -46,7 +47,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP public static final int CLASSID = 0x1000 + 6; private DocumentId docId; - private final List<FieldUpdate> fieldUpdates; + private final Map<Integer, FieldUpdate> id2FieldUpdates; private final List<FieldPathUpdate> fieldPathUpdates; private DocumentType documentType; private Boolean createIfNonExistent; @@ -60,7 +61,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP public DocumentUpdate(DocumentType docType, DocumentId docId) { this.docId = docId; this.documentType = docType; - this.fieldUpdates = new ArrayList<>(); + this.id2FieldUpdates = new HashMap<>(); this.fieldPathUpdates = new ArrayList<>(); } @@ -70,7 +71,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP public DocumentUpdate(DocumentUpdateReader reader) { docId = null; documentType = null; - fieldUpdates = new ArrayList<>(); + id2FieldUpdates = new HashMap<>(); fieldPathUpdates = new ArrayList<>(); reader.read(this); } @@ -113,7 +114,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP public DocumentUpdate applyTo(Document doc) { verifyType(doc); - for (FieldUpdate fieldUpdate : fieldUpdates) { + for (FieldUpdate fieldUpdate : id2FieldUpdates.values()) { fieldUpdate.applyTo(doc); } for (FieldPathUpdate fieldPathUpdate : fieldPathUpdates) { @@ -131,8 +132,9 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP public DocumentUpdate prune(Document doc) { verifyType(doc); - for (Iterator<FieldUpdate> iter = fieldUpdates.iterator(); iter.hasNext();) { - FieldUpdate update = iter.next(); + for (Iterator<Map.Entry<Integer, FieldUpdate>> iter = id2FieldUpdates.entrySet().iterator(); iter.hasNext();) { + Map.Entry<Integer, FieldUpdate> entry = iter.next(); + FieldUpdate update = entry.getValue(); if (!update.isEmpty()) { ValueUpdate last = update.getValueUpdate(update.size() - 1); if (last instanceof AssignValueUpdate) { @@ -158,34 +160,12 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP } /** - * Get an unmodifiable list of all field updates that this document update specifies. - * - * @return a list of all FieldUpdates in this DocumentUpdate - * @deprecated Use fieldUpdates() instead. - */ - @Deprecated - public List<FieldUpdate> getFieldUpdates() { - return Collections.unmodifiableList(fieldUpdates); - } - - /** * Get an unmodifiable collection of all field updates that this document update specifies. * * @return a collection of all FieldUpdates in this DocumentUpdate */ public Collection<FieldUpdate> fieldUpdates() { - return Collections.unmodifiableCollection(fieldUpdates); - } - - /** - * Get an unmodifiable list of all field path updates this document update specifies. - * - * @return Returns a list of all field path updates in this document update. - * @deprecated Use fieldPathUpdates() instead. - */ - @Deprecated - public List<FieldPathUpdate> getFieldPathUpdates() { - return Collections.unmodifiableList(fieldPathUpdates); + return Collections.unmodifiableCollection(id2FieldUpdates.values()); } /** @@ -214,36 +194,6 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP } /** - * Get the field update at the specified index in the list of field updates. - * - * @param index the index of the FieldUpdate to return - * @return the FieldUpdate at the specified index - * @throws IndexOutOfBoundsException if index is out of range - * @deprecated use getFieldUpdate(Field field) instead. - */ - @Deprecated - public FieldUpdate getFieldUpdate(int index) { - return fieldUpdates.get(index); - } - - /** - * Replaces the field update at the specified index in the list of field updates. - * - * @param index index of the FieldUpdate to replace - * @param upd the FieldUpdate to be stored at the specified position - * @return the FieldUpdate previously at the specified position - * @throws IndexOutOfBoundsException if index is out of range - * @deprecated Use removeFieldUpdate/addFieldUpdate instead - */ - @Deprecated - public FieldUpdate setFieldUpdate(int index, FieldUpdate upd) { - FieldUpdate old = fieldUpdates.get(index); - fieldUpdates.set(index, upd); - - return old; - } - - /** * Returns the update for a field * * @param field the field to return the update of @@ -255,7 +205,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP /** Removes all field updates from the list for field updates. */ public void clearFieldUpdates() { - fieldUpdates.clear(); + id2FieldUpdates.clear(); } /** @@ -269,12 +219,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP return field != null ? getFieldUpdate(field) : null; } private FieldUpdate getFieldUpdateById(Integer fieldId) { - for (FieldUpdate fieldUpdate : fieldUpdates) { - if (fieldUpdate.getField().getId() == fieldId) { - return fieldUpdate; - } - } - return null; + return id2FieldUpdates.get(fieldId); } /** @@ -310,7 +255,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * @return the size of the List of FieldUpdates */ public int size() { - return fieldUpdates.size(); + return id2FieldUpdates.size(); } /** @@ -332,7 +277,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP if (prevUpdate != null) { prevUpdate.addAll(update); } else { - fieldUpdates.add(update); + id2FieldUpdates.put(fieldId, update); } } return this; @@ -372,28 +317,8 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP } } - /** - * Removes the field update at the specified position in the list of field updates. - * - * @param index the index of the FieldUpdate to remove - * @return the FieldUpdate previously at the specified position - * @throws IndexOutOfBoundsException if index is out of range - * @deprecated use removeFieldUpdate(Field field) instead. - */ - @Deprecated - public FieldUpdate removeFieldUpdate(int index) { - return fieldUpdates.remove(index); - } - public FieldUpdate removeFieldUpdate(Field field) { - for (Iterator<FieldUpdate> it = fieldUpdates.iterator(); it.hasNext();) { - FieldUpdate fieldUpdate = it.next(); - if (fieldUpdate.getField().equals(field)) { - it.remove(); - return fieldUpdate; - } - } - return null; + return id2FieldUpdates.remove(field.getId()); } public FieldUpdate removeFieldUpdate(String fieldName) { @@ -411,7 +336,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP } public final void serialize(GrowableByteBuffer buf) { - serialize(DocumentSerializerFactory.create42(buf)); + serialize(DocumentSerializerFactory.create6(buf)); } public void serialize(DocumentUpdateWriter data) { @@ -427,9 +352,8 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP if (docId != null ? !docId.equals(that.docId) : that.docId != null) return false; if (documentType != null ? !documentType.equals(that.documentType) : that.documentType != null) return false; - if (fieldPathUpdates != null ? !fieldPathUpdates.equals(that.fieldPathUpdates) : that.fieldPathUpdates != null) - return false; - if (fieldUpdates != null ? !fieldUpdates.equals(that.fieldUpdates) : that.fieldUpdates != null) return false; + if (!fieldPathUpdates.equals(that.fieldPathUpdates)) return false; + if (!id2FieldUpdates.equals(that.id2FieldUpdates)) return false; if (this.getCreateIfNonExistent() != ((DocumentUpdate) o).getCreateIfNonExistent()) return false; return true; @@ -438,8 +362,8 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP @Override public int hashCode() { int result = docId != null ? docId.hashCode() : 0; - result = 31 * result + (fieldUpdates != null ? fieldUpdates.hashCode() : 0); - result = 31 * result + (fieldPathUpdates != null ? fieldPathUpdates.hashCode() : 0); + result = 31 * result + id2FieldUpdates.hashCode(); + result = 31 * result + fieldPathUpdates.hashCode(); result = 31 * result + (documentType != null ? documentType.hashCode() : 0); return result; } @@ -455,7 +379,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP string.append(": "); string.append("["); - for (FieldUpdate fieldUpdate : fieldUpdates) { + for (FieldUpdate fieldUpdate : id2FieldUpdates.values()) { string.append(fieldUpdate).append(" "); } string.append("]"); @@ -482,14 +406,13 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * @return True if this update is empty. */ public boolean isEmpty() { - return fieldUpdates.isEmpty() && fieldPathUpdates.isEmpty(); + return id2FieldUpdates.isEmpty() && fieldPathUpdates.isEmpty(); } /** * Sets whether this update should create the document it updates if that document does not exist. * In this case an empty document is created before the update is applied. * - * @since 5.17 * @param value Whether the document it updates should be created. */ public void setCreateIfNonExistent(boolean value) { @@ -499,8 +422,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP /** * Gets whether this update should create the document it updates if that document does not exist. * - * @since 5.17 - * @return Whether the document it updates should be created. + * @return whether the document it updates should be created. */ public boolean getCreateIfNonExistent() { return createIfNonExistent != null && createIfNonExistent; diff --git a/document/src/main/java/com/yahoo/document/Field.java b/document/src/main/java/com/yahoo/document/Field.java index 5d3d148d832..d0a19a50073 100644 --- a/document/src/main/java/com/yahoo/document/Field.java +++ b/document/src/main/java/com/yahoo/document/Field.java @@ -187,7 +187,7 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab * @param type The new type of the field. * @deprecated do not use */ - @Deprecated // TODO: Remove on Vespa 8 + @Deprecated // todo - refactor SD processing to avoid needing this public void setDataType(DataType type) { dataType = type; fieldId = calculateIdV7(null); diff --git a/document/src/main/java/com/yahoo/document/PositionDataType.java b/document/src/main/java/com/yahoo/document/PositionDataType.java index bb110ee7219..0e69171c106 100644 --- a/document/src/main/java/com/yahoo/document/PositionDataType.java +++ b/document/src/main/java/com/yahoo/document/PositionDataType.java @@ -100,12 +100,12 @@ public final class PositionDataType { } public static String getPositionSummaryFieldName(String fieldName) { - // TODO for 6.0, rename to _position to use a field name that is actually legal + // TODO for Vespa 8, consider renaming to _position to use a field name that is actually legal return fieldName + ".position"; } public static String getDistanceSummaryFieldName(String fieldName) { - // TODO for 6.0, rename to _distance to use a field name that is actually legal + // TODO for Vespa 8, consider renaming to _distance to use a field name that is actually legal return fieldName + ".distance"; } diff --git a/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java index dc3fd36b367..f895ad21b0a 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java @@ -52,7 +52,7 @@ public abstract class FieldValue extends Identifiable implements Comparable<Fiel } final public void serialize(GrowableByteBuffer buf) { - serialize(DocumentSerializerFactory.create42(buf)); + serialize(DocumentSerializerFactory.create6(buf)); } public abstract void printXml(XmlStream xml); @@ -140,9 +140,9 @@ public abstract class FieldValue extends Identifiable implements Comparable<Fiel if (target instanceof FieldWriter) { serialize(null, (FieldWriter) target); } else if (target instanceof BufferSerializer) { - serialize(null, DocumentSerializerFactory.create42(((BufferSerializer) target).getBuf())); + serialize(null, DocumentSerializerFactory.create6(((BufferSerializer) target).getBuf())); } else { - DocumentSerializer fw = DocumentSerializerFactory.create42(new GrowableByteBuffer()); + DocumentSerializer fw = DocumentSerializerFactory.create6(new GrowableByteBuffer()); serialize(null, fw); target.put(null, fw.getBuf().getByteBuffer()); } diff --git a/document/src/main/java/com/yahoo/document/fieldpathupdate/AddFieldPathUpdate.java b/document/src/main/java/com/yahoo/document/fieldpathupdate/AddFieldPathUpdate.java index 9fac025c15e..74b94b8135e 100644 --- a/document/src/main/java/com/yahoo/document/fieldpathupdate/AddFieldPathUpdate.java +++ b/document/src/main/java/com/yahoo/document/fieldpathupdate/AddFieldPathUpdate.java @@ -8,7 +8,7 @@ import com.yahoo.document.datatypes.CollectionFieldValue; import com.yahoo.document.datatypes.FieldPathIteratorHandler; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.serialization.DocumentUpdateReader; -import com.yahoo.document.serialization.VespaDocumentSerializerHead; +import com.yahoo.document.serialization.VespaDocumentSerializer6; /** * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> @@ -96,7 +96,7 @@ public class AddFieldPathUpdate extends FieldPathUpdate { } @Override - public void serialize(VespaDocumentSerializerHead data) { + public void serialize(VespaDocumentSerializer6 data) { data.write(this); } diff --git a/document/src/main/java/com/yahoo/document/fieldpathupdate/AssignFieldPathUpdate.java b/document/src/main/java/com/yahoo/document/fieldpathupdate/AssignFieldPathUpdate.java index e31eefa2c00..b01742018a2 100644 --- a/document/src/main/java/com/yahoo/document/fieldpathupdate/AssignFieldPathUpdate.java +++ b/document/src/main/java/com/yahoo/document/fieldpathupdate/AssignFieldPathUpdate.java @@ -9,7 +9,7 @@ import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.NumericFieldValue; import com.yahoo.document.select.parser.ParseException; import com.yahoo.document.serialization.DocumentUpdateReader; -import com.yahoo.document.serialization.VespaDocumentSerializerHead; +import com.yahoo.document.serialization.VespaDocumentSerializer6; import java.util.HashMap; import java.util.Map; @@ -237,7 +237,7 @@ public class AssignFieldPathUpdate extends FieldPathUpdate { } @Override - public void serialize(VespaDocumentSerializerHead data) { + public void serialize(VespaDocumentSerializer6 data) { data.write(this); } diff --git a/document/src/main/java/com/yahoo/document/fieldpathupdate/FieldPathUpdate.java b/document/src/main/java/com/yahoo/document/fieldpathupdate/FieldPathUpdate.java index 29c876b85fe..d4144116a03 100644 --- a/document/src/main/java/com/yahoo/document/fieldpathupdate/FieldPathUpdate.java +++ b/document/src/main/java/com/yahoo/document/fieldpathupdate/FieldPathUpdate.java @@ -11,7 +11,7 @@ import com.yahoo.document.select.Result; import com.yahoo.document.select.ResultList; import com.yahoo.document.select.parser.ParseException; import com.yahoo.document.serialization.DocumentUpdateReader; -import com.yahoo.document.serialization.VespaDocumentSerializerHead; +import com.yahoo.document.serialization.VespaDocumentSerializer6; import java.util.ListIterator; /** @@ -130,7 +130,7 @@ public abstract class FieldPathUpdate { } } - public void serialize(VespaDocumentSerializerHead data) { + public void serialize(VespaDocumentSerializer6 data) { data.write(this); } diff --git a/document/src/main/java/com/yahoo/document/select/package-info.java b/document/src/main/java/com/yahoo/document/select/package-info.java index e6de6497008..813643a7f06 100644 --- a/document/src/main/java/com/yahoo/document/select/package-info.java +++ b/document/src/main/java/com/yahoo/document/select/package-info.java @@ -1,7 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage -@PublicApi package com.yahoo.document.select; -import com.yahoo.api.annotations.PublicApi; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializerFactory.java b/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializerFactory.java index 072f11f5f22..03aa409ee8d 100644 --- a/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializerFactory.java +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializerFactory.java @@ -13,25 +13,26 @@ public class DocumentDeserializerFactory { /** * Creates a de-serializer for the current head document format. - * This format is an extension of the 4.2 format. + * This format is an extension of the 6.x format. */ public static DocumentDeserializer createHead(DocumentTypeManager manager, GrowableByteBuffer buf) { return new VespaDocumentDeserializerHead(manager, buf); } /** - * Creates a de-serializer for the document format that was created on Vespa 4.2. + * Creates a de-serializer for the 6.x document format. + * This format is an extension of the 4.2 format. */ - @SuppressWarnings("deprecation") - public static DocumentDeserializer create42(DocumentTypeManager manager, GrowableByteBuffer buf) { - return new VespaDocumentDeserializer42(manager, buf); + public static DocumentDeserializer create6(DocumentTypeManager manager, GrowableByteBuffer buf) { + return new VespaDocumentDeserializer6(manager, buf); } /** * Creates a de-serializer for the document format that was created on Vespa 4.2. */ @SuppressWarnings("deprecation") - public static DocumentDeserializer create42(DocumentTypeManager manager, GrowableByteBuffer buf, GrowableByteBuffer body) { - return new VespaDocumentDeserializer42(manager, buf, body); + public static DocumentDeserializer create42(DocumentTypeManager manager, GrowableByteBuffer buf) { + return new VespaDocumentDeserializer42(manager, buf); } + } diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentSerializerFactory.java b/document/src/main/java/com/yahoo/document/serialization/DocumentSerializerFactory.java index 7ff58855c34..54ec4e2fcca 100644 --- a/document/src/main/java/com/yahoo/document/serialization/DocumentSerializerFactory.java +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentSerializerFactory.java @@ -12,26 +12,34 @@ public class DocumentSerializerFactory { /** * Creates a serializer for the current head document format. - * This format is an extension of the 4.2 format. + * This format is an extension of the 6.x format. */ public static DocumentSerializer createHead(GrowableByteBuffer buf) { return new VespaDocumentSerializerHead(buf); } /** - * Creates a serializer for the document format that was created on Vespa 4.2. + * Creates a serializer for the 6.x document format. + * This format is an extension of the 4.2 format. */ - @SuppressWarnings("deprecation") - public static DocumentSerializer create42(GrowableByteBuffer buf) { - return new VespaDocumentSerializer42(buf); + public static DocumentSerializer create6(GrowableByteBuffer buf) { + return new VespaDocumentSerializer6(buf); + } + + /** + * Creates a serializer for the 6.x document format. + * This format is an extension of the 4.2 format. + */ + public static DocumentSerializer create6() { + return new VespaDocumentSerializer6(new GrowableByteBuffer()); } /** * Creates a serializer for the document format that was created on Vespa 4.2. */ @SuppressWarnings("deprecation") - public static DocumentSerializer create42(GrowableByteBuffer buf, boolean headerOnly) { - return new VespaDocumentSerializer42(buf, headerOnly); + public static DocumentSerializer create42(GrowableByteBuffer buf) { + return new VespaDocumentSerializer42(buf); } /** diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java index 7ff5729ca39..7ec4433a24f 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java @@ -63,6 +63,7 @@ import com.yahoo.tensor.serialization.TypedBinaryFormat; import com.yahoo.text.Utf8; import com.yahoo.text.Utf8Array; import com.yahoo.text.Utf8String; +import com.yahoo.vespa.objects.BufferSerializer; import com.yahoo.vespa.objects.FieldBase; import java.nio.ByteBuffer; @@ -80,9 +81,9 @@ import static com.yahoo.text.Utf8.calculateStringPositions; * @deprecated Please use {@link com.yahoo.document.serialization.VespaDocumentDeserializerHead} instead for new code. * @author baldersheim */ -@Deprecated // Remove on Vespa 7 +@Deprecated // TODO: Remove on Vespa 8 // When removing: Move content of this class into VespaDocumentDeserializerHead -public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 implements DocumentDeserializer { +public class VespaDocumentDeserializer42 extends BufferSerializer implements DocumentDeserializer { private final Compressor compressor = new Compressor(); private DocumentTypeManager manager; diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java new file mode 100644 index 00000000000..5424798110c --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java @@ -0,0 +1,880 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.collections.Tuple2; +import com.yahoo.compress.CompressionType; +import com.yahoo.compress.Compressor; +import com.yahoo.document.annotation.AlternateSpanList; +import com.yahoo.document.annotation.Annotation; +import com.yahoo.document.annotation.AnnotationReference; +import com.yahoo.document.annotation.AnnotationType; +import com.yahoo.document.annotation.Span; +import com.yahoo.document.annotation.SpanList; +import com.yahoo.document.annotation.SpanNode; +import com.yahoo.document.annotation.SpanNodeParent; +import com.yahoo.document.annotation.SpanTree; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.DataTypeName; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.BoolFieldValue; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.ReferenceFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.Field; +import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.FieldPathUpdate; +import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate; +import com.yahoo.document.MapDataType; +import com.yahoo.document.predicate.BinaryFormat; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.document.StructDataType; +import com.yahoo.document.update.AddValueUpdate; +import com.yahoo.document.update.ArithmeticValueUpdate; +import com.yahoo.document.update.AssignValueUpdate; +import com.yahoo.document.update.ClearValueUpdate; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.document.update.MapValueUpdate; +import com.yahoo.document.update.RemoveValueUpdate; +import com.yahoo.document.update.ValueUpdate; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.tensor.serialization.TypedBinaryFormat; +import com.yahoo.text.Utf8; +import com.yahoo.text.Utf8Array; +import com.yahoo.text.Utf8String; +import com.yahoo.vespa.objects.BufferSerializer; +import com.yahoo.vespa.objects.FieldBase; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.yahoo.text.Utf8.calculateStringPositions; + +/** + * Class used for de-serializing documents on the Vespa 6.x document format. + * + * @author baldersheim + */ +public class VespaDocumentDeserializer6 extends BufferSerializer implements DocumentDeserializer { + + private final Compressor compressor = new Compressor(); + private DocumentTypeManager manager; + private short version; + private List<SpanNode> spanNodes; + private List<Annotation> annotations; + private int[] stringPositions; + + VespaDocumentDeserializer6(DocumentTypeManager manager, GrowableByteBuffer buf) { + super(buf); + this.manager = manager; + this.version = Document.SERIALIZED_VERSION; + } + + final public DocumentTypeManager getDocumentTypeManager() { return manager; } + + public void read(Document document) { + read(null, document); + } + + @SuppressWarnings("deprecation") + public void read(FieldBase field, Document doc) { + // Verify that we have correct version + version = getShort(null); + if (version < 6 || version > Document.SERIALIZED_VERSION) { + throw new DeserializationException("Unknown version " + version + ", expected " + + Document.SERIALIZED_VERSION + "."); + } + + int dataLength = 0; + int dataPos = 0; + + if (version < 7) { + getInt2_4_8Bytes(null); // Total document size.. Ignore + } else { + dataLength = getInt(null); + dataPos = position(); + } + + doc.setId(readDocumentId()); + + Byte content = getByte(null); + + doc.setDataType(readDocumentType()); + + Struct h = doc.getHeader(); + Struct b = doc.getBody(); + h.clear(); + b.clear(); + if ((content & 0x2) != 0) { + readHeaderBody(h, b); + } + if ((content & 0x4) != 0) { + readHeaderBody(b, h); + } + + if (version < 8) { + int crcVal = getInt(null); + } + + if (version > 6) { + if (dataLength != (position() - dataPos)) { + throw new DeserializationException("Length mismatch"); + } + } + } + public void read(FieldBase field, FieldValue value) { + throw new IllegalArgumentException("read not implemented yet."); + } + + public <T extends FieldValue> void read(FieldBase field, Array<T> array) { + int numElements = getNumCollectionElems(); + ArrayList<T> list = new ArrayList<T>(numElements); + ArrayDataType type = array.getDataType(); + for (int i = 0; i < numElements; i++) { + if (version < 7) { + getInt(null); // We don't need size for anything + } + FieldValue fv = type.getNestedType().createFieldValue(); + fv.deserialize(null, this); + list.add((T) fv); + } + array.clear(); + array.addAll(list); + } + + public <K extends FieldValue, V extends FieldValue> void read(FieldBase field, MapFieldValue<K, V> map) { + int numElements = getNumCollectionElems(); + Map<K,V> hash = new HashMap<>(); + MapDataType type = map.getDataType(); + for (int i = 0; i < numElements; i++) { + if (version < 7) { + getInt(null); // We don't need size for anything + } + K key = (K) type.getKeyType().createFieldValue(); + V val = (V) type.getValueType().createFieldValue(); + key.deserialize(null, this); + val.deserialize(null, this); + hash.put(key, val); + } + map.clear(); + map.putAll(hash); + } + + private int getNumCollectionElems() { + int numElements; + if (version < 7) { + getInt(null); // We already know the nested type, so ignore that.. + numElements = getInt(null); + } else { + numElements = getInt1_2_4Bytes(null); + } + if (numElements < 0) { + throw new DeserializationException("Bad number of array/map elements, " + numElements); + } + return numElements; + } + + public <T extends FieldValue> void read(FieldBase field, CollectionFieldValue<T> value) { + throw new IllegalArgumentException("read not implemented yet."); + } + public void read(FieldBase field, ByteFieldValue value) { value.assign(getByte(null)); } + + @Override + public void read(FieldBase field, BoolFieldValue value) { + value.setBoolean((getByte(null) != 0)); + } + + public void read(FieldBase field, DoubleFieldValue value) { value.assign(getDouble(null)); } + public void read(FieldBase field, FloatFieldValue value) { value.assign(getFloat(null)); } + public void read(FieldBase field, IntegerFieldValue value) { value.assign(getInt(null)); } + public void read(FieldBase field, LongFieldValue value) { value.assign(getLong(null)); } + + public void read(FieldBase field, Raw value) { + int rawsize = getInt(null); + byte[] rawBytes = getBytes(null, rawsize); + value.assign(rawBytes); + } + + @Override + public void read(FieldBase field, PredicateFieldValue value) { + int len = getInt(null); + byte[] buf = getBytes(null, len); + value.assign(BinaryFormat.decode(buf)); + } + + public void read(FieldBase field, StringFieldValue value) { + byte coding = getByte(null); + + int length = getInt1_4Bytes(null); + + //OK, it seems that this length includes null termination. + //NOTE: the following four lines are basically parseNullTerminatedString() inlined, + //but we need to use the UTF-8 buffer below, so not using that method... + byte[] stringArray = new byte[length - 1]; + buf.get(stringArray); + buf.get(); //move past 0-termination + value.setUnChecked(Utf8.toString(stringArray)); + + if ((coding & 64) == 64) { + //we have a span tree! + try { + //we don't support serialization of nested span trees, so this is safe: + stringPositions = calculateStringPositions(stringArray); + //total length: + int size = buf.getInt(); + int startPos = buf.position(); + + int numSpanTrees = buf.getInt1_2_4Bytes(); + + for (int i = 0; i < numSpanTrees; i++) { + SpanTree tree = new SpanTree(); + StringFieldValue treeName = new StringFieldValue(); + treeName.deserialize(this); + tree.setName(treeName.getString()); + value.setSpanTree(tree); + readSpanTree(tree, false); + } + + buf.position(startPos + size); + } finally { + stringPositions = null; + } + } + } + + @Override + public void read(FieldBase field, TensorFieldValue value) { + int encodedTensorLength = buf.getInt1_4Bytes(); + if (encodedTensorLength > 0) { + byte[] encodedTensor = getBytes(null, encodedTensorLength); + value.assign(TypedBinaryFormat.decode(Optional.of(value.getDataType().getTensorType()), + GrowableByteBuffer.wrap(encodedTensor))); + } else { + value.clear(); + } + } + + @Override + public void read(FieldBase field, ReferenceFieldValue value) { + final boolean documentIdPresent = (buf.get() != 0); + if (documentIdPresent) { + value.assign(readDocumentId()); + } else { + value.clear(); + } + } + + public void read(FieldBase fieldDef, Struct s) { + s.setVersion(version); + int startPos = position(); + + if (version < 6) { + throw new DeserializationException("Illegal document serialization version " + version); + } + + int dataSize; + if (version < 7) { + long rSize = getInt2_4_8Bytes(null); + //TODO: Look into how to support data segments larger than INT_MAX bytes + if (rSize > Integer.MAX_VALUE) { + throw new DeserializationException("Raw size of data block is too large."); + } + dataSize = (int)rSize; + } else { + dataSize = getInt(null); + } + + byte comprCode = getByte(null); + CompressionType compression = CompressionType.valueOf(comprCode); + + int uncompressedSize = 0; + if (compression != CompressionType.NONE && + compression != CompressionType.INCOMPRESSIBLE) + { + // uncompressedsize (full size of FIELDS only, after decompression) + long pSize = getInt2_4_8Bytes(null); + //TODO: Look into how to support data segments larger than INT_MAX bytes + if (pSize > Integer.MAX_VALUE) { + throw new DeserializationException("Uncompressed size of data block is too large."); + } + uncompressedSize = (int) pSize; + } + + int numberOfFields = getInt1_4Bytes(null); + + List<Tuple2<Integer, Long>> fieldIdsAndLengths = new ArrayList<>(numberOfFields); + for (int i=0; i<numberOfFields; ++i) { + // id, length (length only used for unknown fields + fieldIdsAndLengths.add(new Tuple2<>(getInt1_4Bytes(null), getInt2_4_8Bytes(null))); + } + + // save a reference to the big buffer we're reading from: + GrowableByteBuffer bigBuf = buf; + + if (version < 7) { + // In V6 and earlier, the length included the header. + int headerSize = position() - startPos; + dataSize -= headerSize; + } + byte[] destination = compressor.decompress(compression, getBuf().array(), position(), uncompressedSize, Optional.of(dataSize)); + + // set position in original buffer to after data + position(position() + dataSize); + + // for a while: deserialize from this buffer instead: + buf = GrowableByteBuffer.wrap(destination); + + s.clear(); + StructDataType type = s.getDataType(); + for (int i=0; i<numberOfFields; ++i) { + Field structField = type.getField(fieldIdsAndLengths.get(i).first, version); + if (structField == null) { + //ignoring unknown field: + position(position() + fieldIdsAndLengths.get(i).second.intValue()); + } else { + int posBefore = position(); + FieldValue value = structField.getDataType().createFieldValue(); + value.deserialize(structField, this); + s.setFieldValue(structField, value); + //jump to beginning of next field: + position(posBefore + fieldIdsAndLengths.get(i).second.intValue()); + } + } + + // restore the original buffer + buf = bigBuf; + } + + private void readHeaderBody(Struct primary, Struct alternate) { + primary.setVersion(version); + int startPos = position(); + + if (version < 6) { + throw new DeserializationException("Illegal document serialization version " + version); + } + + int dataSize; + if (version < 7) { + long rSize = getInt2_4_8Bytes(null); + //TODO: Look into how to support data segments larger than INT_MAX bytes + if (rSize > Integer.MAX_VALUE) { + throw new DeserializationException("Raw size of data block is too large."); + } + dataSize = (int)rSize; + } else { + dataSize = getInt(null); + } + + byte comprCode = getByte(null); + CompressionType compression = CompressionType.valueOf(comprCode); + + int uncompressedSize = 0; + if (compression != CompressionType.NONE && + compression != CompressionType.INCOMPRESSIBLE) + { + // uncompressedsize (full size of FIELDS only, after decompression) + long pSize = getInt2_4_8Bytes(null); + //TODO: Look into how to support data segments larger than INT_MAX bytes + if (pSize > Integer.MAX_VALUE) { + throw new DeserializationException("Uncompressed size of data block is too large."); + } + uncompressedSize = (int) pSize; + } + + int numberOfFields = getInt1_4Bytes(null); + + List<Tuple2<Integer, Long>> fieldIdsAndLengths = new ArrayList<>(numberOfFields); + for (int i=0; i<numberOfFields; ++i) { + // id, length (length only used for unknown fields + fieldIdsAndLengths.add(new Tuple2<>(getInt1_4Bytes(null), getInt2_4_8Bytes(null))); + } + + // save a reference to the big buffer we're reading from: + GrowableByteBuffer bigBuf = buf; + + if (version < 7) { + // In V6 and earlier, the length included the header. + int headerSize = position() - startPos; + dataSize -= headerSize; + } + byte[] destination = compressor.decompress(compression, getBuf().array(), position(), uncompressedSize, Optional.of(dataSize)); + + // set position in original buffer to after data + position(position() + dataSize); + + // for a while: deserialize from this buffer instead: + buf = GrowableByteBuffer.wrap(destination); + + StructDataType priType = primary.getDataType(); + StructDataType altType = alternate.getDataType(); + for (int i=0; i<numberOfFields; ++i) { + int posBefore = position(); + Struct s = null; + Integer f_id = fieldIdsAndLengths.get(i).first; + Field structField = priType.getField(f_id, version); + if (structField != null) { + s = primary; + } else { + structField = altType.getField(f_id, version); + if (structField != null) { + s = alternate; + } + } + if (s != null) { + FieldValue value = structField.getDataType().createFieldValue(); + value.deserialize(structField, this); + s.setFieldValue(structField, value); + } + //jump to beginning of next field: + position(posBefore + fieldIdsAndLengths.get(i).second.intValue()); + } + + // restore the original buffer + buf = bigBuf; + } + + public void read(FieldBase field, StructuredFieldValue value) { + throw new IllegalArgumentException("read not implemented yet."); + } + public <T extends FieldValue> void read(FieldBase field, WeightedSet<T> ws) { + WeightedSetDataType type = ws.getDataType(); + getInt(null); // Have no need for type + + int numElements = getInt(null); + if (numElements < 0) { + throw new DeserializationException("Bad number of weighted set elements, " + numElements); + } + + ws.clearAndReserve(numElements * 2); // Avoid resizing + for (int i = 0; i < numElements; i++) { + int size = getInt(null); + FieldValue value = type.getNestedType().createFieldValue(); + value.deserialize(null, this); + IntegerFieldValue weight = new IntegerFieldValue(getInt(null)); + ws.putUnChecked((T) value, weight); + } + + } + + public void read(FieldBase field, AnnotationReference value) { + int seqId = buf.getInt1_2_4Bytes(); + try { + Annotation a = annotations.get(seqId); + value.setReferenceNoCompatibilityCheck(a); + } catch (IndexOutOfBoundsException iiobe) { + throw new SerializationException("Could not serialize AnnotationReference value, reference not found.", iiobe); + } + } + + private Utf8String deserializeAttributeString() throws DeserializationException { + int length = getByte(null); + return new Utf8String(parseNullTerminatedString(length)); + } + + private Utf8Array parseNullTerminatedString() { return parseNullTerminatedString(getBuf().getByteBuffer()); } + private Utf8Array parseNullTerminatedString(int lengthExcludingNull) { return parseNullTerminatedString(getBuf().getByteBuffer(), lengthExcludingNull); } + + static Utf8Array parseNullTerminatedString(ByteBuffer buf, int lengthExcludingNull) throws DeserializationException { + Utf8Array utf8 = new Utf8Array(buf, lengthExcludingNull); + buf.get(); //move past 0-termination + return utf8; + } + + static Utf8Array parseNullTerminatedString(ByteBuffer buf) throws DeserializationException { + //search for 0-byte + int end = getFirstNullByte(buf); + + if (end == -1) { + throw new DeserializationException("Could not locate terminating 0-byte for string"); + } + + return parseNullTerminatedString(buf, end - buf.position()); + } + + private static int getFirstNullByte(ByteBuffer buf) { + int end = -1; + int start = buf.position(); + + while (true) { + try { + byte dataByte = buf.get(); + if (dataByte == (byte) 0) { + end = buf.position() - 1; + break; + } + } catch (Exception e) { + break; + } + } + + buf.position(start); + return end; + } + + public void read(DocumentUpdate update) { + update.setId(new DocumentId(this)); + update.setDocumentType(readDocumentType()); + + int size = getInt(null); + + for (int i = 0; i < size; i++) { + update.addFieldUpdate(new FieldUpdate(this, update.getDocumentType(), 8)); + } + + int sizeAndFlags = getInt(null); + update.setCreateIfNonExistent(DocumentUpdateFlags.extractFlags(sizeAndFlags).getCreateIfNonExistent()); + size = DocumentUpdateFlags.extractValue(sizeAndFlags); + + for (int i = 0; i < size; i++) { + int type = getByte(null); + update.addFieldPathUpdate(FieldPathUpdate.create(FieldPathUpdate.Type.valueOf(type), + update.getDocumentType(), this)); + } + } + + + public void read(FieldPathUpdate update) { + String fieldPath = getString(null); + String whereClause = getString(null); + update.setFieldPath(fieldPath); + + try { + update.setWhereClause(whereClause); + } catch (ParseException e) { + throw new DeserializationException(e); + } + } + + public void read(AssignFieldPathUpdate update) { + byte flags = getByte(null); + update.setRemoveIfZero((flags & AssignFieldPathUpdate.REMOVE_IF_ZERO) != 0); + update.setCreateMissingPath((flags & AssignFieldPathUpdate.CREATE_MISSING_PATH) != 0); + if ((flags & AssignFieldPathUpdate.ARITHMETIC_EXPRESSION) != 0) { + update.setExpression(getString(null)); + } else { + DataType dt = update.getFieldPath().getResultingDataType(); + FieldValue fv = dt.createFieldValue(); + fv.deserialize(this); + update.setNewValue(fv); + } + } + + public void read(RemoveFieldPathUpdate update) { + + } + + public void read(AddFieldPathUpdate update) { + DataType dt = update.getFieldPath().getResultingDataType(); + FieldValue fv = dt.createFieldValue(); + dt.createFieldValue(); + fv.deserialize(this); + + if (!(fv instanceof Array)) { + throw new DeserializationException("Add only applicable to array types"); + } + update.setNewValues((Array)fv); + } + + public ValueUpdate getValueUpdate(DataType superType, DataType subType) { + int vuTypeId = getInt(null); + + ValueUpdate.ValueUpdateClassID op = ValueUpdate.ValueUpdateClassID.getID(vuTypeId); + if (op == null) { + throw new IllegalArgumentException("Read type "+vuTypeId+" of bytebuffer, but this is not a legal value update type."); + } + + switch (op) { + case ADD: + { + FieldValue fval = subType.createFieldValue(); + fval.deserialize(this); + int weight = getInt(null); + return new AddValueUpdate(fval, weight); + } + case ARITHMETIC: + int opId = getInt(null); + ArithmeticValueUpdate.Operator operator = ArithmeticValueUpdate.Operator.getID(opId); + double operand = getDouble(null); + return new ArithmeticValueUpdate(operator, operand); + case ASSIGN: + { + byte contents = getByte(null); + FieldValue fval = null; + if (contents == (byte) 1) { + fval = superType.createFieldValue(); + fval.deserialize(this); + } + return new AssignValueUpdate(fval); + } + case CLEAR: + return new ClearValueUpdate(); + case MAP: + if (superType instanceof ArrayDataType) { + CollectionDataType type = (CollectionDataType) superType; + IntegerFieldValue index = new IntegerFieldValue(); + index.deserialize(this); + ValueUpdate update = getValueUpdate(type.getNestedType(), null); + return new MapValueUpdate(index, update); + } else if (superType instanceof WeightedSetDataType) { + CollectionDataType type = (CollectionDataType) superType; + FieldValue fval = type.getNestedType().createFieldValue(); + fval.deserialize(this); + ValueUpdate update = getValueUpdate(DataType.INT, null); + return new MapValueUpdate(fval, update); + } else { + throw new DeserializationException("MapValueUpdate only works for arrays and weighted sets"); + } + case REMOVE: + FieldValue fval = ((CollectionDataType) superType).getNestedType().createFieldValue(); + fval.deserialize(this); + return new RemoveValueUpdate(fval); + default: + throw new DeserializationException( + "Could not deserialize ValueUpdate, unknown valueUpdateClassID type " + vuTypeId); + } + } + + public void read(FieldUpdate fieldUpdate) { + int fieldId = getInt(null); + Field field = fieldUpdate.getDocumentType().getField(fieldId, fieldUpdate.getSerializationVersion()); + if (field == null) { + throw new DeserializationException( + "Cannot deserialize FieldUpdate, field fieldId " + fieldId + " not found in " + fieldUpdate.getDocumentType()); + } + + fieldUpdate.setField(field); + int size = getInt(null); + + for (int i = 0; i < size; i++) { + if (field.getDataType() instanceof CollectionDataType) { + CollectionDataType collType = (CollectionDataType) field.getDataType(); + fieldUpdate.addValueUpdate(getValueUpdate(collType, collType.getNestedType())); + } else { + fieldUpdate.addValueUpdate(getValueUpdate(field.getDataType(), null)); + } + } + } + + public DocumentId readDocumentId() { + Utf8String uri = new Utf8String(parseNullTerminatedString(getBuf().getByteBuffer())); + return DocumentId.createFromSerialized(uri.toString()); + } + + public DocumentType readDocumentType() { + Utf8Array docTypeName = parseNullTerminatedString(); + int ignored = getShort(null); // used to hold the version + + DocumentType docType = manager.getDocumentType(new DataTypeName(docTypeName)); + if (docType == null) { + throw new DeserializationException("No known document type with name " + + new Utf8String(docTypeName).toString()); + } + return docType; + } + + private SpanNode readSpanNode() { + byte type = buf.get(); + buf.position(buf.position() - 1); + + SpanNode retval; + if ((type & Span.ID) == Span.ID) { + retval = new Span(); + if (spanNodes != null) { + spanNodes.add(retval); + } + read((Span) retval); + } else if ((type & SpanList.ID) == SpanList.ID) { + retval = new SpanList(); + if (spanNodes != null) { + spanNodes.add(retval); + } + read((SpanList) retval); + } else if ((type & AlternateSpanList.ID) == AlternateSpanList.ID) { + retval = new AlternateSpanList(); + if (spanNodes != null) { + spanNodes.add(retval); + } + read((AlternateSpanList) retval); + } else { + throw new DeserializationException("Cannot read SpanNode of type " + type); + } + return retval; + } + + private void readSpanTree(SpanTree tree, boolean readName) { + //we don't support serialization of nested span trees: + if (spanNodes != null || annotations != null) { + throw new SerializationException("Deserialization of nested SpanTrees is not supported."); + } + + //we're going to write a new SpanTree, create a new Map for nodes: + spanNodes = new ArrayList<SpanNode>(); + annotations = new ArrayList<Annotation>(); + + try { + if (readName) { + StringFieldValue treeName = new StringFieldValue(); + treeName.deserialize(this); + tree.setName(treeName.getString()); + } + + SpanNode root = readSpanNode(); + tree.setRoot(root); + + int numAnnotations = buf.getInt1_2_4Bytes(); + + for (int i = 0; i < numAnnotations; i++) { + Annotation a = new Annotation(); + annotations.add(a); + } + for (int i = 0; i < numAnnotations; i++) { + read(annotations.get(i)); + } + for (Annotation a : annotations) { + tree.annotate(a); + } + + for (SpanNode node: spanNodes) { + if (node instanceof Span) { + correctIndexes((Span) node); + } + } + } finally { + //we're done, let's set this to null to save memory and prevent madness: + spanNodes = null; + annotations = null; + } + } + + public void read(SpanTree tree) { + readSpanTree(tree, true); + } + + public void read(Annotation annotation) { + int annotationTypeId = buf.getInt(); + AnnotationType type = manager.getAnnotationTypeRegistry().getType(annotationTypeId); + + if (type == null) { + throw new DeserializationException("Cannot deserialize annotation of type " + annotationTypeId + " (unknown type)"); + } + + annotation.setType(type); + + byte features = buf.get(); + int length = buf.getInt1_2_4Bytes(); + + if ((features & (byte) 1) == (byte) 1) { + //we have a span node + int spanNodeId = buf.getInt1_2_4Bytes(); + try { + SpanNode node = spanNodes.get(spanNodeId); + annotation.setSpanNode(node); + } catch (IndexOutOfBoundsException ioobe) { + throw new DeserializationException("Could not deserialize annotation, associated span node not found ", ioobe); + } + } + if ((features & (byte) 2) == (byte) 2) { + //we have a value: + int dataTypeId = buf.getInt(); + + //if this data type ID the same as the one in our config? + if (dataTypeId != type.getDataType().getId()) { + //not the same, but we will handle it gracefully, and just skip past the data: + buf.position(buf.position() + length - 4); + } else { + FieldValue value = type.getDataType().createFieldValue(); + value.deserialize(this); + annotation.setFieldValue(value); + } + } + } + + public void read(Span span) { + byte type = buf.get(); + if ((type & Span.ID) != Span.ID) { + throw new DeserializationException("Cannot deserialize Span with type " + type); + } + span.setFrom(buf.getInt1_2_4Bytes()); + span.setLength(buf.getInt1_2_4Bytes()); + } + + private void correctIndexes(Span span) { + if (stringPositions == null) { + throw new DeserializationException("Cannot deserialize Span, no access to parent StringFieldValue."); + } + int fromIndex = stringPositions[span.getFrom()]; + int toIndex = stringPositions[span.getTo()]; + int length = toIndex - fromIndex; + + span.setFrom(fromIndex); + span.setLength(length); + } + + public void read(SpanList spanList) { + byte type = buf.get(); + if ((type & SpanList.ID) != SpanList.ID) { + throw new DeserializationException("Cannot deserialize SpanList with type " + type); + } + List<SpanNode> nodes = readSpanList(spanList); + for (SpanNode node : nodes) { + spanList.add(node); + } + } + + public void read(AlternateSpanList altSpanList) { + byte type = buf.get(); + if ((type & AlternateSpanList.ID) != AlternateSpanList.ID) { + throw new DeserializationException("Cannot deserialize AlternateSpanList with type " + type); + } + int numSubTrees = buf.getInt1_2_4Bytes(); + + for (int i = 0; i < numSubTrees; i++) { + double prob = buf.getDouble(); + List<SpanNode> list = readSpanList(altSpanList); + + if (i == 0) { + for (SpanNode node : list) { + altSpanList.add(node); + } + altSpanList.setProbability(0, prob); + } else { + altSpanList.addChildren(i, list, prob); + } + } + } + + private List<SpanNode> readSpanList(SpanNodeParent parent) { + int size = buf.getInt1_2_4Bytes(); + List<SpanNode> spanList = new ArrayList<SpanNode>(); + for (int i = 0; i < size; i++) { + spanList.add(readSpanNode()); + } + return spanList; + } + +} diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java index 40aec94aec6..1e90395a153 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java @@ -1,14 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.serialization; -import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.DocumentUpdate; -import com.yahoo.document.datatypes.BoolFieldValue; -import com.yahoo.document.fieldpathupdate.FieldPathUpdate; -import com.yahoo.document.update.FieldUpdate; import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.vespa.objects.FieldBase; /** * Class used for de-serializing documents on the current head document format. @@ -16,36 +10,10 @@ import com.yahoo.vespa.objects.FieldBase; * @author baldersheim */ @SuppressWarnings("deprecation") -public class VespaDocumentDeserializerHead extends VespaDocumentDeserializer42 { +public class VespaDocumentDeserializerHead extends VespaDocumentDeserializer6 { public VespaDocumentDeserializerHead(DocumentTypeManager manager, GrowableByteBuffer buffer) { super(manager, buffer); } - @Override - public void read(DocumentUpdate update) { - update.setId(new DocumentId(this)); - update.setDocumentType(readDocumentType()); - - int size = getInt(null); - - for (int i = 0; i < size; i++) { - update.addFieldUpdate(new FieldUpdate(this, update.getDocumentType(), 8)); - } - - int sizeAndFlags = getInt(null); - update.setCreateIfNonExistent(DocumentUpdateFlags.extractFlags(sizeAndFlags).getCreateIfNonExistent()); - size = DocumentUpdateFlags.extractValue(sizeAndFlags); - - for (int i = 0; i < size; i++) { - int type = getByte(null); - update.addFieldPathUpdate(FieldPathUpdate.create(FieldPathUpdate.Type.valueOf(type), - update.getDocumentType(), this)); - } - } - - @Override - public void read(FieldBase field, BoolFieldValue value) { - value.setBoolean((getByte(null) != 0)); - } } diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer42.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer42.java index 581c7df8aee..ebe9a124033 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer42.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer42.java @@ -67,11 +67,10 @@ import static com.yahoo.text.Utf8.calculateBytePositions; * @deprecated use {@link com.yahoo.document.serialization.VespaDocumentSerializerHead} instead for new code * @author baldersheim */ -@Deprecated // OK: Don't remove on Vespa 6: Mail may have documents on this format still +@Deprecated // TODO: Remove on Vespa 8 // When removing: Move content into VespaDocumentSerializerHead public class VespaDocumentSerializer42 extends BufferSerializer implements DocumentSerializer { - private boolean headerOnly; private int spanNodeCounter = -1; private int[] bytePositions; @@ -83,55 +82,45 @@ public class VespaDocumentSerializer42 extends BufferSerializer implements Docum super(); } - VespaDocumentSerializer42(GrowableByteBuffer buf, boolean headerOnly) { - this(buf); - this.headerOnly = headerOnly; - } - - public void setHeaderOnly(boolean headerOnly) { - this.headerOnly = headerOnly; - } - public void write(Document doc) { write(new Field(doc.getDataType().getName(), 0, doc.getDataType(), true), doc); } public void write(FieldBase field, Document doc) { - //save the starting position in the buffer - int startPos = buf.position(); - buf.putShort(Document.SERIALIZED_VERSION); + //save the starting position in the buffer + int lenPos = buf.position(); // Temporary length, fill in after serialization is done. buf.putInt(0); doc.getId().serialize(this); + Struct head = doc.getHeader(); + Struct body = doc.getBody(); + boolean hasHead = (head.getFieldCount() != 0); + boolean hasBody = (body.getFieldCount() != 0); + byte contents = 0x01; // Indicating we have document type which we always have - if (doc.getHeader().getFieldCount() > 0) { + if (hasHead) { contents |= 0x2; // Indicate we have header } - if (!headerOnly && doc.getBody().getFieldCount() > 0) { + if (hasBody) { contents |= 0x4; // Indicate we have a body } buf.put(contents); doc.getDataType().serialize(this); - - if (doc.getHeader().getFieldCount() > 0) { - doc.getHeader().serialize(doc.getDataType().getField("header"), this); + if (hasHead) { + head.serialize(null, this); } - - if (!headerOnly && doc.getBody().getFieldCount() > 0) { - doc.getBody().serialize(doc.getDataType().getField("body"), this); + if (hasBody) { + body.serialize(null, this); } - int finalPos = buf.position(); - - buf.position(startPos + 2); - buf.putInt(finalPos - startPos - 2 - 4); // Don't include the length itself or the version + buf.position(lenPos); + buf.putInt(finalPos - lenPos - 4); // Don't include the length itself or the version buf.position(finalPos); - } /** diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java new file mode 100644 index 00000000000..7b7878083a2 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java @@ -0,0 +1,709 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.compress.Compressor; + +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.Field; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.document.annotation.AlternateSpanList; +import com.yahoo.document.annotation.Annotation; +import com.yahoo.document.annotation.AnnotationReference; +import com.yahoo.document.annotation.Span; +import com.yahoo.document.annotation.SpanList; +import com.yahoo.document.annotation.SpanNode; +import com.yahoo.document.annotation.SpanTree; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.BoolFieldValue; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.ReferenceFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.FieldPathUpdate; +import com.yahoo.document.predicate.BinaryFormat; +import com.yahoo.document.update.AddValueUpdate; +import com.yahoo.document.update.ArithmeticValueUpdate; +import com.yahoo.document.update.AssignValueUpdate; +import com.yahoo.document.update.ClearValueUpdate; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.document.update.MapValueUpdate; +import com.yahoo.document.update.RemoveValueUpdate; +import com.yahoo.document.update.ValueUpdate; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.tensor.serialization.TypedBinaryFormat; +import com.yahoo.vespa.objects.BufferSerializer; +import com.yahoo.vespa.objects.FieldBase; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; + +import static com.yahoo.text.Utf8.calculateBytePositions; + +/** + * Class used for serializing documents on the Vespa 6.x document format. + * + * @author baldersheim + **/ +public class VespaDocumentSerializer6 extends BufferSerializer implements DocumentSerializer { + + private int spanNodeCounter = -1; + private int[] bytePositions; + + VespaDocumentSerializer6(GrowableByteBuffer buf) { + super(buf); + } + + public void write(Document doc) { + write(new Field(doc.getDataType().getName(), 0, doc.getDataType(), true), doc); + } + + @SuppressWarnings("deprecation") + public void write(FieldBase field, Document doc) { + buf.putShort(Document.SERIALIZED_VERSION); + + //save the position of the length in the buffer + int lenPos = buf.position(); + // Temporary length, fill in after serialization is done. + buf.putInt(0); + + doc.getId().serialize(this); + + Struct head = doc.getHeader(); + Struct body = doc.getBody(); + boolean hasHead = (head.getFieldCount() != 0); + boolean hasBody = (body.getFieldCount() != 0); + + byte contents = 0x01; // Indicating we have document type which we always have + if (hasHead) { + contents |= 0x2; // Indicate we have header + } + if (hasBody) { + contents |= 0x4; // Indicate we have a body + } + buf.put(contents); + + doc.getDataType().serialize(this); + if (hasHead) { + head.serialize(null, this); + } + if (hasBody) { + body.serialize(null, this); + } + int finalPos = buf.position(); + buf.position(lenPos); + buf.putInt(finalPos - lenPos - 4); // Don't include the length itself or the version + buf.position(finalPos); + } + + /** + * Write out the value of field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, FieldValue value) { + throw new IllegalArgumentException("Not Implemented"); + } + + /** + * Write out the value of array field + * + * @param field - field description (name and data type) + * @param array - field value + */ + public <T extends FieldValue> void write(FieldBase field, Array<T> array) { + buf.putInt1_2_4Bytes(array.size()); + + List<T> lst = array.getValues(); + for (FieldValue value : lst) { + value.serialize(this); + } + + } + + public <K extends FieldValue, V extends FieldValue> void write(FieldBase field, MapFieldValue<K, V> map) { + buf.putInt1_2_4Bytes(map.size()); + for (Map.Entry<K, V> e : map.entrySet()) { + e.getKey().serialize(this); + e.getValue().serialize(this); + } + } + + /** + * Write out the value of byte field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, ByteFieldValue value) { + buf.put(value.getByte()); + } + + @Override + public void write(FieldBase field, BoolFieldValue value) { + byte asByte = value.getBoolean() ? (byte)1 : (byte)0; + buf.put(asByte); + } + + /** + * Write out the value of collection field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public <T extends FieldValue> void write(FieldBase field, CollectionFieldValue<T> value) { + throw new IllegalArgumentException("Not Implemented"); + } + + /** + * Write out the value of double field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, DoubleFieldValue value) { + buf.putDouble(value.getDouble()); + } + + /** + * Write out the value of float field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, FloatFieldValue value) { + buf.putFloat(value.getFloat()); + } + + /** + * Write out the value of integer field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, IntegerFieldValue value) { + buf.putInt(value.getInteger()); + } + + /** + * Write out the value of long field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, LongFieldValue value) { + buf.putLong(value.getLong()); + } + + /** + * Write out the value of raw field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, Raw value) { + ByteBuffer rawBuf = value.getByteBuffer(); + int origPos = rawBuf.position(); + buf.putInt(rawBuf.remaining()); + buf.put(rawBuf); + rawBuf.position(origPos); + + } + + @Override + public void write(FieldBase field, PredicateFieldValue value) { + byte[] buf = BinaryFormat.encode(value.getPredicate()); + this.buf.putInt(buf.length); + this.buf.put(buf); + } + + /** + * Write out the value of string field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, StringFieldValue value) { + byte[] stringBytes = createUTF8CharArray(value.getString()); + + byte coding = 0; + //Use bit 6 of "coding" to say whether span tree is available or not + if (!value.getSpanTrees().isEmpty()) { + coding |= 64; + } + buf.put(coding); + buf.putInt1_4Bytes(stringBytes.length + 1); + + buf.put(stringBytes); + buf.put(((byte) 0)); + + Map<String, SpanTree> trees = value.getSpanTreeMap(); + if ((trees != null) && !trees.isEmpty()) { + try { + //we don't support serialization of nested span trees, so this is safe: + bytePositions = calculateBytePositions(value.getString()); + //total length. record position and go back here if necessary: + int posBeforeSize = buf.position(); + buf.putInt(0); + buf.putInt1_2_4Bytes(trees.size()); + + for (SpanTree tree : trees.values()) { + try { + write(tree); + } catch (SerializationException e) { + throw e; + } catch (RuntimeException e) { + throw new SerializationException("Exception thrown while serializing span tree '" + + tree.getName() + "'; string='" + value.getString() + "'", e); + } + } + int endPos = buf.position(); + buf.position(posBeforeSize); + buf.putInt(endPos - posBeforeSize - 4); //length shall exclude itself + buf.position(endPos); + } finally { + bytePositions = null; + } + } + } + + @Override + public void write(FieldBase field, TensorFieldValue value) { + if (value.getTensor().isPresent()) { + byte[] encodedTensor = TypedBinaryFormat.encode(value.getTensor().get()); + buf.putInt1_4Bytes(encodedTensor.length); + buf.put(encodedTensor); + } else { + buf.putInt1_4Bytes(0); + } + } + + @Override + public void write(FieldBase field, ReferenceFieldValue value) { + if (value.getDocumentId().isPresent()) { + // We piggyback on DocumentId's existing serialization code, but need to know + // whether or not it's present or merely the empty string. + buf.put((byte)1); + write(value.getDocumentId().get()); + } else { + buf.put((byte)0); + } + } + + /** + * Write out the value of struct field + * + * @param field - field description (name and data type) + * @param s - field value + */ + public void write(FieldBase field, Struct s) { + // Serialize all parts first.. As we need to know length before starting + // Serialize all the fields. + + //keep the buffer we're serializing everything into: + GrowableByteBuffer bigBuffer = buf; + + //create a new buffer and serialize into that for a while: + GrowableByteBuffer buffer = new GrowableByteBuffer(4096, 2.0f); + buf = buffer; + + List<Integer> fieldIds = new LinkedList<>(); + List<java.lang.Integer> fieldLengths = new LinkedList<>(); + + for (Map.Entry<Field, FieldValue> value : s.getFields()) { + + int startPos = buffer.position(); + value.getValue().serialize(value.getKey(), this); + + fieldLengths.add(buffer.position() - startPos); + fieldIds.add(value.getKey().getId(s.getVersion())); + } + + // Switch buffers again: + buffer.flip(); + buf = bigBuffer; + + int uncompressedSize = buffer.remaining(); + Compressor.Compression compression = + s.getDataType().getCompressor().compress(buffer.getByteBuffer().array(), buffer.remaining()); + + // Actual serialization starts here. + int lenPos = buf.position(); + putInt(null, 0); // Move back to this after compression is done. + buf.put(compression.type().getCode()); + + if (compression.data() != null && compression.type().isCompressed()) { + buf.putInt2_4_8Bytes(uncompressedSize); + } + + buf.putInt1_4Bytes(s.getFieldCount()); + + for (int i = 0; i < s.getFieldCount(); ++i) { + putInt1_4Bytes(null, fieldIds.get(i)); + putInt2_4_8Bytes(null, fieldLengths.get(i)); + } + + int pos = buf.position(); + if (compression.data() != null && compression.type().isCompressed()) { + put(null, compression.data()); + } else { + put(null, buffer.getByteBuffer()); + } + int dataLength = buf.position() - pos; + + int posNow = buf.position(); + buf.position(lenPos); + putInt(null, dataLength); + buf.position(posNow); + } + + /** + * Write out the value of structured field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, StructuredFieldValue value) { + throw new IllegalArgumentException("Not Implemented"); + } + + /** + * Write out the value of weighted set field + * + * @param field - field description (name and data type) + * @param ws - field value + */ + public <T extends FieldValue> void write(FieldBase field, WeightedSet<T> ws) { + WeightedSetDataType type = ws.getDataType(); + putInt(null, type.getNestedType().getId()); + putInt(null, ws.size()); + + Iterator<T> it = ws.fieldValueIterator(); + while (it.hasNext()) { + FieldValue key = it.next(); + java.lang.Integer value = ws.get(key); + int sizePos = buf.position(); + putInt(null, 0); + int startPos = buf.position(); + key.serialize(this); + putInt(null, value); + int finalPos = buf.position(); + int size = finalPos - startPos; + buf.position(sizePos); + putInt(null, size); + buf.position(finalPos); + } + + } + + public void write(FieldBase field, AnnotationReference value) { + int annotationId = value.getReference().getScratchId(); + if (annotationId >= 0) { + buf.putInt1_2_4Bytes(annotationId); + } else { + throw new SerializationException("Could not serialize AnnotationReference value, reference not found (" + value + ")"); + } + } + + public void write(DocumentId id) { + put(null, id.getScheme().toUtf8().getBytes()); + putByte(null, (byte) 0); + } + + public void write(DocumentType type) { + byte[] docType = createUTF8CharArray(type.getName()); + put(null, docType); + putByte(null, ((byte) 0)); + putShort(null, (short) 0); // Used to hold the version. Is now always 0. + } + + public void write(Annotation annotation) { + buf.putInt(annotation.getType().getId()); //name hash + + byte features = 0; + if (annotation.isSpanNodeValid()) { + features |= ((byte) 1); + } + if (annotation.hasFieldValue()) { + features |= ((byte) 2); + } + buf.put(features); + + int posBeforeSize = buf.position(); + buf.putInt1_2_4BytesAs4(0); + + //write ID of span node: + if (annotation.isSpanNodeValid()) { + int spanNodeId = annotation.getSpanNode().getScratchId(); + if (spanNodeId >= 0) { + buf.putInt1_2_4Bytes(spanNodeId); + } else { + throw new SerializationException("Could not serialize annotation, associated SpanNode not found (" + annotation + ")"); + } + } + + //write annotation value: + if (annotation.hasFieldValue()) { + buf.putInt(annotation.getType().getDataType().getId()); + annotation.getFieldValue().serialize(this); + } + + int end = buf.position(); + buf.position(posBeforeSize); + buf.putInt1_2_4BytesAs4(end - posBeforeSize - 4); + buf.position(end); + } + + public void write(SpanTree tree) { + //we don't support serialization of nested span trees: + if (spanNodeCounter >= 0) { + throw new SerializationException("Serialization of nested SpanTrees is not supported."); + } + + //we're going to write a new SpanTree, create a new Map for nodes: + spanNodeCounter = 0; + + //make sure tree is consistent before continuing: + tree.cleanup(); + + try { + new StringFieldValue(tree.getName()).serialize(this); + + write(tree.getRoot()); + { + //add all annotations to temporary list and sort it, to get predictable serialization + List<Annotation> tmpAnnotationList = new ArrayList<Annotation>(tree.numAnnotations()); + for (Annotation annotation : tree) { + tmpAnnotationList.add(annotation); + } + Collections.sort(tmpAnnotationList); + + int annotationCounter = 0; + //add all annotations to map here, in case of back-references: + for (Annotation annotation : tmpAnnotationList) { + annotation.setScratchId(annotationCounter++); + } + + buf.putInt1_2_4Bytes(tmpAnnotationList.size()); + for (Annotation annotation : tmpAnnotationList) { + write(annotation); + } + } + } finally { + //we're done, let's set these to null to save memory and prevent madness: + spanNodeCounter = -1; + } + } + + public void write(SpanNode spanNode) { + if (spanNodeCounter >= 0) { + spanNode.setScratchId(spanNodeCounter++); + } + if (spanNode instanceof Span) { + write((Span) spanNode); + } else if (spanNode instanceof AlternateSpanList) { + write((AlternateSpanList) spanNode); + } else if (spanNode instanceof SpanList) { + write((SpanList) spanNode); + } else { + throw new IllegalStateException("BUG!! Unable to serialize " + spanNode); + } + } + + public void write(Span span) { + buf.put(Span.ID); + + if (bytePositions != null) { + int byteFrom = bytePositions[span.getFrom()]; + int byteLength = bytePositions[span.getFrom() + span.getLength()] - byteFrom; + + buf.putInt1_2_4Bytes(byteFrom); + buf.putInt1_2_4Bytes(byteLength); + } else { + throw new SerializationException("Cannot serialize Span " + span + ", no access to parent StringFieldValue."); + } + } + + public void write(SpanList spanList) { + buf.put(SpanList.ID); + buf.putInt1_2_4Bytes(spanList.numChildren()); + Iterator<SpanNode> children = spanList.childIterator(); + while (children.hasNext()) { + write(children.next()); + } + } + + public void write(AlternateSpanList altSpanList) { + buf.put(AlternateSpanList.ID); + buf.putInt1_2_4Bytes(altSpanList.getNumSubTrees()); + for (int i = 0; i < altSpanList.getNumSubTrees(); i++) { + buf.putDouble(altSpanList.getProbability(i)); + buf.putInt1_2_4Bytes(altSpanList.numChildren(i)); + Iterator<SpanNode> children = altSpanList.childIterator(i); + while (children.hasNext()) { + write(children.next()); + } + } + } + + @Override + public void write(DocumentUpdate update) { + update.getId().serialize(this); + + update.getDocumentType().serialize(this); + + putInt(null, update.fieldUpdates().size()); + + for (FieldUpdate up : update.fieldUpdates()) { + up.serialize(this); + } + + DocumentUpdateFlags flags = new DocumentUpdateFlags(); + flags.setCreateIfNonExistent(update.getCreateIfNonExistent()); + putInt(null, flags.injectInto(update.fieldPathUpdates().size())); + + for (FieldPathUpdate up : update.fieldPathUpdates()) { + up.serialize(this); + } + } + + public void write(FieldPathUpdate update) { + putByte(null, (byte)update.getUpdateType().getCode()); + put(null, update.getOriginalFieldPath()); + put(null, update.getOriginalWhereClause()); + } + + public void write(AssignFieldPathUpdate update) { + write((FieldPathUpdate)update); + byte flags = 0; + if (update.getRemoveIfZero()) { + flags |= AssignFieldPathUpdate.REMOVE_IF_ZERO; + } + if (update.getCreateMissingPath()) { + flags |= AssignFieldPathUpdate.CREATE_MISSING_PATH; + } + if (update.isArithmetic()) { + flags |= AssignFieldPathUpdate.ARITHMETIC_EXPRESSION; + putByte(null, flags); + put(null, update.getExpression()); + } else { + putByte(null, flags); + update.getFieldValue().serialize(this); + } + } + + public void write(AddFieldPathUpdate update) { + write((FieldPathUpdate)update); + update.getNewValues().serialize(this); + } + + @Override + public void write(FieldUpdate update) { + putInt(null, update.getField().getId(Document.SERIALIZED_VERSION)); + putInt(null, update.getValueUpdates().size()); + for (ValueUpdate vupd : update.getValueUpdates()) { + putInt(null, vupd.getValueUpdateClassID().id); + vupd.serialize(this, update.getField().getDataType()); + } + } + + @Override + public void write(AddValueUpdate update, DataType superType) { + writeValue(this, ((CollectionDataType)superType).getNestedType(), update.getValue()); + putInt(null, update.getWeight()); + } + + @Override + public void write(MapValueUpdate update, DataType superType) { + if (superType instanceof ArrayDataType) { + CollectionDataType type = (CollectionDataType) superType; + IntegerFieldValue index = (IntegerFieldValue) update.getValue(); + index.serialize(this); + putInt(null, update.getUpdate().getValueUpdateClassID().id); + update.getUpdate().serialize(this, type.getNestedType()); + } else if (superType instanceof WeightedSetDataType) { + writeValue(this, ((CollectionDataType)superType).getNestedType(), update.getValue()); + putInt(null, update.getUpdate().getValueUpdateClassID().id); + update.getUpdate().serialize(this, DataType.INT); + } else { + throw new SerializationException("MapValueUpdate only works for arrays and weighted sets"); + } + } + + @Override + public void write(ArithmeticValueUpdate update) { + putInt(null, update.getOperator().id); + putDouble(null, update.getOperand().doubleValue()); + } + + @Override + public void write(AssignValueUpdate update, DataType superType) { + if (update.getValue() == null) { + putByte(null, (byte) 0); + } else { + putByte(null, (byte) 1); + writeValue(this, superType, update.getValue()); + } + } + + @Override + public void write(RemoveValueUpdate update, DataType superType) { + writeValue(this, ((CollectionDataType)superType).getNestedType(), update.getValue()); + } + + @Override + public void write(ClearValueUpdate clearValueUpdate, DataType superType) { + //TODO: This has never ever been implemented. Has this ever worked? + } + + /** + * Returns the serialized size of the given {@link Document}. Please note that this method performs actual + * serialization of the document, but simply return the size of the final {@link GrowableByteBuffer}. If you need + * the buffer itself, do NOT use this method. + * + * @param doc The Document whose size to calculate. + * @return The size in bytes. + */ + public static long getSerializedSize(Document doc) { + DocumentSerializer serializer = new VespaDocumentSerializer6(new GrowableByteBuffer()); + serializer.write(doc); + return serializer.getBuf().position(); + } + + private static void writeValue(VespaDocumentSerializer6 serializer, DataType dataType, Object value) { + FieldValue fieldValue; + if (value instanceof FieldValue) { + fieldValue = (FieldValue)value; + } else { + fieldValue = dataType.createFieldValue(value); + } + fieldValue.serialize(serializer); + } + +} diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializerHead.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializerHead.java index 92bce41ba8c..bfa746a02b1 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializerHead.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializerHead.java @@ -1,80 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.serialization; -import com.yahoo.document.DocumentUpdate; -import com.yahoo.document.datatypes.ByteFieldValue; -import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate; -import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate; -import com.yahoo.document.fieldpathupdate.FieldPathUpdate; -import com.yahoo.document.update.FieldUpdate; import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.vespa.objects.FieldBase; /** * Class used for serializing documents on the current head document format. * * @author baldersheim */ -@SuppressWarnings("deprecation") -public class VespaDocumentSerializerHead extends VespaDocumentSerializer42 { +public class VespaDocumentSerializerHead extends VespaDocumentSerializer6 { public VespaDocumentSerializerHead(GrowableByteBuffer buf) { super(buf); } - @Override - public void write(DocumentUpdate update) { - update.getId().serialize(this); - - update.getDocumentType().serialize(this); - - putInt(null, update.fieldUpdates().size()); - - for (FieldUpdate up : update.fieldUpdates()) { - up.serialize(this); - } - - DocumentUpdateFlags flags = new DocumentUpdateFlags(); - flags.setCreateIfNonExistent(update.getCreateIfNonExistent()); - putInt(null, flags.injectInto(update.fieldPathUpdates().size())); - - for (FieldPathUpdate up : update.fieldPathUpdates()) { - up.serialize(this); - } - } - - public void write(FieldPathUpdate update) { - putByte(null, (byte)update.getUpdateType().getCode()); - put(null, update.getOriginalFieldPath()); - put(null, update.getOriginalWhereClause()); - } - - public void write(AssignFieldPathUpdate update) { - write((FieldPathUpdate)update); - byte flags = 0; - if (update.getRemoveIfZero()) { - flags |= AssignFieldPathUpdate.REMOVE_IF_ZERO; - } - if (update.getCreateMissingPath()) { - flags |= AssignFieldPathUpdate.CREATE_MISSING_PATH; - } - if (update.isArithmetic()) { - flags |= AssignFieldPathUpdate.ARITHMETIC_EXPRESSION; - putByte(null, flags); - put(null, update.getExpression()); - } else { - putByte(null, flags); - update.getFieldValue().serialize(this); - } - } - - public void write(AddFieldPathUpdate update) { - write((FieldPathUpdate)update); - update.getNewValues().serialize(this); - } - - @Override - public void write(FieldBase field, ByteFieldValue value) { - buf.put(value.getByte()); - } } diff --git a/document/src/main/java/com/yahoo/document/update/FieldUpdate.java b/document/src/main/java/com/yahoo/document/update/FieldUpdate.java index 163bda5b623..192afba80b3 100644 --- a/document/src/main/java/com/yahoo/document/update/FieldUpdate.java +++ b/document/src/main/java/com/yahoo/document/update/FieldUpdate.java @@ -291,7 +291,7 @@ public class FieldUpdate { } public final void serialize(GrowableByteBuffer buf) { - serialize(DocumentSerializerFactory.create42(buf)); + serialize(DocumentSerializerFactory.create6(buf)); } public void serialize(DocumentUpdateWriter data) { diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/package-info.java b/document/src/main/java/com/yahoo/vespaxmlparser/package-info.java index ba33b6a83ad..68116e777d6 100644 --- a/document/src/main/java/com/yahoo/vespaxmlparser/package-info.java +++ b/document/src/main/java/com/yahoo/vespaxmlparser/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// TODO: Remove this package on Vespa 7 +// TODO: Remove this package on Vespa 8 @ExportPackage package com.yahoo.vespaxmlparser; diff --git a/document/src/test/java/com/yahoo/document/DocInDocTestCase.java b/document/src/test/java/com/yahoo/document/DocInDocTestCase.java index cdc868af987..f5c158595d2 100644 --- a/document/src/test/java/com/yahoo/document/DocInDocTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocInDocTestCase.java @@ -37,13 +37,13 @@ public class DocInDocTestCase { Document outer = new Document(manager.getDocumentType("outerdoc"), "doc:outer:the:only:one"); outer.setFieldValue("innerdocuments", innerArray); - DocumentSerializer serializer = DocumentSerializerFactory.create42(); + DocumentSerializer serializer = DocumentSerializerFactory.create6(); serializer.write(outer); GrowableByteBuffer buf = serializer.getBuf(); buf.flip(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(manager, buf); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(manager, buf); Document outerDeserialized = new Document(deserializer); assertEquals(outer, outerDeserialized); diff --git a/document/src/test/java/com/yahoo/document/DocumentPathUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentPathUpdateTestCase.java index ce55a5a501c..d5fc1386900 100755 --- a/document/src/test/java/com/yahoo/document/DocumentPathUpdateTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentPathUpdateTestCase.java @@ -581,9 +581,9 @@ public class DocumentPathUpdateTestCase { docUp.addFieldPathUpdate(ass); GrowableByteBuffer buffer = new GrowableByteBuffer(); - docUp.serialize(DocumentSerializerFactory.createHead(buffer)); + docUp.serialize(DocumentSerializerFactory.create6(buffer)); buffer.flip(); - DocumentUpdate docUp2 = new DocumentUpdate(DocumentDeserializerFactory.createHead(docMan, buffer)); + DocumentUpdate docUp2 = new DocumentUpdate(DocumentDeserializerFactory.create6(docMan, buffer)); assertEquals(docUp, docUp2); } @@ -599,9 +599,9 @@ public class DocumentPathUpdateTestCase { docUp.addFieldPathUpdate(add); GrowableByteBuffer buffer = new GrowableByteBuffer(); - docUp.serialize(DocumentSerializerFactory.createHead(buffer)); + docUp.serialize(DocumentSerializerFactory.create6(buffer)); buffer.flip(); - DocumentUpdate docUp2 = new DocumentUpdate(DocumentDeserializerFactory.createHead(docMan, buffer)); + DocumentUpdate docUp2 = new DocumentUpdate(DocumentDeserializerFactory.create6(docMan, buffer)); assertEquals(docUp, docUp2); } @@ -613,9 +613,9 @@ public class DocumentPathUpdateTestCase { docUp.addFieldPathUpdate(remove); GrowableByteBuffer buffer = new GrowableByteBuffer(); - docUp.serialize(DocumentSerializerFactory.createHead(buffer)); + docUp.serialize(DocumentSerializerFactory.create6(buffer)); buffer.flip(); - DocumentUpdate docUp2 = new DocumentUpdate(DocumentDeserializerFactory.createHead(docMan, buffer)); + DocumentUpdate docUp2 = new DocumentUpdate(DocumentDeserializerFactory.create6(docMan, buffer)); assertEquals(docUp, docUp2); } @@ -658,7 +658,7 @@ public class DocumentPathUpdateTestCase { DocumentUpdate docUp = createDocumentUpdateForSerialization(); GrowableByteBuffer buffer = new GrowableByteBuffer(); - docUp.serialize(DocumentSerializerFactory.createHead(buffer)); + docUp.serialize(DocumentSerializerFactory.create6(buffer)); int size = buffer.position(); buffer.position(0); @@ -672,7 +672,7 @@ public class DocumentPathUpdateTestCase { public void testReadSerializedFile() throws IOException { docMan = DocumentTestCase.setUpCppDocType(); byte[] data = DocumentTestCase.readFile("src/tests/data/serialize-fieldpathupdate-cpp.dat"); - DocumentDeserializer buf = DocumentDeserializerFactory.createHead(docMan, GrowableByteBuffer.wrap(data)); + DocumentDeserializer buf = DocumentDeserializerFactory.create6(docMan, GrowableByteBuffer.wrap(data)); DocumentUpdate upd = new DocumentUpdate(buf); diff --git a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java index 7d1992225e4..8ff168adc4b 100644 --- a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java @@ -170,7 +170,7 @@ public class DocumentSerializationTestCase extends AbstractTypesTest { } System.err.println("Checking doc from file " + test.testFile); - Document doc = new Document(DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(buffer))); + Document doc = new Document(DocumentDeserializerFactory.create6(docMan, GrowableByteBuffer.wrap(buffer))); System.err.println("Id: " + doc.getId()); @@ -214,7 +214,7 @@ public class DocumentSerializationTestCase extends AbstractTypesTest { doc.setFieldValue("friend", 2384L); GrowableByteBuffer buffer = new GrowableByteBuffer(1024); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); serializer.write(doc); buffer.flip(); @@ -222,7 +222,7 @@ public class DocumentSerializationTestCase extends AbstractTypesTest { fos.write(buffer.array(), 0, buffer.limit()); fos.close(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(man, buffer); Document doc2 = new Document(deserializer); assertEquals(doc, doc2); diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTestCase.java index 3eebc4396e8..28267a22a56 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTestCase.java @@ -708,54 +708,6 @@ public class DocumentTestCase extends DocumentTestCaseBase { } @Test - public void testCppDocSplit() throws IOException { - docMan = setUpCppDocType(); - byte[] headerData = readFile("src/test/document/serializecppsplit_header.dat"); - byte[] bodyData = readFile("src/test/document/serializecppsplit_body.dat"); - - DocumentDeserializer header = DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(headerData), - GrowableByteBuffer.wrap(bodyData)); - - Document doc = new Document(header); - - assertEquals("doc:serializetest:http://test.doc.id/", doc.getId().toString()); - assertEquals(new IntegerFieldValue(5), doc.getFieldValue("intfield")); - assertEquals(new FloatFieldValue((float)-9.23), doc.getFieldValue("floatfield")); - assertEquals(new StringFieldValue("This is a string."), doc.getFieldValue("stringfield")); - assertEquals(new LongFieldValue(398420092938472983L), doc.getFieldValue("longfield")); - assertEquals(new DoubleFieldValue(98374532.398820d), doc.getFieldValue("doublefield")); - assertEquals(new StringFieldValue("http://this.is.a.test/"), doc.getFieldValue("urifield")); - //NOTE: The value really is unsigned 254, which becomes signed -2: - assertEquals(new ByteFieldValue((byte)-2), doc.getFieldValue("bytefield")); - ByteBuffer raw = ByteBuffer.wrap("RAW DATA".getBytes()); - assertEquals(new Raw(raw), doc.getFieldValue("rawfield")); - - Document docindoc = (Document)doc.getFieldValue("docfield"); - assertEquals(docMan.getDocumentType("docindoc"), docindoc.getDataType()); - assertEquals(new DocumentId("doc:docindoc:http://embedded"), docindoc.getId()); - - WeightedSet wset = (WeightedSet)doc.getFieldValue("wsfield"); - assertEquals(Integer.valueOf(50), wset.get(new StringFieldValue("Weighted 0"))); - assertEquals(Integer.valueOf(199), wset.get(new StringFieldValue("Weighted 1"))); - } - - @Test - public void testCppDocSplitNoBody() throws IOException { - docMan = setUpCppDocType(); - byte[] headerData = readFile("src/test/document/serializecppsplit_header.dat"); - - DocumentDeserializer header = DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(headerData)); - - Document doc = new Document(header); - - assertEquals("doc:serializetest:http://test.doc.id/", doc.getId().toString()); - assertEquals(new FloatFieldValue((float)-9.23), doc.getFieldValue("floatfield")); - assertEquals(new StringFieldValue("This is a string."), doc.getFieldValue("stringfield")); - assertEquals(new LongFieldValue(398420092938472983L), doc.getFieldValue("longfield")); - assertEquals(new StringFieldValue("http://this.is.a.test/"), doc.getFieldValue("urifield")); - } - - @Test @SuppressWarnings("deprecation") public void testGenerateSerializedFile() throws IOException { @@ -862,48 +814,6 @@ public class DocumentTestCase extends DocumentTestCaseBase { } assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield")); assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield")); - // Do the same thing, splitting document in two - DocumentSerializer header = DocumentSerializerFactory.create42(new GrowableByteBuffer(), true); - DocumentSerializer body = DocumentSerializerFactory.create42(new GrowableByteBuffer()); - doc.serializeHeader(header); - doc.serializeBody(body); - header.getBuf().flip(); - body.getBuf().flip(); - - try { - FileOutputStream fos = new FileOutputStream("src/test/files/testser-split.header.dat"); - fos.write(header.getBuf().array(), 0, header.getBuf().remaining()); - fos.close(); - fos = new FileOutputStream("src/test/files/testser-split.body.dat"); - fos.write(body.getBuf().array(), 0, body.getBuf().remaining()); - fos.close(); - } catch (Exception e) { - } - - DocumentDeserializer deser = DocumentDeserializerFactory.create42(docMan, header.getBuf(), body.getBuf()); - - doc2 = new Document(deser); - - assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid")); - assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date")); - assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from")); - assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to")); - assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject")); - assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body")); - assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount")); - assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments")); - docRawBytes = ((Raw)doc.getFieldValue("rawfield")).getByteBuffer().array(); - doc2RawBytes = ((Raw)doc2.getFieldValue("rawfield")).getByteBuffer().array(); - assertEquals(docRawBytes.length, doc2RawBytes.length); - for (int i = 0; i < docRawBytes.length; i++) { - assertEquals(docRawBytes[i], doc2RawBytes[i]); - } - assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield")); - assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield")); - - Document docInDoc = (Document)doc.getFieldValue("docindoc"); - assert (docInDoc != null); - assertEquals(new StringFieldValue("ball"), docInDoc.getFieldValue("tull")); } @Test @@ -953,45 +863,6 @@ public class DocumentTestCase extends DocumentTestCaseBase { } assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield")); assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield")); - - // Do the same thing, splitting document in two - BufferSerializer header = new BufferSerializer(new GrowableByteBuffer()); - BufferSerializer body = new BufferSerializer(new GrowableByteBuffer()); - doc.serializeHeader(header); - doc.serializeBody(body); - header.getBuf().flip(); - body.getBuf().flip(); - - try { - FileOutputStream fos = new FileOutputStream("src/test/files/testser-split.header.dat"); - fos.write(header.getBuf().array(), 0, header.getBuf().remaining()); - fos.close(); - fos = new FileOutputStream("src/test/files/testser-split.body.dat"); - fos.write(body.getBuf().array(), 0, body.getBuf().remaining()); - fos.close(); - } catch (Exception e) { - } - - DocumentDeserializer deser = DocumentDeserializerFactory.create42(docMan, header.getBuf(), body.getBuf()); - - doc2 = new Document(deser); - - assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid")); - assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date")); - assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from")); - assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to")); - assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject")); - assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body")); - assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount")); - assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments")); - docRawBytes = ((Raw)doc.getFieldValue("rawfield")).getByteBuffer().array(); - doc2RawBytes = ((Raw)doc2.getFieldValue("rawfield")).getByteBuffer().array(); - assertEquals(docRawBytes.length, doc2RawBytes.length); - for (int i = 0; i < docRawBytes.length; i++) { - assertEquals(docRawBytes[i], doc2RawBytes[i]); - } - assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield")); - assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield")); } @Test @@ -1000,7 +871,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { BufferSerializer buf = new BufferSerializer(); try { - new Document(DocumentDeserializerFactory.create42(docMan, buf.getBuf())); + new Document(DocumentDeserializerFactory.create6(docMan, buf.getBuf())); assertTrue(false); } catch (Exception e) { assertTrue(true); @@ -1008,7 +879,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { buf = BufferSerializer.wrap("Hello world".getBytes()); try { - new Document(DocumentDeserializerFactory.create42(docMan, buf.getBuf())); + new Document(DocumentDeserializerFactory.create6(docMan, buf.getBuf())); assertTrue(false); } catch (Exception e) { assertTrue(true); @@ -1450,11 +1321,11 @@ public class DocumentTestCase extends DocumentTestCaseBase { } public void serialize(String docId) { new Document(docType, DocumentId.createFromSerialized(docId)) - .serialize(DocumentSerializerFactory.createHead(buffer)); + .serialize(DocumentSerializerFactory.create6(buffer)); buffer.flip(); } public Document deserialize() { - return new Document(DocumentDeserializerFactory.createHead(docMan, buffer)); + return new Document(DocumentDeserializerFactory.create6(docMan, buffer)); } } diff --git a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java index a9f77cb5eb0..b86094e5a4d 100644 --- a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java @@ -299,13 +299,13 @@ public class DocumentUpdateTestCase { docUp.addFieldUpdate(assignSingle); docUp.addFieldUpdate(addMultiList); - DocumentDeserializer buf = DocumentDeserializerFactory.create42(docMan, new GrowableByteBuffer()); - docUp.serialize((DocumentUpdateWriter)buf); - buf.getBuf().flip(); + GrowableByteBuffer buf = new GrowableByteBuffer(); + docUp.serialize(DocumentSerializerFactory.create42(buf)); + buf.flip(); try { FileOutputStream fos = new FileOutputStream("src/test/files/updateser.dat"); - fos.write(buf.getBuf().array(), 0, buf.getBuf().remaining()); + fos.write(buf.array(), 0, buf.remaining()); fos.close(); } catch (Exception e) { } @@ -330,10 +330,9 @@ public class DocumentUpdateTestCase { + (4 //valueUpdateClassID + (4 + 4 + 4 + (1 + 1 + 2 + 1) + 4 + (1 + 1 + 2 + 1) + 4 + (1 + 1 + 2 + 1))))) //value - , buf.getBuf().remaining()); - - DocumentUpdate docUpDeser = new DocumentUpdate(buf); + , buf.remaining()); + DocumentUpdate docUpDeser = new DocumentUpdate(DocumentDeserializerFactory.create42(docMan, buf)); assertEquals(docUp.getDocumentType(), docUpDeser.getDocumentType()); assertEquals(docUp, docUpDeser); } @@ -402,7 +401,7 @@ public class DocumentUpdateTestCase { upd.addFieldUpdate(serAdd); GrowableByteBuffer buf = new GrowableByteBuffer(100, 2.0f); - upd.serialize(buf); + upd.serialize(DocumentSerializerFactory.create42(buf)); int size = buf.position(); buf.position(0); @@ -630,11 +629,11 @@ public class DocumentUpdateTestCase { public void testRequireThatCreateIfNonExistentFlagIsSerializedAndDeserialized() { docUp.setCreateIfNonExistent(true); - DocumentSerializer serializer = DocumentSerializerFactory.createHead(new GrowableByteBuffer()); + DocumentSerializer serializer = DocumentSerializerFactory.create6(new GrowableByteBuffer()); docUp.serialize(serializer); serializer.getBuf().flip(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.createHead(docMan, serializer.getBuf()); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(docMan, serializer.getBuf()); DocumentUpdate deserialized = new DocumentUpdate(deserializer); assertEquals(docUp, deserialized); assertTrue(deserialized.getCreateIfNonExistent()); @@ -655,11 +654,11 @@ public class DocumentUpdateTestCase { @Test public void testThatAssignValueUpdateForTensorFieldCanBeSerializedAndDeserialized() { DocumentUpdate serializedUpdate = createTensorAssignUpdate(); - DocumentSerializer serializer = DocumentSerializerFactory.createHead(new GrowableByteBuffer()); + DocumentSerializer serializer = DocumentSerializerFactory.create6(new GrowableByteBuffer()); serializedUpdate.serialize(serializer); serializer.getBuf().flip(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.createHead(docMan, serializer.getBuf()); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(docMan, serializer.getBuf()); DocumentUpdate deserializedUpdate = new DocumentUpdate(deserializer); assertEquals(serializedUpdate, deserializedUpdate); } diff --git a/document/src/test/java/com/yahoo/document/annotation/AlternateSpanListTestCase.java b/document/src/test/java/com/yahoo/document/annotation/AlternateSpanListTestCase.java index 47e514d74f5..186766f799b 100755 --- a/document/src/test/java/com/yahoo/document/annotation/AlternateSpanListTestCase.java +++ b/document/src/test/java/com/yahoo/document/annotation/AlternateSpanListTestCase.java @@ -57,7 +57,7 @@ public class AlternateSpanListTestCase extends AbstractTypesTest { GrowableByteBuffer buffer; { buffer = new GrowableByteBuffer(1024); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); StringFieldValue value = new StringFieldValue("lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lk"); SpanTree tree = new SpanTree("bababa", alternateSpanList); value.setSpanTree(tree); @@ -66,7 +66,7 @@ public class AlternateSpanListTestCase extends AbstractTypesTest { } AlternateSpanList alternateSpanList2; { - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(man, buffer); StringFieldValue value = new StringFieldValue(); deserializer.read(null, value); alternateSpanList2 = (AlternateSpanList)value.getSpanTree("bababa").getRoot(); diff --git a/document/src/test/java/com/yahoo/document/annotation/AnnotationTestCase.java b/document/src/test/java/com/yahoo/document/annotation/AnnotationTestCase.java index de674aa3cca..cb04ca1de62 100644 --- a/document/src/test/java/com/yahoo/document/annotation/AnnotationTestCase.java +++ b/document/src/test/java/com/yahoo/document/annotation/AnnotationTestCase.java @@ -115,11 +115,11 @@ public class AnnotationTestCase extends AbstractTypesTest { private void serializeAndAssert(Annotation annotation) { GrowableByteBuffer buffer = new GrowableByteBuffer(1024); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); serializer.write(annotation); buffer.flip(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(man, buffer); Annotation annotation2 = new Annotation(); deserializer.read(annotation2); diff --git a/document/src/test/java/com/yahoo/document/annotation/Bug6394548TestCase.java b/document/src/test/java/com/yahoo/document/annotation/Bug6394548TestCase.java index 18f19030c9e..8b9649d697f 100644 --- a/document/src/test/java/com/yahoo/document/annotation/Bug6394548TestCase.java +++ b/document/src/test/java/com/yahoo/document/annotation/Bug6394548TestCase.java @@ -43,11 +43,11 @@ public class Bug6394548TestCase { String annotationsBefore = dumpAllAnnotations(tree); GrowableByteBuffer buffer = new GrowableByteBuffer(); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); serializer.write(doc); buffer.flip(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(manager, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(manager, buffer); Document doc2 = new Document(deserializer); System.out.println(doc2.toXml()); diff --git a/document/src/test/java/com/yahoo/document/annotation/Bug6425939TestCase.java b/document/src/test/java/com/yahoo/document/annotation/Bug6425939TestCase.java index 8f071650382..82b730ae4ca 100644 --- a/document/src/test/java/com/yahoo/document/annotation/Bug6425939TestCase.java +++ b/document/src/test/java/com/yahoo/document/annotation/Bug6425939TestCase.java @@ -37,13 +37,13 @@ public class Bug6425939TestCase { emptyString.setSpanTree(createSpanTree()); GrowableByteBuffer buffer = new GrowableByteBuffer(1024); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); Field strField = new Field("flarn", DataType.STRING); serializer.write(strField, emptyString); buffer.flip(); // Should not throw exception if bug 6425939 is fixed: - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(man, buffer); StringFieldValue deserializedString = new StringFieldValue(); deserializer.read(strField, deserializedString); diff --git a/document/src/test/java/com/yahoo/document/annotation/SpanListTestCase.java b/document/src/test/java/com/yahoo/document/annotation/SpanListTestCase.java index 887bfec5059..6f338fc47a3 100755 --- a/document/src/test/java/com/yahoo/document/annotation/SpanListTestCase.java +++ b/document/src/test/java/com/yahoo/document/annotation/SpanListTestCase.java @@ -46,7 +46,7 @@ public class SpanListTestCase extends AbstractTypesTest { GrowableByteBuffer buffer; { buffer = new GrowableByteBuffer(1024); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); StringFieldValue value = new StringFieldValue("lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lk"); SpanTree tree = new SpanTree("bababa", spanList); value.setSpanTree(tree); @@ -55,7 +55,7 @@ public class SpanListTestCase extends AbstractTypesTest { } SpanList spanList2; { - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(man, buffer); StringFieldValue value = new StringFieldValue(); deserializer.read(null, value); spanList2 = (SpanList)value.getSpanTree("bababa").getRoot(); diff --git a/document/src/test/java/com/yahoo/document/annotation/SpanTestCase.java b/document/src/test/java/com/yahoo/document/annotation/SpanTestCase.java index 701f6917ecd..23d62cae239 100755 --- a/document/src/test/java/com/yahoo/document/annotation/SpanTestCase.java +++ b/document/src/test/java/com/yahoo/document/annotation/SpanTestCase.java @@ -79,7 +79,7 @@ public class SpanTestCase extends AbstractTypesTest { GrowableByteBuffer buffer; { buffer = new GrowableByteBuffer(1024); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); StringFieldValue value = new StringFieldValue("lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lk"); SpanTree tree = new SpanTree("bababa", span); value.setSpanTree(tree); @@ -88,7 +88,7 @@ public class SpanTestCase extends AbstractTypesTest { } Span span2; { - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(man, buffer); StringFieldValue value = new StringFieldValue(); deserializer.read(null, value); span2 = (Span)value.getSpanTree("bababa").getRoot(); diff --git a/document/src/test/java/com/yahoo/document/annotation/SpanTreeTestCase.java b/document/src/test/java/com/yahoo/document/annotation/SpanTreeTestCase.java index 393fe8b6688..b200e72514d 100755 --- a/document/src/test/java/com/yahoo/document/annotation/SpanTreeTestCase.java +++ b/document/src/test/java/com/yahoo/document/annotation/SpanTreeTestCase.java @@ -865,12 +865,12 @@ public class SpanTreeTestCase extends AbstractTypesTest { GrowableByteBuffer buffer = new GrowableByteBuffer(1024); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); serializer.write(new Field("stringfield", DataType.STRING), strfval); buffer.flip(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(docMan, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(docMan, buffer); StringFieldValue stringFieldValue2 = new StringFieldValue(); deserializer.read(new Field("stringfield", DataType.STRING), stringFieldValue2); diff --git a/document/src/test/java/com/yahoo/document/annotation/SystemTestCase.java b/document/src/test/java/com/yahoo/document/annotation/SystemTestCase.java index ffa71ea7939..9163b773bcf 100755 --- a/document/src/test/java/com/yahoo/document/annotation/SystemTestCase.java +++ b/document/src/test/java/com/yahoo/document/annotation/SystemTestCase.java @@ -124,10 +124,10 @@ public class SystemTestCase { annotate(inDocument); GrowableByteBuffer buffer = new GrowableByteBuffer(); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); serializer.write(inDocument); buffer.flip(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(manager, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(manager, buffer); Document outDocument = new Document(deserializer); consume(outDocument); diff --git a/document/src/test/java/com/yahoo/document/datatypes/MapTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/MapTestCase.java index c4441910f2a..eb47fb1425d 100644 --- a/document/src/test/java/com/yahoo/document/datatypes/MapTestCase.java +++ b/document/src/test/java/com/yahoo/document/datatypes/MapTestCase.java @@ -144,10 +144,10 @@ public class MapTestCase { DocumentTypeManager man = new DocumentTypeManager(); man.register(mapType); GrowableByteBuffer buffer = new GrowableByteBuffer(1024); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); serializer.write(f, map); buffer.flip(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(man, buffer); MapFieldValue<FieldValue, FieldValue> map2 = new MapFieldValue<FieldValue, FieldValue>(mapType); deserializer.read(f, map2); assertNotSame(map, map2); diff --git a/document/src/test/java/com/yahoo/document/datatypes/StringTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/StringTestCase.java index e9fce35ed01..296ab1ac3fc 100644 --- a/document/src/test/java/com/yahoo/document/datatypes/StringTestCase.java +++ b/document/src/test/java/com/yahoo/document/datatypes/StringTestCase.java @@ -50,7 +50,7 @@ public class StringTestCase extends AbstractTypesTest { data.position(0); StringFieldValue tmp = new StringFieldValue(); - DocumentDeserializer deser = DocumentDeserializerFactory.create42(null, data); + DocumentDeserializer deser = DocumentDeserializerFactory.create6(null, data); tmp.deserialize(deser); java.lang.String foo2 = tmp.getString(); @@ -111,7 +111,7 @@ public class StringTestCase extends AbstractTypesTest { tmp = new StringFieldValue(); - deser = DocumentDeserializerFactory.create42(null, data); + deser = DocumentDeserializerFactory.create6(null, data); tmp.deserialize(deser); java.lang.String blah2 = tmp.getString(); @@ -129,7 +129,7 @@ public class StringTestCase extends AbstractTypesTest { data.getBuf().position(0); StringFieldValue tmp = new StringFieldValue(); - DocumentDeserializer deser = DocumentDeserializerFactory.create42(null, data.getBuf()); + DocumentDeserializer deser = DocumentDeserializerFactory.create6(null, data.getBuf()); tmp.deserialize(deser); java.lang.String test2 = tmp.getString(); assertEquals(test, test2); @@ -146,11 +146,11 @@ public class StringTestCase extends AbstractTypesTest { Field f = new Field("text", DataType.STRING); GrowableByteBuffer buffer = new GrowableByteBuffer(1024); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); serializer.write(f, stringFieldValue); buffer.flip(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(man, buffer); StringFieldValue stringFieldValue2 = new StringFieldValue(); deserializer.read(f, stringFieldValue2); @@ -183,7 +183,7 @@ public class StringTestCase extends AbstractTypesTest { innerTree.annotate(innerSpan, new Annotation(type)); GrowableByteBuffer buffer = new GrowableByteBuffer(1024); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); try { serializer.write(null, outerString); @@ -216,11 +216,11 @@ public class StringTestCase extends AbstractTypesTest { private Document serializeAndDeserialize(Document doc, DocumentTypeManager manager) { GrowableByteBuffer buffer = new GrowableByteBuffer(1024); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); serializer.write(doc); buffer.flip(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(manager, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(manager, buffer); return new Document(deserializer); } diff --git a/document/src/test/java/com/yahoo/document/datatypes/blog.sd b/document/src/test/java/com/yahoo/document/datatypes/blog.sd index 545c46cb149..1723e942370 100644 --- a/document/src/test/java/com/yahoo/document/datatypes/blog.sd +++ b/document/src/test/java/com/yahoo/document/datatypes/blog.sd @@ -4,26 +4,19 @@ search blog { document blog { field title type string { - header indexing: summary | index - # index-to: default } field author type string { - header indexing: summary | index - # index-to: default } field body type string { - header indexing: summary | index } field url type uri { - header indexing: index | summary - # index-to: default } } diff --git a/document/src/test/java/com/yahoo/document/docindoc.sd b/document/src/test/java/com/yahoo/document/docindoc.sd index 8b6acd71f47..b05f8d71c89 100644 --- a/document/src/test/java/com/yahoo/document/docindoc.sd +++ b/document/src/test/java/com/yahoo/document/docindoc.sd @@ -1,7 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. search docindoc { document docindoc { - field name type string { header } - field content type string { body } + field name type string { } + field content type string { } } } diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index b3d502ea56f..d82935e3c3b 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -323,7 +323,7 @@ public class JsonReaderTestCase { assertEquals(2, s.getFieldCount()); assertEquals(new StringFieldValue("person"), s.getFieldValue(s.getField("sandra"))); GrowableByteBuffer buf = new GrowableByteBuffer(); - DocumentSerializer serializer = DocumentSerializerFactory.createHead(buf); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buf); put.serialize(serializer); assertEquals(107, buf.position()); } @@ -345,7 +345,7 @@ public class JsonReaderTestCase { Struct s = (Struct) avu.getValue(); assertEquals(0, s.getFieldCount()); GrowableByteBuffer buf = new GrowableByteBuffer(); - DocumentSerializer serializer = DocumentSerializerFactory.createHead(buf); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buf); put.serialize(serializer); assertEquals(69, buf.position()); } diff --git a/document/src/test/java/com/yahoo/document/outerdoc.sd b/document/src/test/java/com/yahoo/document/outerdoc.sd index 6fdbd402ffe..57a1bda323a 100644 --- a/document/src/test/java/com/yahoo/document/outerdoc.sd +++ b/document/src/test/java/com/yahoo/document/outerdoc.sd @@ -1,6 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. search outerdoc { document outerdoc { - field innerdocuments type array<docindoc> { body } + field innerdocuments type array<docindoc> { } } } diff --git a/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java b/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java index 38cf2b0e9d1..25021b0d2f8 100644 --- a/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java +++ b/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java @@ -24,7 +24,7 @@ public class SerializationTestUtils { public static byte[] serializeDocument(Document doc) { GrowableByteBuffer out = new GrowableByteBuffer(); - DocumentSerializerFactory.create42(out).write(doc); + DocumentSerializerFactory.create6(out).write(doc); out.flip(); byte[] buf = new byte[out.remaining()]; out.get(buf); @@ -33,7 +33,7 @@ public class SerializationTestUtils { public static Document deserializeDocument(byte[] buf, TestDocumentFactory factory) { Document document = factory.createDocument(); - DocumentDeserializerFactory.create42(factory.typeManager(), new GrowableByteBuffer(ByteBuffer.wrap(buf))).read(document); + DocumentDeserializerFactory.create6(factory.typeManager(), new GrowableByteBuffer(ByteBuffer.wrap(buf))).read(document); return document; } diff --git a/document/src/test/java/com/yahoo/document/serialization/SerializeAnnotationsTestCase.java b/document/src/test/java/com/yahoo/document/serialization/SerializeAnnotationsTestCase.java index 30db0af6607..653f121b2d4 100644 --- a/document/src/test/java/com/yahoo/document/serialization/SerializeAnnotationsTestCase.java +++ b/document/src/test/java/com/yahoo/document/serialization/SerializeAnnotationsTestCase.java @@ -79,7 +79,7 @@ public class SerializeAnnotationsTestCase { assertEquals(serialized.limit(), serializedFromFile.limit()); StringFieldValue valueFromFile = new StringFieldValue(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(docMan, new GrowableByteBuffer(serializedFromFile)); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(docMan, new GrowableByteBuffer(serializedFromFile)); deserializer.read(null, valueFromFile); assertEquals(value, valueFromFile); } @@ -182,7 +182,7 @@ public class SerializeAnnotationsTestCase { assertEquals(serialized.limit(), serializedFromFile.limit()); StringFieldValue valueFromFile = new StringFieldValue(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(docMan, new GrowableByteBuffer(serializedFromFile)); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(docMan, new GrowableByteBuffer(serializedFromFile)); deserializer.read(null, valueFromFile); assertEquals(value, valueFromFile); } diff --git a/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java b/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java index 4f34431c729..5a8d458c74b 100644 --- a/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java +++ b/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java @@ -53,7 +53,7 @@ public class VespaDocumentSerializerTestCase { PredicateFieldValue predicate = Mockito.mock(PredicateFieldValue.class); doc.setFieldValue("my_predicate", predicate); - DocumentSerializerFactory.create42(new GrowableByteBuffer()).write(doc); + DocumentSerializerFactory.create6(new GrowableByteBuffer()).write(doc); Mockito.verify(predicate, Mockito.times(1)).serialize(Mockito.same(field), Mockito.any(FieldWriter.class)); } diff --git a/document/src/test/java/com/yahoo/document/update/FieldUpdateTestCase.java b/document/src/test/java/com/yahoo/document/update/FieldUpdateTestCase.java index 3cf0fab80ae..11e0cc783af 100644 --- a/document/src/test/java/com/yahoo/document/update/FieldUpdateTestCase.java +++ b/document/src/test/java/com/yahoo/document/update/FieldUpdateTestCase.java @@ -186,10 +186,10 @@ public class FieldUpdateTestCase { // Copy all field updates using serialization to verify that it is supported private FieldUpdate serializedCopy(FieldUpdate source, DocumentType docType) { - DocumentSerializer buffer = DocumentSerializerFactory.create42(); + DocumentSerializer buffer = DocumentSerializerFactory.create6(); source.serialize(buffer); buffer.getBuf().flip(); - FieldUpdate copy = new FieldUpdate(DocumentDeserializerFactory.create42(docman, buffer.getBuf()), docType, Document.SERIALIZED_VERSION); + FieldUpdate copy = new FieldUpdate(DocumentDeserializerFactory.create6(docman, buffer.getBuf()), docType, Document.SERIALIZED_VERSION); assertEquals(source, copy); return copy; } diff --git a/document/src/test/java/com/yahoo/document/update/SerializationTestCase.java b/document/src/test/java/com/yahoo/document/update/SerializationTestCase.java index f1cce1e4c4a..05492d41b09 100644 --- a/document/src/test/java/com/yahoo/document/update/SerializationTestCase.java +++ b/document/src/test/java/com/yahoo/document/update/SerializationTestCase.java @@ -30,7 +30,7 @@ public class SerializationTestCase { @Test public void testAddSerialization() { FieldUpdate update = FieldUpdate.createAdd(field, new StringFieldValue("value1")); - DocumentSerializer buffer = DocumentSerializerFactory.create42(); + DocumentSerializer buffer = DocumentSerializerFactory.create6(); update.serialize(buffer); buffer.getBuf().rewind(); @@ -41,18 +41,18 @@ public class SerializationTestCase { fos.close(); } catch (Exception e) {} - FieldUpdate deserializedUpdate = new FieldUpdate(DocumentDeserializerFactory.create42(new DocumentTypeManager(), buffer.getBuf()), documentType, Document.SERIALIZED_VERSION); + FieldUpdate deserializedUpdate = new FieldUpdate(DocumentDeserializerFactory.create6(new DocumentTypeManager(), buffer.getBuf()), documentType, Document.SERIALIZED_VERSION); assertEquals("'field1' [add value1 1]", deserializedUpdate.toString()); } @Test public void testClearSerialization() { FieldUpdate update = FieldUpdate.createClear(field); - DocumentSerializer buffer = DocumentSerializerFactory.create42(); + DocumentSerializer buffer = DocumentSerializerFactory.create6(); update.serialize(buffer); buffer.getBuf().rewind(); - FieldUpdate deserializedUpdate = new FieldUpdate(DocumentDeserializerFactory.create42(new DocumentTypeManager(), buffer.getBuf()), documentType, Document.SERIALIZED_VERSION); + FieldUpdate deserializedUpdate = new FieldUpdate(DocumentDeserializerFactory.create6(new DocumentTypeManager(), buffer.getBuf()), documentType, Document.SERIALIZED_VERSION); assertEquals("'field1' [clear]", deserializedUpdate.toString()); } diff --git a/document/src/tests/data/serializeupdatejava.dat b/document/src/tests/data/serializeupdatejava.dat Binary files differindex e2a98d42fb1..20c56228bde 100644 --- a/document/src/tests/data/serializeupdatejava.dat +++ b/document/src/tests/data/serializeupdatejava.dat diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index b4d98f0dd21..4962b708ef5 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -553,7 +553,7 @@ void DocumentUpdateTest::testReadSerializedFile() CPPUNIT_ASSERT_EQUAL(*type, upd.getType()); // Verify assign value update. - FieldUpdate serField = upd.getUpdates()[0]; + FieldUpdate serField = upd.getUpdates()[1]; CPPUNIT_ASSERT_EQUAL(serField.getField().getId(), type->getField("intfield").getId()); const ValueUpdate* serValue = &serField[0]; @@ -563,7 +563,7 @@ void DocumentUpdateTest::testReadSerializedFile() CPPUNIT_ASSERT_EQUAL(IntFieldValue(4), static_cast<const IntFieldValue&>(assign->getValue())); // Verify clear field update. - serField = upd.getUpdates()[1]; + serField = upd.getUpdates()[2]; CPPUNIT_ASSERT_EQUAL(serField.getField().getId(), type->getField("floatfield").getId()); serValue = &serField[0]; @@ -571,7 +571,7 @@ void DocumentUpdateTest::testReadSerializedFile() CPPUNIT_ASSERT(serValue->inherits(ClearValueUpdate::classId)); // Verify add value update. - serField = upd.getUpdates()[2]; + serField = upd.getUpdates()[0]; CPPUNIT_ASSERT_EQUAL(serField.getField().getId(), type->getField("arrayoffloatfield").getId()); serValue = &serField[0]; diff --git a/documentapi/abi-spec.json b/documentapi/abi-spec.json index 9f1384570d2..e1cdc6747e3 100644 --- a/documentapi/abi-spec.json +++ b/documentapi/abi-spec.json @@ -37,13 +37,14 @@ ], "methods": [ "public abstract com.yahoo.documentapi.Result put(com.yahoo.document.Document)", - "public abstract com.yahoo.documentapi.Result put(com.yahoo.document.Document, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result put(com.yahoo.document.Document, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public abstract com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId)", - "public abstract com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, boolean, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, boolean, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public abstract com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId)", - "public abstract com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public abstract com.yahoo.documentapi.Result update(com.yahoo.document.DocumentUpdate)", - "public abstract com.yahoo.documentapi.Result update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result update(com.yahoo.document.DocumentUpdate, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public abstract double getCurrentWindowSize()" ], "fields": [] @@ -132,15 +133,15 @@ ], "fields": [] }, - "com.yahoo.documentapi.DocumentListVisitorResponse": { + "com.yahoo.documentapi.DocumentOpVisitorResponse": { "superClass": "com.yahoo.documentapi.VisitorResponse", "interfaces": [], "attributes": [ "public" ], "methods": [ - "public void <init>(com.yahoo.vdslib.DocumentList, com.yahoo.documentapi.AckToken)", - "public com.yahoo.vdslib.DocumentList getDocumentList()" + "public void <init>(com.yahoo.document.DocumentOperation, com.yahoo.documentapi.AckToken)", + "public com.yahoo.document.DocumentOperation getDocumentOperation()" ], "fields": [] }, @@ -415,7 +416,6 @@ "public boolean isSuccess()", "public java.lang.Error getError()", "public long getRequestId()", - "public com.yahoo.documentapi.Result$ResultType getType()", "public com.yahoo.documentapi.Result$ResultType type()" ], "fields": [] @@ -496,7 +496,6 @@ "public" ], "methods": [ - "public void <init>()", "public java.util.Optional defaultTimeout()" ], "fields": [] @@ -513,11 +512,11 @@ ], "methods": [ "public abstract void put(com.yahoo.document.DocumentPut)", - "public abstract void put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", - "public abstract com.yahoo.document.Document get(com.yahoo.document.DocumentId)", - "public abstract com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", - "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.time.Duration)", - "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority, java.time.Duration)", + "public void put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.document.Document get(com.yahoo.document.DocumentId)", + "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public abstract com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.time.Duration)", + "public abstract com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority, java.time.Duration)", "public abstract boolean remove(com.yahoo.document.DocumentRemove)", "public abstract boolean remove(com.yahoo.document.DocumentRemove, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public abstract boolean update(com.yahoo.document.DocumentUpdate)", @@ -672,7 +671,6 @@ "public void <init>()", "public void reset()", "public void onMessage(com.yahoo.messagebus.Message, com.yahoo.documentapi.AckToken)", - "public void onDocuments(com.yahoo.vdslib.DocumentList, com.yahoo.documentapi.AckToken)", "public com.yahoo.documentapi.VisitorResponse getNext()", "public com.yahoo.documentapi.VisitorResponse getNext(int)" ], @@ -931,6 +929,7 @@ "public com.yahoo.documentapi.Result put(com.yahoo.document.Document, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId)", "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, boolean, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId)", "public com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public com.yahoo.documentapi.Result update(com.yahoo.document.DocumentUpdate)", @@ -970,8 +969,8 @@ "public void <init>(com.yahoo.documentapi.local.LocalDocumentAccess)", "public void put(com.yahoo.document.DocumentPut)", "public void put(com.yahoo.document.DocumentPut, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", - "public com.yahoo.document.Document get(com.yahoo.document.DocumentId)", - "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.time.Duration)", + "public com.yahoo.document.Document get(com.yahoo.document.DocumentId, java.lang.String, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority, java.time.Duration)", "public boolean remove(com.yahoo.document.DocumentRemove)", "public boolean remove(com.yahoo.document.DocumentRemove, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public boolean update(com.yahoo.document.DocumentUpdate)", @@ -996,6 +995,7 @@ "public com.yahoo.documentapi.Result put(com.yahoo.document.Document, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId)", "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, boolean, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", + "public com.yahoo.documentapi.Result get(com.yahoo.document.DocumentId, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId)", "public com.yahoo.documentapi.Result remove(com.yahoo.document.DocumentId, com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority)", "public com.yahoo.documentapi.Result update(com.yahoo.document.DocumentUpdate)", @@ -2405,8 +2405,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$CreateVisitorMessageFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentMessageFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$CreateVisitorMessageFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", "interfaces": [], "attributes": [ "public" @@ -2420,8 +2420,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$CreateVisitorReplyFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentReplyFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$CreateVisitorReplyFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", "interfaces": [], "attributes": [ "public" @@ -2433,8 +2433,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DestroyVisitorMessageFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentMessageFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DestroyVisitorMessageFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", "interfaces": [], "attributes": [ "public" @@ -2446,8 +2446,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DestroyVisitorReplyFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentReplyFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DestroyVisitorReplyFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", "interfaces": [], "attributes": [ "public" @@ -2459,8 +2459,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentIgnoredReplyFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentReplyFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentIgnoredReplyFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", "interfaces": [], "attributes": [ "public" @@ -2472,8 +2472,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentListMessageFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentMessageFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentListMessageFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", "interfaces": [], "attributes": [ "public" @@ -2485,8 +2485,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentListReplyFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentReplyFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentListReplyFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", "interfaces": [], "attributes": [ "public" @@ -2498,7 +2498,7 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentMessageFactory": { + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory": { "superClass": "com.yahoo.documentapi.messagebus.protocol.AbstractRoutableFactory", "interfaces": [], "attributes": [ @@ -2514,7 +2514,7 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentReplyFactory": { + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory": { "superClass": "com.yahoo.documentapi.messagebus.protocol.AbstractRoutableFactory", "interfaces": [], "attributes": [ @@ -2530,8 +2530,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentSummaryMessageFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentMessageFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentSummaryMessageFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", "interfaces": [], "attributes": [ "public" @@ -2543,8 +2543,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentSummaryReplyFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentReplyFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentSummaryReplyFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", "interfaces": [], "attributes": [ "public" @@ -2556,8 +2556,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$EmptyBucketsMessageFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentMessageFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$EmptyBucketsMessageFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", "interfaces": [], "attributes": [ "public" @@ -2569,8 +2569,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$EmptyBucketsReplyFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentReplyFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$EmptyBucketsReplyFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", "interfaces": [], "attributes": [ "public" @@ -2582,8 +2582,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$GetBucketListMessageFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentMessageFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$GetBucketListMessageFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", "interfaces": [], "attributes": [ "public" @@ -2597,8 +2597,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$GetBucketListReplyFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentReplyFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$GetBucketListReplyFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", "interfaces": [], "attributes": [ "public" @@ -2610,8 +2610,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$GetBucketStateMessageFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentMessageFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$GetBucketStateMessageFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", "interfaces": [], "attributes": [ "public" @@ -2623,8 +2623,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$GetBucketStateReplyFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentReplyFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$GetBucketStateReplyFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", "interfaces": [], "attributes": [ "public" @@ -2636,8 +2636,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$GetDocumentMessageFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentMessageFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$GetDocumentMessageFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", "interfaces": [], "attributes": [ "public" @@ -2649,8 +2649,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$GetDocumentReplyFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentReplyFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$GetDocumentReplyFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", "interfaces": [], "attributes": [ "public" @@ -2662,8 +2662,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$MapVisitorMessageFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentMessageFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$MapVisitorMessageFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", "interfaces": [], "attributes": [ "public" @@ -2675,8 +2675,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$MapVisitorReplyFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentReplyFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$MapVisitorReplyFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", "interfaces": [], "attributes": [ "public" @@ -2688,8 +2688,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$PutDocumentMessageFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentMessageFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$PutDocumentMessageFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", "interfaces": [], "attributes": [ "public" @@ -2702,8 +2702,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$PutDocumentReplyFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentReplyFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$PutDocumentReplyFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", "interfaces": [], "attributes": [ "public" @@ -2715,8 +2715,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$QueryResultMessageFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentMessageFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$QueryResultMessageFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", "interfaces": [], "attributes": [ "public" @@ -2728,8 +2728,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$QueryResultReplyFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentReplyFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$QueryResultReplyFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", "interfaces": [], "attributes": [ "public" @@ -2741,8 +2741,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$RemoveDocumentMessageFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentMessageFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$RemoveDocumentMessageFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", "interfaces": [], "attributes": [ "public" @@ -2755,8 +2755,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$RemoveDocumentReplyFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentReplyFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$RemoveDocumentReplyFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", "interfaces": [], "attributes": [ "public" @@ -2768,8 +2768,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$RemoveLocationMessageFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentMessageFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$RemoveLocationMessageFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", "interfaces": [], "attributes": [ "public" @@ -2781,8 +2781,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$RemoveLocationReplyFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentReplyFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$RemoveLocationReplyFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", "interfaces": [], "attributes": [ "public" @@ -2794,8 +2794,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$SearchResultMessageFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentMessageFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$SearchResultMessageFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", "interfaces": [], "attributes": [ "public" @@ -2807,8 +2807,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$SearchResultReplyFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentReplyFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$SearchResultReplyFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", "interfaces": [], "attributes": [ "public" @@ -2820,8 +2820,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$StatBucketMessageFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentMessageFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$StatBucketMessageFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", "interfaces": [], "attributes": [ "public" @@ -2835,8 +2835,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$StatBucketReplyFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentReplyFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$StatBucketReplyFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", "interfaces": [], "attributes": [ "public" @@ -2848,8 +2848,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$UpdateDocumentMessageFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentMessageFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$UpdateDocumentMessageFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", "interfaces": [], "attributes": [ "public" @@ -2862,8 +2862,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$UpdateDocumentReplyFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentReplyFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$UpdateDocumentReplyFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", "interfaces": [], "attributes": [ "public" @@ -2875,8 +2875,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$VisitorInfoMessageFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentMessageFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$VisitorInfoMessageFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentMessageFactory", "interfaces": [], "attributes": [ "public" @@ -2888,8 +2888,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$VisitorInfoReplyFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentReplyFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$VisitorInfoReplyFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", "interfaces": [], "attributes": [ "public" @@ -2901,8 +2901,8 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$WrongDistributionReplyFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$DocumentReplyFactory", + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$WrongDistributionReplyFactory": { + "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$DocumentReplyFactory", "interfaces": [], "attributes": [ "public" @@ -2914,7 +2914,7 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52": { + "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60": { "superClass": "java.lang.Object", "interfaces": [], "attributes": [ @@ -2926,56 +2926,6 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$CreateVisitorMessageFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$CreateVisitorMessageFactory", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "protected java.lang.String decodeBucketSpace(com.yahoo.vespa.objects.Deserializer)", - "protected boolean encodeBucketSpace(java.lang.String, com.yahoo.document.serialization.DocumentSerializer)" - ], - "fields": [] - }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$GetBucketListMessageFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$GetBucketListMessageFactory", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "protected java.lang.String decodeBucketSpace(com.yahoo.vespa.objects.Deserializer)", - "protected boolean encodeBucketSpace(java.lang.String, com.yahoo.document.serialization.DocumentSerializer)" - ], - "fields": [] - }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60$StatBucketMessageFactory": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52$StatBucketMessageFactory", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "protected java.lang.String decodeBucketSpace(com.yahoo.vespa.objects.Deserializer)", - "protected boolean encodeBucketSpace(java.lang.String, com.yahoo.document.serialization.DocumentSerializer)" - ], - "fields": [] - }, - "com.yahoo.documentapi.messagebus.protocol.RoutableFactories60": { - "superClass": "com.yahoo.documentapi.messagebus.protocol.RoutableFactories52", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()" - ], - "fields": [] - }, "com.yahoo.documentapi.messagebus.protocol.RoutableFactory": { "superClass": "java.lang.Object", "interfaces": [], diff --git a/documentapi/pom.xml b/documentapi/pom.xml index 87985cab37b..46f7755f654 100644 --- a/documentapi/pom.xml +++ b/documentapi/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>documentapi</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>com.google.guava</groupId> diff --git a/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java index 2e78a023910..c9db5223283 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/AsyncSession.java @@ -43,7 +43,9 @@ public interface AsyncSession extends Session { * @param document the Document to put * @return the synchronous result of this operation */ - Result put(Document document, DocumentProtocol.Priority priority); + default Result put(Document document, DocumentProtocol.Priority priority) { + return put(document); + } /** * <p>Gets a document. This method returns immediately.</p> @@ -71,8 +73,30 @@ public interface AsyncSession extends Session { * @param priority The priority with which to perform this operation. * @return the synchronous result of this operation * @throws UnsupportedOperationException if this access implementation does not support retrieving + * @deprecated the 'headersonly' flag has no effect */ - Result get(DocumentId id, boolean headersOnly, DocumentProtocol.Priority priority); + @Deprecated // TODO: Remove on Vespa 8 + default Result get(DocumentId id, boolean headersOnly, DocumentProtocol.Priority priority) { + return get(id); + } + + /** + * <p>Gets a document. This method returns immediately.</p> + * + * <p>If this result is a success, this + * call will cause one or more {@link DocumentResponse} objects to appear within the timeout time of this session. + * The response returned later will contain the requested document if it is a success. + * If it was not a success, this method has no further effects.</p> + * + * @param id the id of the document to get + * @param priority The priority with which to perform this operation. + * @return the synchronous result of this operation + * @throws UnsupportedOperationException if this access implementation does not support retrieving + */ + default Result get(DocumentId id, DocumentProtocol.Priority priority) { + return get(id); + } + /** * <p>Removes a document if it is present. This method returns immediately.</p> @@ -101,7 +125,9 @@ public interface AsyncSession extends Session { * @return the synchronous result of this operation * @throws UnsupportedOperationException if this access implementation does not support removal */ - Result remove(DocumentId id, DocumentProtocol.Priority priority); + default Result remove(DocumentId id, DocumentProtocol.Priority priority) { + return remove(id); + } /** * <p>Updates a document. This method returns immediately.</p> @@ -130,7 +156,9 @@ public interface AsyncSession extends Session { * @return the synchronous result of this operation * @throws UnsupportedOperationException if this access implementation does not support update */ - Result update(DocumentUpdate update, DocumentProtocol.Priority priority); + default Result update(DocumentUpdate update, DocumentProtocol.Priority priority) { + return update(update); + } /** * Returns the current send window size of the session. diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentListVisitorResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentListVisitorResponse.java deleted file mode 100644 index 3acbf70a21e..00000000000 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentListVisitorResponse.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi; - -import com.yahoo.vdslib.DocumentList; - -/** - * Visitor response containing a document list. All visitor responses have ack - * tokens that must be acked. - * - * @author <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> - */ -public class DocumentListVisitorResponse extends VisitorResponse { - private DocumentList documents; - - /** - * Creates visitor response containing a document list and an ack token. - * - * @param docs the document list - * @param ack the ack token - */ - public DocumentListVisitorResponse(DocumentList docs, AckToken ack) { - super(ack); - documents = docs; - } - - /** @return the document list */ - public DocumentList getDocumentList() { return documents; } -} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentOpVisitorResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentOpVisitorResponse.java new file mode 100644 index 00000000000..f5641b915f4 --- /dev/null +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentOpVisitorResponse.java @@ -0,0 +1,28 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentapi; + +import com.yahoo.document.DocumentOperation; + +/** + * Visitor response containing a document operation. All visitor responses have ack + * tokens that must be acked. + * + * @author Arne H Juul + */ +public class DocumentOpVisitorResponse extends VisitorResponse { + private DocumentOperation op; + + /** + * Creates visitor response containing a document operation and an ack token. + * + * @param op the document operation + * @param ack the ack token + */ + public DocumentOpVisitorResponse(DocumentOperation op, AckToken ack) { + super(ack); + this.op = op; + } + + /** @return the document operation */ + public DocumentOperation getDocumentOperation() { return op; } +} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java b/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java index f5701802a84..9a6c2ea3bd3 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java @@ -7,11 +7,10 @@ import java.util.Map; import java.util.TreeMap; import java.util.logging.Logger; -import com.yahoo.document.*; -import com.yahoo.document.serialization.*; +import com.yahoo.document.BucketId; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.log.LogLevel; -import com.yahoo.vespa.objects.Serializer; +import com.yahoo.vespa.objects.BufferSerializer; /** * Token to use to keep track of progress for visiting. Can be used to resume @@ -213,7 +212,7 @@ public class ProgressToken { } public ProgressToken(byte[] serialized) { - DocumentDeserializer in = DocumentDeserializerFactory.create42(null, GrowableByteBuffer.wrap(serialized)); + BufferSerializer in = new BufferSerializer(GrowableByteBuffer.wrap(serialized)); distributionBits = in.getInt(null); bucketCursor = in.getLong(null); finishedBucketCount = in.getLong(null); @@ -228,7 +227,7 @@ public class ProgressToken { } public byte[] serialize() { - DocumentSerializer out = DocumentSerializerFactory.create42(new GrowableByteBuffer()); + BufferSerializer out = new BufferSerializer(new GrowableByteBuffer()); out.putInt(null, distributionBits); out.putLong(null, bucketCursor); out.putLong(null, finishedBucketCount); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/Result.java b/documentapi/src/main/java/com/yahoo/documentapi/Result.java index c905fa24cb1..1242ccfe472 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/Result.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/Result.java @@ -66,20 +66,6 @@ public class Result { public long getRequestId() { return requestId; } /** - * @deprecated use type() instead. - * Returns the type of result. - * - * @return the type of result, typically if this is an error or a success, and what kind of error. - * Does not return CONDITION_NOT_MET_ERROR for backward compatibility. - * @see com.yahoo.documentapi.Result.ResultType - */ - @Deprecated - public ResultType getType() { return - type == ResultType.CONDITION_NOT_MET_ERROR - ? ResultType.FATAL_ERROR - : type;} - - /** * Returns the type of result. * * @return the type of result, typically if this is an error or a success, and what kind of error. diff --git a/documentapi/src/main/java/com/yahoo/documentapi/SimpleVisitorDocumentQueue.java b/documentapi/src/main/java/com/yahoo/documentapi/SimpleVisitorDocumentQueue.java index 102f17a64b1..e1dbe615f7d 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/SimpleVisitorDocumentQueue.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/SimpleVisitorDocumentQueue.java @@ -3,7 +3,6 @@ package com.yahoo.documentapi; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; -import com.yahoo.vdslib.DocumentList; import java.util.LinkedList; import java.util.List; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/SyncParameters.java b/documentapi/src/main/java/com/yahoo/documentapi/SyncParameters.java index b1986680532..acf027dd0aa 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/SyncParameters.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/SyncParameters.java @@ -13,12 +13,7 @@ import java.util.Optional; public class SyncParameters extends Parameters { private final Duration defaultTimeout; - /** - * @deprecated Use {@link Builder} instead. - */ - @Deprecated - // TODO Vespa 7: Make private - public SyncParameters() { + private SyncParameters() { this(null); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java index 0ce0ccc2377..c254df3ba02 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java @@ -34,7 +34,9 @@ public interface SyncSession extends Session { * @param documentPut The DocumentPut operation * @param priority The priority with which to perform this operation. */ - void put(DocumentPut documentPut, DocumentProtocol.Priority priority); + default void put(DocumentPut documentPut, DocumentProtocol.Priority priority) { + put(documentPut); + } /** * <p>Gets a document.</p> @@ -45,10 +47,10 @@ public interface SyncSession extends Session { * @throws UnsupportedOperationException Thrown if this access does not * support retrieving. */ - Document get(DocumentId id); + default Document get(DocumentId id) { return get(id, null); } /** - * <p>Gets a document.</p> + * Gets a document with an unspecified timeout * * @param id The id of the document to get. * @param fieldSet A comma-separated list of fields to retrieve @@ -58,7 +60,9 @@ public interface SyncSession extends Session { * @throws UnsupportedOperationException Thrown if this access does not * support retrieving. */ - Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority priority); + default Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority priority) { + return get(id, fieldSet, priority, null); + } /** * <p>Gets a document with timeout.</p> @@ -70,10 +74,7 @@ public interface SyncSession extends Session { * @throws UnsupportedOperationException Thrown if this access does not support retrieving. * @throws DocumentAccessException on any messagebus error, including timeout ({@link com.yahoo.messagebus.ErrorCode#TIMEOUT}). */ - // TODO Vespa 7: Remove default implementation. Consider removing get() overloads without timeout. - default Document get(DocumentId id, Duration timeout) { - return get(id); - } + Document get(DocumentId id, Duration timeout); /** * <p>Gets a document with timeout. </p> @@ -87,10 +88,7 @@ public interface SyncSession extends Session { * @throws UnsupportedOperationException Thrown if this access does not support retrieving. * @throws DocumentAccessException on any messagebus error, including timeout ({@link com.yahoo.messagebus.ErrorCode#TIMEOUT}). */ - // TODO Vespa 7: Remove default implementation. Consider removing get() overloads without timeout. - default Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority priority, Duration timeout) { - return get(id, fieldSet, priority); - } + Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority priority, Duration timeout); /** * <p>Removes a document if it is present and condition is fulfilled.</p> diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorDataHandler.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorDataHandler.java index fc27af2c306..57d4c78d9be 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/VisitorDataHandler.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorDataHandler.java @@ -5,8 +5,6 @@ import com.yahoo.document.Document; import com.yahoo.document.DocumentId; import com.yahoo.documentapi.messagebus.protocol.*; import com.yahoo.messagebus.Message; -import com.yahoo.vdslib.DocumentList; -import com.yahoo.vdslib.Entry; import com.yahoo.vdslib.SearchResult; import com.yahoo.vdslib.DocumentSummary; import com.yahoo.document.BucketId; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorDataQueue.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorDataQueue.java index a6648a6c89b..5c4e6c625a9 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/VisitorDataQueue.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorDataQueue.java @@ -4,9 +4,8 @@ package com.yahoo.documentapi; import com.yahoo.document.DocumentOperation; import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; import com.yahoo.messagebus.Message; -import com.yahoo.vdslib.DocumentList; -import com.yahoo.vdslib.Entry; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -18,7 +17,7 @@ import java.util.concurrent.TimeUnit; * implements the <code>getNext</code> methods, thus implementing the polling * API defined in VisitorDataHandler. * <p> - * Visitor responses containing document lists should be polled for with the + * Visitor responses should be polled for with the * <code>getNext</code> methods and need to be acked when processed for * visiting not to halt. The class is thread safe. * @@ -41,8 +40,7 @@ public class VisitorDataQueue extends VisitorDataHandler { } private void appendSingleOpToPendingList(final DocumentOperation op, final AckToken token) { - final DocumentList docList = DocumentList.create(Entry.create(op)); - final DocumentListVisitorResponse response = new DocumentListVisitorResponse(docList, token); + final DocumentOpVisitorResponse response = new DocumentOpVisitorResponse(op, token); pendingResponses.add(response); } @@ -59,14 +57,6 @@ public class VisitorDataQueue extends VisitorDataHandler { } } - /** - * @deprecated This method is no longer called by the visitor subsystem. See onMessage instead. - */ - @Deprecated - public void onDocuments(DocumentList docs, AckToken token) { - pendingResponses.add(new DocumentListVisitorResponse(docs, token)); - } - // Inherit doc from VisitorDataHandler @Override public VisitorResponse getNext() { diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java index 0cf8e05c8ed..64854373e05 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java @@ -140,7 +140,7 @@ public class VisitorParameters extends Parameters { public boolean getVisitRemoves() { return visitRemoves; } - public boolean getVisitHeadersOnly() { return "[header]".equals(fieldSet); } + public boolean getVisitHeadersOnly() { return false; } /** @return The field set to use. */ public String fieldSet() { return fieldSet; } @@ -231,7 +231,9 @@ public class VisitorParameters extends Parameters { public void setVisitRemoves(boolean visitRemoves) { this.visitRemoves = visitRemoves; } - public void setVisitHeadersOnly(boolean headersOnly) { this.fieldSet = headersOnly ? "[header]" : "[all]"; } + /** @deprecated this option is ignored */ + @Deprecated // TODO: Remove on Vespa 8 + public void setVisitHeadersOnly(boolean headersOnly) { } /** Set field set to use. */ public void fieldSet(String fieldSet) { this.fieldSet = fieldSet; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java index 4e331071be7..00d6bb5ae33 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalAsyncSession.java @@ -74,7 +74,13 @@ public class LocalAsyncSession implements AsyncSession { } @Override + @Deprecated // TODO: Remove on Vespa 8 public Result get(DocumentId id, boolean headersOnly, DocumentProtocol.Priority pri) { + return get(id, pri); + } + + @Override + public Result get(DocumentId id, DocumentProtocol.Priority pri) { long req = getNextRequestId(); try { addResponse(new DocumentResponse(req, syncSession.get(id))); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java index 45df2015da4..ea0ee2eaee1 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalSyncSession.java @@ -10,6 +10,8 @@ import com.yahoo.documentapi.Response; import com.yahoo.documentapi.SyncSession; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import java.time.Duration; + /** * @author bratseth */ @@ -36,15 +38,12 @@ public class LocalSyncSession implements SyncSession { } @Override - public Document get(DocumentId id) { + public Document get(DocumentId id, Duration timeout) { return access.documents.get(id); } @Override - public Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority pri) { - // FIXME: More than half the get() methods are deprecated, but they all - // call exactly the same method, including this one, throwing away most - // of the parameters + public Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority priority, Duration timeout) { return access.documents.get(id); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java index 8a6fa85c68b..67b9444e73f 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java @@ -86,8 +86,14 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { } @Override + @Deprecated // TODO: Remove on Vespa 8 public Result get(DocumentId id, boolean headersOnly, DocumentProtocol.Priority pri) { - GetDocumentMessage msg = new GetDocumentMessage(id, headersOnly ? "[header]" : "[all]"); + return get(id, pri); + } + + @Override + public Result get(DocumentId id, DocumentProtocol.Priority pri) { + GetDocumentMessage msg = new GetDocumentMessage(id, "[all]"); msg.setPriority(pri); return send(msg); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java index 10a316f7751..e627316de30 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java @@ -301,55 +301,48 @@ public class DocumentProtocol implements Protocol { putRoutingPolicyFactory("SubsetService", new RoutingPolicyFactories.SubsetServicePolicyFactory()); // Prepare version specifications to use when adding routable factories. - VersionSpecification version52 = new VersionSpecification(5, 115); VersionSpecification version6 = new VersionSpecification(6, 221); - List<VersionSpecification> from52 = Arrays.asList(version52, version6); - List<VersionSpecification> from6 = Collections.singletonList(version6); // TODO decide minor version... - - // 5.2 serialization (keep alphabetized please) - putRoutableFactory(MESSAGE_CREATEVISITOR, new RoutableFactories52.CreateVisitorMessageFactory(), from52); - putRoutableFactory(MESSAGE_DESTROYVISITOR, new RoutableFactories52.DestroyVisitorMessageFactory(), from52); - putRoutableFactory(MESSAGE_DOCUMENTLIST, new RoutableFactories52.DocumentListMessageFactory(), from52); - putRoutableFactory(MESSAGE_DOCUMENTSUMMARY, new RoutableFactories52.DocumentSummaryMessageFactory(), from52); - putRoutableFactory(MESSAGE_EMPTYBUCKETS, new RoutableFactories52.EmptyBucketsMessageFactory(), from52); - putRoutableFactory(MESSAGE_GETBUCKETLIST, new RoutableFactories52.GetBucketListMessageFactory(), from52); - putRoutableFactory(MESSAGE_GETBUCKETSTATE, new RoutableFactories52.GetBucketStateMessageFactory(), from52); - putRoutableFactory(MESSAGE_GETDOCUMENT, new RoutableFactories52.GetDocumentMessageFactory(), from52); - putRoutableFactory(MESSAGE_MAPVISITOR, new RoutableFactories52.MapVisitorMessageFactory(), from52); - putRoutableFactory(MESSAGE_PUTDOCUMENT, new RoutableFactories52.PutDocumentMessageFactory(), from52); - putRoutableFactory(MESSAGE_QUERYRESULT, new RoutableFactories52.QueryResultMessageFactory(), from52); - putRoutableFactory(MESSAGE_REMOVEDOCUMENT, new RoutableFactories52.RemoveDocumentMessageFactory(), from52); - putRoutableFactory(MESSAGE_REMOVELOCATION, new RoutableFactories52.RemoveLocationMessageFactory(), from52); - putRoutableFactory(MESSAGE_SEARCHRESULT, new RoutableFactories52.SearchResultMessageFactory(), from52); - putRoutableFactory(MESSAGE_STATBUCKET, new RoutableFactories52.StatBucketMessageFactory(), from52); - putRoutableFactory(MESSAGE_UPDATEDOCUMENT, new RoutableFactories52.UpdateDocumentMessageFactory(), from52); - putRoutableFactory(MESSAGE_VISITORINFO, new RoutableFactories52.VisitorInfoMessageFactory(), from52); - putRoutableFactory(REPLY_CREATEVISITOR, new RoutableFactories52.CreateVisitorReplyFactory(), from52); - putRoutableFactory(REPLY_DESTROYVISITOR, new RoutableFactories52.DestroyVisitorReplyFactory(), from52); - putRoutableFactory(REPLY_DOCUMENTIGNORED, new RoutableFactories52.DocumentIgnoredReplyFactory(), from52); - putRoutableFactory(REPLY_DOCUMENTLIST, new RoutableFactories52.DocumentListReplyFactory(), from52); - putRoutableFactory(REPLY_DOCUMENTSUMMARY, new RoutableFactories52.DocumentSummaryReplyFactory(), from52); - putRoutableFactory(REPLY_EMPTYBUCKETS, new RoutableFactories52.EmptyBucketsReplyFactory(), from52); - putRoutableFactory(REPLY_GETBUCKETLIST, new RoutableFactories52.GetBucketListReplyFactory(), from52); - putRoutableFactory(REPLY_GETBUCKETSTATE, new RoutableFactories52.GetBucketStateReplyFactory(), from52); - putRoutableFactory(REPLY_GETDOCUMENT, new RoutableFactories52.GetDocumentReplyFactory(), from52); - putRoutableFactory(REPLY_MAPVISITOR, new RoutableFactories52.MapVisitorReplyFactory(), from52); - putRoutableFactory(REPLY_PUTDOCUMENT, new RoutableFactories52.PutDocumentReplyFactory(), from52); - putRoutableFactory(REPLY_QUERYRESULT, new RoutableFactories52.QueryResultReplyFactory(), from52); - putRoutableFactory(REPLY_REMOVEDOCUMENT, new RoutableFactories52.RemoveDocumentReplyFactory(), from52); - putRoutableFactory(REPLY_REMOVELOCATION, new RoutableFactories52.RemoveLocationReplyFactory(), from52); - putRoutableFactory(REPLY_SEARCHRESULT, new RoutableFactories52.SearchResultReplyFactory(), from52); - putRoutableFactory(REPLY_STATBUCKET, new RoutableFactories52.StatBucketReplyFactory(), from52); - putRoutableFactory(REPLY_UPDATEDOCUMENT, new RoutableFactories52.UpdateDocumentReplyFactory(), from52); - putRoutableFactory(REPLY_UPDATEDOCUMENT, new RoutableFactories52.UpdateDocumentReplyFactory(), from52); - putRoutableFactory(REPLY_VISITORINFO, new RoutableFactories52.VisitorInfoReplyFactory(), from52); - putRoutableFactory(REPLY_WRONGDISTRIBUTION, new RoutableFactories52.WrongDistributionReplyFactory(), from52); - - // 6.x serialization + List<VersionSpecification> from6 = Collections.singletonList(version6); + + // 6.x serialization (keep alphabetized please) putRoutableFactory(MESSAGE_CREATEVISITOR, new RoutableFactories60.CreateVisitorMessageFactory(), from6); - putRoutableFactory(MESSAGE_STATBUCKET, new RoutableFactories60.StatBucketMessageFactory(), from6); + putRoutableFactory(MESSAGE_DESTROYVISITOR, new RoutableFactories60.DestroyVisitorMessageFactory(), from6); + putRoutableFactory(MESSAGE_DOCUMENTLIST, new RoutableFactories60.DocumentListMessageFactory(), from6); + putRoutableFactory(MESSAGE_DOCUMENTSUMMARY, new RoutableFactories60.DocumentSummaryMessageFactory(), from6); + putRoutableFactory(MESSAGE_EMPTYBUCKETS, new RoutableFactories60.EmptyBucketsMessageFactory(), from6); putRoutableFactory(MESSAGE_GETBUCKETLIST, new RoutableFactories60.GetBucketListMessageFactory(), from6); + putRoutableFactory(MESSAGE_GETBUCKETSTATE, new RoutableFactories60.GetBucketStateMessageFactory(), from6); + putRoutableFactory(MESSAGE_GETDOCUMENT, new RoutableFactories60.GetDocumentMessageFactory(), from6); + putRoutableFactory(MESSAGE_MAPVISITOR, new RoutableFactories60.MapVisitorMessageFactory(), from6); + putRoutableFactory(MESSAGE_PUTDOCUMENT, new RoutableFactories60.PutDocumentMessageFactory(), from6); + putRoutableFactory(MESSAGE_QUERYRESULT, new RoutableFactories60.QueryResultMessageFactory(), from6); + putRoutableFactory(MESSAGE_REMOVEDOCUMENT, new RoutableFactories60.RemoveDocumentMessageFactory(), from6); + putRoutableFactory(MESSAGE_REMOVELOCATION, new RoutableFactories60.RemoveLocationMessageFactory(), from6); + putRoutableFactory(MESSAGE_SEARCHRESULT, new RoutableFactories60.SearchResultMessageFactory(), from6); + putRoutableFactory(MESSAGE_STATBUCKET, new RoutableFactories60.StatBucketMessageFactory(), from6); + putRoutableFactory(MESSAGE_UPDATEDOCUMENT, new RoutableFactories60.UpdateDocumentMessageFactory(), from6); + putRoutableFactory(MESSAGE_VISITORINFO, new RoutableFactories60.VisitorInfoMessageFactory(), from6); + putRoutableFactory(REPLY_CREATEVISITOR, new RoutableFactories60.CreateVisitorReplyFactory(), from6); + putRoutableFactory(REPLY_DESTROYVISITOR, new RoutableFactories60.DestroyVisitorReplyFactory(), from6); + putRoutableFactory(REPLY_DOCUMENTIGNORED, new RoutableFactories60.DocumentIgnoredReplyFactory(), from6); + putRoutableFactory(REPLY_DOCUMENTLIST, new RoutableFactories60.DocumentListReplyFactory(), from6); + putRoutableFactory(REPLY_DOCUMENTSUMMARY, new RoutableFactories60.DocumentSummaryReplyFactory(), from6); + putRoutableFactory(REPLY_EMPTYBUCKETS, new RoutableFactories60.EmptyBucketsReplyFactory(), from6); + putRoutableFactory(REPLY_GETBUCKETLIST, new RoutableFactories60.GetBucketListReplyFactory(), from6); + putRoutableFactory(REPLY_GETBUCKETSTATE, new RoutableFactories60.GetBucketStateReplyFactory(), from6); + putRoutableFactory(REPLY_GETDOCUMENT, new RoutableFactories60.GetDocumentReplyFactory(), from6); + putRoutableFactory(REPLY_MAPVISITOR, new RoutableFactories60.MapVisitorReplyFactory(), from6); + putRoutableFactory(REPLY_PUTDOCUMENT, new RoutableFactories60.PutDocumentReplyFactory(), from6); + putRoutableFactory(REPLY_QUERYRESULT, new RoutableFactories60.QueryResultReplyFactory(), from6); + putRoutableFactory(REPLY_REMOVEDOCUMENT, new RoutableFactories60.RemoveDocumentReplyFactory(), from6); + putRoutableFactory(REPLY_REMOVELOCATION, new RoutableFactories60.RemoveLocationReplyFactory(), from6); + putRoutableFactory(REPLY_SEARCHRESULT, new RoutableFactories60.SearchResultReplyFactory(), from6); + putRoutableFactory(REPLY_STATBUCKET, new RoutableFactories60.StatBucketReplyFactory(), from6); + putRoutableFactory(REPLY_UPDATEDOCUMENT, new RoutableFactories60.UpdateDocumentReplyFactory(), from6); + putRoutableFactory(REPLY_UPDATEDOCUMENT, new RoutableFactories60.UpdateDocumentReplyFactory(), from6); + putRoutableFactory(REPLY_VISITORINFO, new RoutableFactories60.VisitorInfoReplyFactory(), from6); + putRoutableFactory(REPLY_WRONGDISTRIBUTION, new RoutableFactories60.WrongDistributionReplyFactory(), from6); } /** diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories52.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories52.java deleted file mode 100644 index 89a0aa22bb3..00000000000 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories52.java +++ /dev/null @@ -1,953 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol; - -import com.yahoo.document.BucketId; -import com.yahoo.document.Document; -import com.yahoo.document.DocumentId; -import com.yahoo.document.DocumentPut; -import com.yahoo.document.DocumentUpdate; -import com.yahoo.document.FixedBucketSpaces; -import com.yahoo.document.TestAndSetCondition; -import com.yahoo.document.serialization.DocumentDeserializer; -import com.yahoo.document.serialization.DocumentSerializer; -import com.yahoo.document.serialization.DocumentSerializerFactory; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; -import com.yahoo.log.LogLevel; -import com.yahoo.messagebus.Routable; -import com.yahoo.vdslib.DocumentList; -import com.yahoo.vdslib.DocumentSummary; -import com.yahoo.vdslib.SearchResult; -import com.yahoo.vdslib.VisitorStatistics; -import com.yahoo.vespa.objects.Deserializer; -import com.yahoo.vespa.objects.Serializer; - -import java.util.Map; -import java.util.logging.Logger; - -import static com.yahoo.documentapi.messagebus.protocol.AbstractRoutableFactory.decodeString; -import static com.yahoo.documentapi.messagebus.protocol.AbstractRoutableFactory.encodeString; - - -/** - * @author Vegard Sjonfjell - * This class encapsulates all the {@link RoutableFactory} classes needed to implement serialization for the document - * protocol. When adding new factories to this class, please KEEP THE THEM ORDERED alphabetically like they are now. - */ -public abstract class RoutableFactories52 { - - /** - * Implements the shared factory logic required for {@link DocumentMessage} objects, and it offers a more convenient - * interface for implementing {@link RoutableFactory}. - * - * @author Simon Thoresen Hult - */ - public static abstract class DocumentMessageFactory extends AbstractRoutableFactory { - - /** - * This method encodes the given message using the given serializer. You are guaranteed to only receive messages - * of the type that this factory was registered for. - * <p> - * This method is NOT exception safe. Return false to - * signal failure. - * - * @param msg The message to encode. - * @param serializer The serializer to use for encoding. - * @return True if the message was encoded. - */ - protected abstract boolean doEncode(DocumentMessage msg, DocumentSerializer serializer); - - /** - * This method decodes a message from the given deserializer. You are guaranteed to only receive byte buffers - * generated by a previous call to {@link #doEncode(DocumentMessage, DocumentSerializer)}. - * <p> - * This method is NOT exception safe. Return null to signal failure. - * - * @param deserializer The deserializer to use for decoding. - * @return The decoded message. - */ - protected abstract DocumentMessage doDecode(DocumentDeserializer deserializer); - - public boolean encode(Routable obj, DocumentSerializer out) { - if (!(obj instanceof DocumentMessage)) { - throw new AssertionError( - "Document message factory (" + getClass().getName() + ") registered for incompatible " + - "routable type " + obj.getType() + "(" + obj.getClass().getName() + ")."); - } - DocumentMessage msg = (DocumentMessage)obj; - out.putByte(null, (byte)(msg.getPriority().getValue())); - out.putInt(null, msg.getLoadType().getId()); - return doEncode(msg, out); - } - - public Routable decode(DocumentDeserializer in, LoadTypeSet loadTypes) { - byte pri = in.getByte(null); - int loadType = in.getInt(null); - DocumentMessage msg = doDecode(in); - if (msg != null) { - msg.setPriority(DocumentProtocol.getPriority(pri)); - msg.setLoadType(loadTypes.getIdMap().get(loadType)); - } - return msg; - } - } - - /** - * Implements the shared factory logic required for {@link DocumentReply} objects, and it offers a more convenient - * interface for implementing {@link RoutableFactory}. - * - * @author Simon Thoresen Hult - */ - public static abstract class DocumentReplyFactory extends AbstractRoutableFactory { - - /** - * This method encodes the given reply into the given byte buffer. You are guaranteed to only receive replies of - * the type that this factory was registered for. - * <p> - * This method is NOT exception safe. Return false to signal - * failure. - * - * @param reply The reply to encode. - * @param buf The byte buffer to write to. - * @return True if the message was encoded. - */ - protected abstract boolean doEncode(DocumentReply reply, DocumentSerializer buf); - - /** - * This method decodes a reply from the given byte buffer. You are guaranteed to only receive byte buffers - * generated by a previous call to {@link #doEncode(DocumentReply, com.yahoo.document.serialization.DocumentSerializer)}. - * - * <p> - * This method is NOT exception safe. Return null to signal failure. - * - * @param buf The byte buffer to read from. - * @return The decoded reply. - */ - protected abstract DocumentReply doDecode(DocumentDeserializer buf); - - public boolean encode(Routable obj, DocumentSerializer out) { - if (!(obj instanceof DocumentReply)) { - throw new AssertionError( - "Document reply factory (" + getClass().getName() + ") registered for incompatible " + - "routable type " + obj.getType() + "(" + obj.getClass().getName() + ")."); - } - DocumentReply reply = (DocumentReply)obj; - out.putByte(null, (byte)(reply.getPriority().getValue())); - return doEncode(reply, out); - } - - public Routable decode(DocumentDeserializer in, LoadTypeSet loadTypes) { - byte pri = in.getByte(null); - DocumentReply reply = doDecode(in); - if (reply != null) { - reply.setPriority(DocumentProtocol.getPriority(pri)); - } - return reply; - } - } - - public static class CreateVisitorMessageFactory extends DocumentMessageFactory { - - protected String decodeBucketSpace(Deserializer deserializer) { - return FixedBucketSpaces.defaultSpace(); - } - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - CreateVisitorMessage msg = new CreateVisitorMessage(); - msg.setLibraryName(decodeString(buf)); - msg.setInstanceId(decodeString(buf)); - msg.setControlDestination(decodeString(buf)); - msg.setDataDestination(decodeString(buf)); - msg.setDocumentSelection(decodeString(buf)); - msg.setMaxPendingReplyCount(buf.getInt(null)); - - int size = buf.getInt(null); - for (int i = 0; i < size; i++) { - long reversed = buf.getLong(null); - long rawid = ((reversed >>> 56) & 0x00000000000000FFl) | ((reversed >>> 40) & 0x000000000000FF00l) | - ((reversed >>> 24) & 0x0000000000FF0000l) | ((reversed >>> 8) & 0x00000000FF000000l) | - ((reversed << 8) & 0x000000FF00000000l) | ((reversed << 24) & 0x0000FF0000000000l) | - ((reversed << 40) & 0x00FF000000000000l) | ((reversed << 56) & 0xFF00000000000000l); - msg.getBuckets().add(new BucketId(rawid)); - } - - msg.setFromTimestamp(buf.getLong(null)); - msg.setToTimestamp(buf.getLong(null)); - msg.setVisitRemoves(buf.getByte(null) == (byte)1); - msg.setFieldSet(decodeString(buf)); - msg.setVisitInconsistentBuckets(buf.getByte(null) == (byte)1); - - size = buf.getInt(null); - for (int i = 0; i < size; i++) { - String key = decodeString(buf); - int sz = buf.getInt(null); - msg.getParameters().put(key, buf.getBytes(null, sz)); - } - - msg.setVisitorOrdering(buf.getInt(null)); - msg.setMaxBucketsPerVisitor(buf.getInt(null)); - msg.setVisitorDispatcherVersion(50); - msg.setBucketSpace(decodeBucketSpace(buf)); - return msg; - } - - protected boolean encodeBucketSpace(String bucketSpace, DocumentSerializer buf) { - return FixedBucketSpaces.defaultSpace().equals(bucketSpace); - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - CreateVisitorMessage msg = (CreateVisitorMessage)obj; - encodeString(msg.getLibraryName(), buf); - encodeString(msg.getInstanceId(), buf); - encodeString(msg.getControlDestination(), buf); - encodeString(msg.getDataDestination(), buf); - encodeString(msg.getDocumentSelection(), buf); - buf.putInt(null, msg.getMaxPendingReplyCount()); - - buf.putInt(null, msg.getBuckets().size()); - for (BucketId id : msg.getBuckets()) { - long rawid = id.getRawId(); - long reversed = ((rawid >>> 56) & 0x00000000000000FFl) | ((rawid >>> 40) & 0x000000000000FF00l) | - ((rawid >>> 24) & 0x0000000000FF0000l) | ((rawid >>> 8) & 0x00000000FF000000l) | - ((rawid << 8) & 0x000000FF00000000l) | ((rawid << 24) & 0x0000FF0000000000l) | - ((rawid << 40) & 0x00FF000000000000l) | ((rawid << 56) & 0xFF00000000000000l); - buf.putLong(null, reversed); - } - - buf.putLong(null, msg.getFromTimestamp()); - buf.putLong(null, msg.getToTimestamp()); - buf.putByte(null, msg.getVisitRemoves() ? (byte)1 : (byte)0); - encodeString(msg.getFieldSet(), buf); - buf.putByte(null, msg.getVisitInconsistentBuckets() ? (byte)1 : (byte)0); - - buf.putInt(null, msg.getParameters().size()); - for (Map.Entry<String, byte[]> pairs : msg.getParameters().entrySet()) { - encodeString(pairs.getKey(), buf); - byte[] b = pairs.getValue(); - buf.putInt(null, b.length); - buf.put(null, b); - } - - buf.putInt(null, msg.getVisitorOrdering()); - buf.putInt(null, msg.getMaxBucketsPerVisitor()); - return encodeBucketSpace(msg.getBucketSpace(), buf); - } - } - - public static class CreateVisitorReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - CreateVisitorReply reply = new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR); - reply.setLastBucket(new BucketId(buf.getLong(null))); - - VisitorStatistics vs = new VisitorStatistics(); - vs.setBucketsVisited(buf.getInt(null)); - vs.setDocumentsVisited(buf.getLong(null)); - vs.setBytesVisited(buf.getLong(null)); - vs.setDocumentsReturned(buf.getLong(null)); - vs.setBytesReturned(buf.getLong(null)); - vs.setSecondPassDocumentsReturned(buf.getLong(null)); - vs.setSecondPassBytesReturned(buf.getLong(null)); - reply.setVisitorStatistics(vs); - return reply; - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - CreateVisitorReply reply = (CreateVisitorReply)obj; - buf.putLong(null, reply.getLastBucket().getRawId()); - buf.putInt(null, reply.getVisitorStatistics().getBucketsVisited()); - buf.putLong(null, reply.getVisitorStatistics().getDocumentsVisited()); - buf.putLong(null, reply.getVisitorStatistics().getBytesVisited()); - buf.putLong(null, reply.getVisitorStatistics().getDocumentsReturned()); - buf.putLong(null, reply.getVisitorStatistics().getBytesReturned()); - buf.putLong(null, reply.getVisitorStatistics().getSecondPassDocumentsReturned()); - buf.putLong(null, reply.getVisitorStatistics().getSecondPassBytesReturned()); - return true; - } - } - - public static class DestroyVisitorMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - DestroyVisitorMessage msg = new DestroyVisitorMessage(); - msg.setInstanceId(decodeString(buf)); - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - DestroyVisitorMessage msg = (DestroyVisitorMessage)obj; - encodeString(msg.getInstanceId(), buf); - return true; - } - } - - public static class DestroyVisitorReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new VisitorReply(DocumentProtocol.REPLY_DESTROYVISITOR); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - - public static class DocumentIgnoredReplyFactory extends DocumentReplyFactory { - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new DocumentIgnoredReply(); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - - public static class DocumentListMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - DocumentListMessage msg = new DocumentListMessage(); - msg.setBucketId(new BucketId(buf.getLong(null))); - int len = buf.getInt(null); - for (int i = 0; i < len; i++) { - msg.getDocuments().add(new DocumentListEntry(buf)); - } - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - DocumentListMessage msg = (DocumentListMessage)obj; - buf.putLong(null, msg.getBucketId().getRawId()); - buf.putInt(null, msg.getDocuments().size()); - - for (int i = 0; i < msg.getDocuments().size(); i++) { - msg.getDocuments().get(i).serialize(buf); - } - return true; - } - } - - public static class DocumentListReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new VisitorReply(DocumentProtocol.REPLY_DOCUMENTLIST); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - - public static class DocumentSummaryMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - DocumentSummaryMessage msg = new DocumentSummaryMessage(); - msg.setDocumentSummary(new DocumentSummary(buf)); - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - return false; // not supported - } - } - - public static class DocumentSummaryReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new VisitorReply(DocumentProtocol.REPLY_DOCUMENTSUMMARY); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - - public static class EmptyBucketsMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - EmptyBucketsMessage msg = new EmptyBucketsMessage(); - int size = buf.getInt(null); - for (int i = 0; i < size; ++i) { - msg.getBucketIds().add(new BucketId(buf.getLong(null))); - } - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - EmptyBucketsMessage msg = (EmptyBucketsMessage)obj; - buf.putInt(null, msg.getBucketIds().size()); - for (BucketId bid : msg.getBucketIds()) { - buf.putLong(null, bid.getRawId()); - } - return true; - } - } - - public static class EmptyBucketsReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new VisitorReply(DocumentProtocol.REPLY_EMPTYBUCKETS); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - - public static class GetBucketListMessageFactory extends DocumentMessageFactory { - - protected String decodeBucketSpace(Deserializer deserializer) { - return FixedBucketSpaces.defaultSpace(); - } - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - GetBucketListMessage msg = new GetBucketListMessage(); - msg.setBucketId(new BucketId(buf.getLong(null))); - msg.setBucketSpace(decodeBucketSpace(buf)); - return msg; - } - - protected boolean encodeBucketSpace(String bucketSpace, DocumentSerializer buf) { - return FixedBucketSpaces.defaultSpace().equals(bucketSpace); - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - GetBucketListMessage msg = (GetBucketListMessage)obj; - buf.putLong(null, msg.getBucketId().getRawId()); - return encodeBucketSpace(msg.getBucketSpace(), buf); - } - } - - public static class GetBucketListReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - GetBucketListReply reply = new GetBucketListReply(); - int len = buf.getInt(null); - for (int i = 0; i < len; i++) { - GetBucketListReply.BucketInfo info = new GetBucketListReply.BucketInfo(); - info.bucket = new BucketId(buf.getLong(null)); - info.bucketInformation = decodeString(buf); - reply.getBuckets().add(info); - } - return reply; - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - GetBucketListReply reply = (GetBucketListReply)obj; - buf.putInt(null, reply.getBuckets().size()); - for (GetBucketListReply.BucketInfo info : reply.getBuckets()) { - buf.putLong(null, info.bucket.getRawId()); - encodeString(info.bucketInformation, buf); - } - return true; - } - } - - public static class GetBucketStateMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - GetBucketStateMessage msg = new GetBucketStateMessage(); - msg.setBucketId(new BucketId(buf.getLong(null))); - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - GetBucketStateMessage msg = (GetBucketStateMessage)obj; - buf.putLong(null, msg.getBucketId().getRawId()); - return true; - } - } - - public static class GetBucketStateReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - GetBucketStateReply reply = new GetBucketStateReply(); - int size = buf.getInt(null); - for (int i = 0; i < size; i++) { - reply.getBucketState().add(new DocumentState(buf)); - } - return reply; - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - GetBucketStateReply reply = (GetBucketStateReply)obj; - buf.putInt(null, reply.getBucketState().size()); - for (DocumentState stat : reply.getBucketState()) { - stat.serialize(buf); - } - return true; - } - } - - public static class GetDocumentMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - return new GetDocumentMessage(new DocumentId(buf), decodeString(buf)); - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - GetDocumentMessage msg = (GetDocumentMessage)obj; - msg.getDocumentId().serialize(buf); - encodeString(msg.getFieldSet(), buf); - return true; - } - } - - public static class GetDocumentReplyFactory extends DocumentReplyFactory { - - private final LazyDecoder decoder = new LazyDecoder() { - - public void decode(Routable obj, DocumentDeserializer buf) { - GetDocumentReply reply = (GetDocumentReply)obj; - - Document doc = null; - byte flag = buf.getByte(null); - if (flag != 0) { - doc = Document.createDocument(buf); - reply.setDocument(doc); - } - long lastModified = buf.getLong(null); - reply.setLastModified(lastModified); - if (doc != null) { - doc.setLastModified(lastModified); - } - } - }; - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - GetDocumentReply reply = new GetDocumentReply(decoder, buf); - - return reply; - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - GetDocumentReply reply = (GetDocumentReply)obj; - if (reply.getSerializedBuffer() != null) { - buf.put(null, reply.getSerializedBuffer()); - } else { - Document document = reply.getDocument(); - buf.putByte(null, (byte)(document == null ? 0 : 1)); - if (document != null) { - document.serialize(buf); - } - buf.putLong(null, reply.getLastModified()); - } - return true; - } - } - - public static class MapVisitorMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - MapVisitorMessage msg = new MapVisitorMessage(); - int size = buf.getInt(null); - for (int i = 0; i < size; i++) { - String key = decodeString(buf); - String value = decodeString(buf); - msg.getData().put(key, value); - } - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - MapVisitorMessage msg = (MapVisitorMessage)obj; - buf.putInt(null, msg.getData().size()); - for (Map.Entry<String, String> pairs : msg.getData().entrySet()) { - encodeString(pairs.getKey(), buf); - encodeString(pairs.getValue(), buf); - } - return true; - } - } - - public static class MapVisitorReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new VisitorReply(DocumentProtocol.REPLY_MAPVISITOR); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - - public static class PutDocumentMessageFactory extends DocumentMessageFactory { - protected void decodeInto(PutDocumentMessage msg, DocumentDeserializer buf) { - msg.setDocumentPut(new DocumentPut(Document.createDocument(buf))); - msg.setTimestamp(buf.getLong(null)); - decodeTasCondition(msg, buf); - } - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buffer) { - final LazyDecoder decoder = (obj, buf) -> { - decodeInto((PutDocumentMessage) obj, buf); - }; - - return new PutDocumentMessage(decoder, buffer); - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - PutDocumentMessage msg = (PutDocumentMessage)obj; - if (msg.getSerializedBuffer() != null) { - buf.put(null, msg.getSerializedBuffer()); - } else { - msg.getDocumentPut().getDocument().serialize(buf); - buf.putLong(null, msg.getTimestamp()); - encodeTasCondition(buf, (TestAndSetMessage) obj); - } - return true; - } - } - - public static class PutDocumentReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - WriteDocumentReply rep = new WriteDocumentReply(DocumentProtocol.REPLY_PUTDOCUMENT); - rep.setHighestModificationTimestamp(buf.getLong(null)); - return rep; - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - WriteDocumentReply rep = (WriteDocumentReply)obj; - buf.putLong(null, rep.getHighestModificationTimestamp()); - return true; - } - } - - public static class RemoveDocumentMessageFactory extends DocumentMessageFactory { - protected void decodeInto(RemoveDocumentMessage msg, DocumentDeserializer buf) { - msg.setDocumentId(new DocumentId(buf)); - decodeTasCondition(msg, buf); - } - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - RemoveDocumentMessage msg = new RemoveDocumentMessage(); - decodeInto(msg, buf); - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - RemoveDocumentMessage msg = (RemoveDocumentMessage)obj; - msg.getDocumentId().serialize(buf); - encodeTasCondition(buf, (TestAndSetMessage) obj); - return true; - } - } - - public static class RemoveDocumentReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - RemoveDocumentReply reply = new RemoveDocumentReply(); - byte flag = buf.getByte(null); - reply.setWasFound(flag != 0); - reply.setHighestModificationTimestamp(buf.getLong(null)); - return reply; - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - RemoveDocumentReply reply = (RemoveDocumentReply)obj; - buf.putByte(null, (byte)(reply.wasFound() ? 1 : 0)); - buf.putLong(null, reply.getHighestModificationTimestamp()); - return true; - } - } - - public static class RemoveLocationMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - return new RemoveLocationMessage(decodeString(buf)); - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - RemoveLocationMessage msg = (RemoveLocationMessage)obj; - encodeString(msg.getDocumentSelection(), buf); - return true; - } - } - - public static class RemoveLocationReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new DocumentReply(DocumentProtocol.REPLY_REMOVELOCATION); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - - public static class SearchResultMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - SearchResultMessage msg = new SearchResultMessage(); - msg.setSearchResult(new SearchResult(buf)); - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - return false; // not supported - } - } - - public static class QueryResultMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - QueryResultMessage msg = new QueryResultMessage(); - msg.setSearchResult(new SearchResult(buf)); - msg.setSummary(new DocumentSummary(buf)); - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - return false; // not supported - } - } - - public static class SearchResultReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new VisitorReply(DocumentProtocol.REPLY_SEARCHRESULT); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - - public static class QueryResultReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new VisitorReply(DocumentProtocol.REPLY_QUERYRESULT); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - - public static class StatBucketMessageFactory extends DocumentMessageFactory { - - protected String decodeBucketSpace(Deserializer deserializer) { - return FixedBucketSpaces.defaultSpace(); - } - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - StatBucketMessage msg = new StatBucketMessage(); - msg.setBucketId(new BucketId(buf.getLong(null))); - msg.setDocumentSelection(decodeString(buf)); - msg.setBucketSpace(decodeBucketSpace(buf)); - return msg; - } - - protected boolean encodeBucketSpace(String bucketSpace, DocumentSerializer buf) { - return FixedBucketSpaces.defaultSpace().equals(bucketSpace); - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - StatBucketMessage msg = (StatBucketMessage)obj; - buf.putLong(null, msg.getBucketId().getRawId()); - encodeString(msg.getDocumentSelection(), buf); - return encodeBucketSpace(msg.getBucketSpace(), buf); - } - } - - public static class StatBucketReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - StatBucketReply reply = new StatBucketReply(); - reply.setResults(decodeString(buf)); - return reply; - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - StatBucketReply reply = (StatBucketReply)obj; - encodeString(reply.getResults(), buf); - return true; - } - } - - public static class UpdateDocumentMessageFactory extends DocumentMessageFactory { - protected void decodeInto(UpdateDocumentMessage msg, DocumentDeserializer buf) { - msg.setDocumentUpdate(new DocumentUpdate(buf)); - msg.setOldTimestamp(buf.getLong(null)); - msg.setNewTimestamp(buf.getLong(null)); - decodeTasCondition(msg, buf); - } - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buffer) { - final LazyDecoder decoder = (obj, buf) -> { - decodeInto((UpdateDocumentMessage) obj, buf); - }; - - return new UpdateDocumentMessage(decoder, buffer); - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - UpdateDocumentMessage msg = (UpdateDocumentMessage)obj; - if (msg.getSerializedBuffer() != null) { - buf.put(null, msg.getSerializedBuffer()); - } else { - msg.getDocumentUpdate().serialize(buf); - buf.putLong(null, msg.getOldTimestamp()); - buf.putLong(null, msg.getNewTimestamp()); - encodeTasCondition(buf, (TestAndSetMessage) obj); - } - return true; - } - } - - public static class UpdateDocumentReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - UpdateDocumentReply rep = new UpdateDocumentReply(); - byte flag = buf.getByte(null); - rep.setWasFound(flag != 0); - rep.setHighestModificationTimestamp(buf.getLong(null)); - return rep; - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - UpdateDocumentReply rep = (UpdateDocumentReply)obj; - buf.putByte(null, (byte)(rep.wasFound() ? 1 : 0)); - buf.putLong(null, rep.getHighestModificationTimestamp()); - return true; - } - } - - public static class VisitorInfoMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - VisitorInfoMessage msg = new VisitorInfoMessage(); - int size = buf.getInt(null); - for (int i = 0; i < size; i++) { - long reversed = buf.getLong(null); - long rawid = ((reversed >>> 56) & 0x00000000000000FFl) | ((reversed >>> 40) & 0x000000000000FF00l) | - ((reversed >>> 24) & 0x0000000000FF0000l) | ((reversed >>> 8) & 0x00000000FF000000l) | - ((reversed << 8) & 0x000000FF00000000l) | ((reversed << 24) & 0x0000FF0000000000l) | - ((reversed << 40) & 0x00FF000000000000l) | ((reversed << 56) & 0xFF00000000000000l); - msg.getFinishedBuckets().add(new BucketId(rawid)); - } - - msg.setErrorMessage(decodeString(buf)); - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - VisitorInfoMessage msg = (VisitorInfoMessage)obj; - buf.putInt(null, msg.getFinishedBuckets().size()); - for (BucketId id : msg.getFinishedBuckets()) { - long rawid = id.getRawId(); - long reversed = ((rawid >>> 56) & 0x00000000000000FFl) | ((rawid >>> 40) & 0x000000000000FF00l) | - ((rawid >>> 24) & 0x0000000000FF0000l) | ((rawid >>> 8) & 0x00000000FF000000l) | - ((rawid << 8) & 0x000000FF00000000l) | ((rawid << 24) & 0x0000FF0000000000l) | - ((rawid << 40) & 0x00FF000000000000l) | ((rawid << 56) & 0xFF00000000000000l); - buf.putLong(null, reversed); - } - encodeString(msg.getErrorMessage(), buf); - return true; - } - } - - public static class VisitorInfoReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new VisitorReply(DocumentProtocol.REPLY_VISITORINFO); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - - public static class WrongDistributionReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - WrongDistributionReply reply = new WrongDistributionReply(); - reply.setSystemState(decodeString(buf)); - return reply; - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - WrongDistributionReply reply = (WrongDistributionReply)obj; - encodeString(reply.getSystemState(), buf); - return true; - } - } - static void decodeTasCondition(TestAndSetMessage msg, DocumentDeserializer buf) { - msg.setCondition(new TestAndSetCondition(decodeString(buf))); - } - - static void encodeTasCondition(DocumentSerializer buf, TestAndSetMessage msg) { - encodeString(msg.getCondition().getSelection(), buf); - } -} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java index 723760df117..af3ff2992e5 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java @@ -1,48 +1,955 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi.messagebus.protocol; +import com.yahoo.document.BucketId; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.FixedBucketSpaces; +import com.yahoo.document.TestAndSetCondition; +import com.yahoo.document.serialization.DocumentDeserializer; import com.yahoo.document.serialization.DocumentSerializer; +import com.yahoo.document.serialization.DocumentSerializerFactory; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.log.LogLevel; +import com.yahoo.messagebus.Routable; +import com.yahoo.vdslib.DocumentSummary; +import com.yahoo.vdslib.SearchResult; +import com.yahoo.vdslib.VisitorStatistics; import com.yahoo.vespa.objects.Deserializer; +import com.yahoo.vespa.objects.Serializer; -public class RoutableFactories60 extends RoutableFactories52 { +import java.util.Map; +import java.util.logging.Logger; + +import static com.yahoo.documentapi.messagebus.protocol.AbstractRoutableFactory.decodeString; +import static com.yahoo.documentapi.messagebus.protocol.AbstractRoutableFactory.encodeString; + + +/** + * @author Vegard Sjonfjell + * This class encapsulates all the {@link RoutableFactory} classes needed to implement serialization for the document + * protocol. When adding new factories to this class, please KEEP THE THEM ORDERED alphabetically like they are now. + */ +public abstract class RoutableFactories60 { + + /** + * Implements the shared factory logic required for {@link DocumentMessage} objects, and it offers a more convenient + * interface for implementing {@link RoutableFactory}. + * + * @author Simon Thoresen Hult + */ + public static abstract class DocumentMessageFactory extends AbstractRoutableFactory { + + /** + * This method encodes the given message using the given serializer. You are guaranteed to only receive messages + * of the type that this factory was registered for. + * <p> + * This method is NOT exception safe. Return false to + * signal failure. + * + * @param msg The message to encode. + * @param serializer The serializer to use for encoding. + * @return True if the message was encoded. + */ + protected abstract boolean doEncode(DocumentMessage msg, DocumentSerializer serializer); + + /** + * This method decodes a message from the given deserializer. You are guaranteed to only receive byte buffers + * generated by a previous call to {@link #doEncode(DocumentMessage, DocumentSerializer)}. + * <p> + * This method is NOT exception safe. Return null to signal failure. + * + * @param deserializer The deserializer to use for decoding. + * @return The decoded message. + */ + protected abstract DocumentMessage doDecode(DocumentDeserializer deserializer); + + public boolean encode(Routable obj, DocumentSerializer out) { + if (!(obj instanceof DocumentMessage)) { + throw new AssertionError( + "Document message factory (" + getClass().getName() + ") registered for incompatible " + + "routable type " + obj.getType() + "(" + obj.getClass().getName() + ")."); + } + DocumentMessage msg = (DocumentMessage)obj; + out.putByte(null, (byte)(msg.getPriority().getValue())); + out.putInt(null, msg.getLoadType().getId()); + return doEncode(msg, out); + } + + public Routable decode(DocumentDeserializer in, LoadTypeSet loadTypes) { + byte pri = in.getByte(null); + int loadType = in.getInt(null); + DocumentMessage msg = doDecode(in); + if (msg != null) { + msg.setPriority(DocumentProtocol.getPriority(pri)); + msg.setLoadType(loadTypes.getIdMap().get(loadType)); + } + return msg; + } + } + + /** + * Implements the shared factory logic required for {@link DocumentReply} objects, and it offers a more convenient + * interface for implementing {@link RoutableFactory}. + * + * @author Simon Thoresen Hult + */ + public static abstract class DocumentReplyFactory extends AbstractRoutableFactory { + + /** + * This method encodes the given reply into the given byte buffer. You are guaranteed to only receive replies of + * the type that this factory was registered for. + * <p> + * This method is NOT exception safe. Return false to signal + * failure. + * + * @param reply The reply to encode. + * @param buf The byte buffer to write to. + * @return True if the message was encoded. + */ + protected abstract boolean doEncode(DocumentReply reply, DocumentSerializer buf); + + /** + * This method decodes a reply from the given byte buffer. You are guaranteed to only receive byte buffers + * generated by a previous call to {@link #doEncode(DocumentReply, com.yahoo.document.serialization.DocumentSerializer)}. + * + * <p> + * This method is NOT exception safe. Return null to signal failure. + * + * @param buf The byte buffer to read from. + * @return The decoded reply. + */ + protected abstract DocumentReply doDecode(DocumentDeserializer buf); + + public boolean encode(Routable obj, DocumentSerializer out) { + if (!(obj instanceof DocumentReply)) { + throw new AssertionError( + "Document reply factory (" + getClass().getName() + ") registered for incompatible " + + "routable type " + obj.getType() + "(" + obj.getClass().getName() + ")."); + } + DocumentReply reply = (DocumentReply)obj; + out.putByte(null, (byte)(reply.getPriority().getValue())); + return doEncode(reply, out); + } + + public Routable decode(DocumentDeserializer in, LoadTypeSet loadTypes) { + byte pri = in.getByte(null); + DocumentReply reply = doDecode(in); + if (reply != null) { + reply.setPriority(DocumentProtocol.getPriority(pri)); + } + return reply; + } + } + + public static class CreateVisitorMessageFactory extends DocumentMessageFactory { - public static class CreateVisitorMessageFactory extends RoutableFactories52.CreateVisitorMessageFactory { - @Override protected String decodeBucketSpace(Deserializer deserializer) { return decodeString(deserializer); } @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + CreateVisitorMessage msg = new CreateVisitorMessage(); + msg.setLibraryName(decodeString(buf)); + msg.setInstanceId(decodeString(buf)); + msg.setControlDestination(decodeString(buf)); + msg.setDataDestination(decodeString(buf)); + msg.setDocumentSelection(decodeString(buf)); + msg.setMaxPendingReplyCount(buf.getInt(null)); + + int size = buf.getInt(null); + for (int i = 0; i < size; i++) { + long reversed = buf.getLong(null); + long rawid = ((reversed >>> 56) & 0x00000000000000FFl) | ((reversed >>> 40) & 0x000000000000FF00l) | + ((reversed >>> 24) & 0x0000000000FF0000l) | ((reversed >>> 8) & 0x00000000FF000000l) | + ((reversed << 8) & 0x000000FF00000000l) | ((reversed << 24) & 0x0000FF0000000000l) | + ((reversed << 40) & 0x00FF000000000000l) | ((reversed << 56) & 0xFF00000000000000l); + msg.getBuckets().add(new BucketId(rawid)); + } + + msg.setFromTimestamp(buf.getLong(null)); + msg.setToTimestamp(buf.getLong(null)); + msg.setVisitRemoves(buf.getByte(null) == (byte)1); + msg.setFieldSet(decodeString(buf)); + msg.setVisitInconsistentBuckets(buf.getByte(null) == (byte)1); + + size = buf.getInt(null); + for (int i = 0; i < size; i++) { + String key = decodeString(buf); + int sz = buf.getInt(null); + msg.getParameters().put(key, buf.getBytes(null, sz)); + } + + msg.setVisitorOrdering(buf.getInt(null)); + msg.setMaxBucketsPerVisitor(buf.getInt(null)); + msg.setVisitorDispatcherVersion(50); + msg.setBucketSpace(decodeBucketSpace(buf)); + return msg; + } + protected boolean encodeBucketSpace(String bucketSpace, DocumentSerializer buf) { encodeString(bucketSpace, buf); return true; } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + CreateVisitorMessage msg = (CreateVisitorMessage)obj; + encodeString(msg.getLibraryName(), buf); + encodeString(msg.getInstanceId(), buf); + encodeString(msg.getControlDestination(), buf); + encodeString(msg.getDataDestination(), buf); + encodeString(msg.getDocumentSelection(), buf); + buf.putInt(null, msg.getMaxPendingReplyCount()); + + buf.putInt(null, msg.getBuckets().size()); + for (BucketId id : msg.getBuckets()) { + long rawid = id.getRawId(); + long reversed = ((rawid >>> 56) & 0x00000000000000FFl) | ((rawid >>> 40) & 0x000000000000FF00l) | + ((rawid >>> 24) & 0x0000000000FF0000l) | ((rawid >>> 8) & 0x00000000FF000000l) | + ((rawid << 8) & 0x000000FF00000000l) | ((rawid << 24) & 0x0000FF0000000000l) | + ((rawid << 40) & 0x00FF000000000000l) | ((rawid << 56) & 0xFF00000000000000l); + buf.putLong(null, reversed); + } + + buf.putLong(null, msg.getFromTimestamp()); + buf.putLong(null, msg.getToTimestamp()); + buf.putByte(null, msg.getVisitRemoves() ? (byte)1 : (byte)0); + encodeString(msg.getFieldSet(), buf); + buf.putByte(null, msg.getVisitInconsistentBuckets() ? (byte)1 : (byte)0); + + buf.putInt(null, msg.getParameters().size()); + for (Map.Entry<String, byte[]> pairs : msg.getParameters().entrySet()) { + encodeString(pairs.getKey(), buf); + byte[] b = pairs.getValue(); + buf.putInt(null, b.length); + buf.put(null, b); + } + + buf.putInt(null, msg.getVisitorOrdering()); + buf.putInt(null, msg.getMaxBucketsPerVisitor()); + return encodeBucketSpace(msg.getBucketSpace(), buf); + } + } + + public static class CreateVisitorReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + CreateVisitorReply reply = new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR); + reply.setLastBucket(new BucketId(buf.getLong(null))); + + VisitorStatistics vs = new VisitorStatistics(); + vs.setBucketsVisited(buf.getInt(null)); + vs.setDocumentsVisited(buf.getLong(null)); + vs.setBytesVisited(buf.getLong(null)); + vs.setDocumentsReturned(buf.getLong(null)); + vs.setBytesReturned(buf.getLong(null)); + vs.setSecondPassDocumentsReturned(buf.getLong(null)); + vs.setSecondPassBytesReturned(buf.getLong(null)); + reply.setVisitorStatistics(vs); + return reply; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + CreateVisitorReply reply = (CreateVisitorReply)obj; + buf.putLong(null, reply.getLastBucket().getRawId()); + buf.putInt(null, reply.getVisitorStatistics().getBucketsVisited()); + buf.putLong(null, reply.getVisitorStatistics().getDocumentsVisited()); + buf.putLong(null, reply.getVisitorStatistics().getBytesVisited()); + buf.putLong(null, reply.getVisitorStatistics().getDocumentsReturned()); + buf.putLong(null, reply.getVisitorStatistics().getBytesReturned()); + buf.putLong(null, reply.getVisitorStatistics().getSecondPassDocumentsReturned()); + buf.putLong(null, reply.getVisitorStatistics().getSecondPassBytesReturned()); + return true; + } + } + + public static class DestroyVisitorMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + DestroyVisitorMessage msg = new DestroyVisitorMessage(); + msg.setInstanceId(decodeString(buf)); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + DestroyVisitorMessage msg = (DestroyVisitorMessage)obj; + encodeString(msg.getInstanceId(), buf); + return true; + } + } + + public static class DestroyVisitorReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_DESTROYVISITOR); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class DocumentIgnoredReplyFactory extends DocumentReplyFactory { + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new DocumentIgnoredReply(); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class DocumentListMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + DocumentListMessage msg = new DocumentListMessage(); + msg.setBucketId(new BucketId(buf.getLong(null))); + int len = buf.getInt(null); + for (int i = 0; i < len; i++) { + msg.getDocuments().add(new DocumentListEntry(buf)); + } + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + DocumentListMessage msg = (DocumentListMessage)obj; + buf.putLong(null, msg.getBucketId().getRawId()); + buf.putInt(null, msg.getDocuments().size()); + + for (int i = 0; i < msg.getDocuments().size(); i++) { + msg.getDocuments().get(i).serialize(buf); + } + return true; + } + } + + public static class DocumentListReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_DOCUMENTLIST); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class DocumentSummaryMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + DocumentSummaryMessage msg = new DocumentSummaryMessage(); + msg.setDocumentSummary(new DocumentSummary(buf)); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + return false; // not supported + } + } + + public static class DocumentSummaryReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_DOCUMENTSUMMARY); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class EmptyBucketsMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + EmptyBucketsMessage msg = new EmptyBucketsMessage(); + int size = buf.getInt(null); + for (int i = 0; i < size; ++i) { + msg.getBucketIds().add(new BucketId(buf.getLong(null))); + } + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + EmptyBucketsMessage msg = (EmptyBucketsMessage)obj; + buf.putInt(null, msg.getBucketIds().size()); + for (BucketId bid : msg.getBucketIds()) { + buf.putLong(null, bid.getRawId()); + } + return true; + } } - public static class StatBucketMessageFactory extends RoutableFactories52.StatBucketMessageFactory { + public static class EmptyBucketsReplyFactory extends DocumentReplyFactory { + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_EMPTYBUCKETS); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class GetBucketListMessageFactory extends DocumentMessageFactory { + protected String decodeBucketSpace(Deserializer deserializer) { return decodeString(deserializer); } @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + GetBucketListMessage msg = new GetBucketListMessage(); + msg.setBucketId(new BucketId(buf.getLong(null))); + msg.setBucketSpace(decodeBucketSpace(buf)); + return msg; + } + protected boolean encodeBucketSpace(String bucketSpace, DocumentSerializer buf) { encodeString(bucketSpace, buf); return true; } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + GetBucketListMessage msg = (GetBucketListMessage)obj; + buf.putLong(null, msg.getBucketId().getRawId()); + return encodeBucketSpace(msg.getBucketSpace(), buf); + } + } + + public static class GetBucketListReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + GetBucketListReply reply = new GetBucketListReply(); + int len = buf.getInt(null); + for (int i = 0; i < len; i++) { + GetBucketListReply.BucketInfo info = new GetBucketListReply.BucketInfo(); + info.bucket = new BucketId(buf.getLong(null)); + info.bucketInformation = decodeString(buf); + reply.getBuckets().add(info); + } + return reply; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + GetBucketListReply reply = (GetBucketListReply)obj; + buf.putInt(null, reply.getBuckets().size()); + for (GetBucketListReply.BucketInfo info : reply.getBuckets()) { + buf.putLong(null, info.bucket.getRawId()); + encodeString(info.bucketInformation, buf); + } + return true; + } + } + + public static class GetBucketStateMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + GetBucketStateMessage msg = new GetBucketStateMessage(); + msg.setBucketId(new BucketId(buf.getLong(null))); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + GetBucketStateMessage msg = (GetBucketStateMessage)obj; + buf.putLong(null, msg.getBucketId().getRawId()); + return true; + } + } + + public static class GetBucketStateReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + GetBucketStateReply reply = new GetBucketStateReply(); + int size = buf.getInt(null); + for (int i = 0; i < size; i++) { + reply.getBucketState().add(new DocumentState(buf)); + } + return reply; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + GetBucketStateReply reply = (GetBucketStateReply)obj; + buf.putInt(null, reply.getBucketState().size()); + for (DocumentState stat : reply.getBucketState()) { + stat.serialize(buf); + } + return true; + } + } + + public static class GetDocumentMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + return new GetDocumentMessage(new DocumentId(buf), decodeString(buf)); + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + GetDocumentMessage msg = (GetDocumentMessage)obj; + msg.getDocumentId().serialize(buf); + encodeString(msg.getFieldSet(), buf); + return true; + } } - public static class GetBucketListMessageFactory extends RoutableFactories52.GetBucketListMessageFactory { + public static class GetDocumentReplyFactory extends DocumentReplyFactory { + + private final LazyDecoder decoder = new LazyDecoder() { + + public void decode(Routable obj, DocumentDeserializer buf) { + GetDocumentReply reply = (GetDocumentReply)obj; + + Document doc = null; + byte flag = buf.getByte(null); + if (flag != 0) { + doc = Document.createDocument(buf); + reply.setDocument(doc); + } + long lastModified = buf.getLong(null); + reply.setLastModified(lastModified); + if (doc != null) { + doc.setLastModified(lastModified); + } + } + }; + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + GetDocumentReply reply = new GetDocumentReply(decoder, buf); + + return reply; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + GetDocumentReply reply = (GetDocumentReply)obj; + if (reply.getSerializedBuffer() != null) { + buf.put(null, reply.getSerializedBuffer()); + } else { + Document document = reply.getDocument(); + buf.putByte(null, (byte)(document == null ? 0 : 1)); + if (document != null) { + document.serialize(buf); + } + buf.putLong(null, reply.getLastModified()); + } + return true; + } + } + + public static class MapVisitorMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + MapVisitorMessage msg = new MapVisitorMessage(); + int size = buf.getInt(null); + for (int i = 0; i < size; i++) { + String key = decodeString(buf); + String value = decodeString(buf); + msg.getData().put(key, value); + } + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + MapVisitorMessage msg = (MapVisitorMessage)obj; + buf.putInt(null, msg.getData().size()); + for (Map.Entry<String, String> pairs : msg.getData().entrySet()) { + encodeString(pairs.getKey(), buf); + encodeString(pairs.getValue(), buf); + } + return true; + } + } + + public static class MapVisitorReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_MAPVISITOR); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class PutDocumentMessageFactory extends DocumentMessageFactory { + protected void decodeInto(PutDocumentMessage msg, DocumentDeserializer buf) { + msg.setDocumentPut(new DocumentPut(Document.createDocument(buf))); + msg.setTimestamp(buf.getLong(null)); + decodeTasCondition(msg, buf); + } + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buffer) { + final LazyDecoder decoder = (obj, buf) -> { + decodeInto((PutDocumentMessage) obj, buf); + }; + + return new PutDocumentMessage(decoder, buffer); + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + PutDocumentMessage msg = (PutDocumentMessage)obj; + if (msg.getSerializedBuffer() != null) { + buf.put(null, msg.getSerializedBuffer()); + } else { + msg.getDocumentPut().getDocument().serialize(buf); + buf.putLong(null, msg.getTimestamp()); + encodeTasCondition(buf, (TestAndSetMessage) obj); + } + return true; + } + } + + public static class PutDocumentReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + WriteDocumentReply rep = new WriteDocumentReply(DocumentProtocol.REPLY_PUTDOCUMENT); + rep.setHighestModificationTimestamp(buf.getLong(null)); + return rep; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + WriteDocumentReply rep = (WriteDocumentReply)obj; + buf.putLong(null, rep.getHighestModificationTimestamp()); + return true; + } + } + + public static class RemoveDocumentMessageFactory extends DocumentMessageFactory { + protected void decodeInto(RemoveDocumentMessage msg, DocumentDeserializer buf) { + msg.setDocumentId(new DocumentId(buf)); + decodeTasCondition(msg, buf); + } + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + RemoveDocumentMessage msg = new RemoveDocumentMessage(); + decodeInto(msg, buf); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + RemoveDocumentMessage msg = (RemoveDocumentMessage)obj; + msg.getDocumentId().serialize(buf); + encodeTasCondition(buf, (TestAndSetMessage) obj); + return true; + } + } + + public static class RemoveDocumentReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + RemoveDocumentReply reply = new RemoveDocumentReply(); + byte flag = buf.getByte(null); + reply.setWasFound(flag != 0); + reply.setHighestModificationTimestamp(buf.getLong(null)); + return reply; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + RemoveDocumentReply reply = (RemoveDocumentReply)obj; + buf.putByte(null, (byte)(reply.wasFound() ? 1 : 0)); + buf.putLong(null, reply.getHighestModificationTimestamp()); + return true; + } + } + + public static class RemoveLocationMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + return new RemoveLocationMessage(decodeString(buf)); + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + RemoveLocationMessage msg = (RemoveLocationMessage)obj; + encodeString(msg.getDocumentSelection(), buf); + return true; + } + } + + public static class RemoveLocationReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new DocumentReply(DocumentProtocol.REPLY_REMOVELOCATION); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class SearchResultMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + SearchResultMessage msg = new SearchResultMessage(); + msg.setSearchResult(new SearchResult(buf)); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + return false; // not supported + } + } + + public static class QueryResultMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + QueryResultMessage msg = new QueryResultMessage(); + msg.setSearchResult(new SearchResult(buf)); + msg.setSummary(new DocumentSummary(buf)); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + return false; // not supported + } + } + + public static class SearchResultReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_SEARCHRESULT); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class QueryResultReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_QUERYRESULT); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class StatBucketMessageFactory extends DocumentMessageFactory { + protected String decodeBucketSpace(Deserializer deserializer) { return decodeString(deserializer); } @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + StatBucketMessage msg = new StatBucketMessage(); + msg.setBucketId(new BucketId(buf.getLong(null))); + msg.setDocumentSelection(decodeString(buf)); + msg.setBucketSpace(decodeBucketSpace(buf)); + return msg; + } + protected boolean encodeBucketSpace(String bucketSpace, DocumentSerializer buf) { encodeString(bucketSpace, buf); return true; } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + StatBucketMessage msg = (StatBucketMessage)obj; + buf.putLong(null, msg.getBucketId().getRawId()); + encodeString(msg.getDocumentSelection(), buf); + return encodeBucketSpace(msg.getBucketSpace(), buf); + } + } + + public static class StatBucketReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + StatBucketReply reply = new StatBucketReply(); + reply.setResults(decodeString(buf)); + return reply; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + StatBucketReply reply = (StatBucketReply)obj; + encodeString(reply.getResults(), buf); + return true; + } } + public static class UpdateDocumentMessageFactory extends DocumentMessageFactory { + protected void decodeInto(UpdateDocumentMessage msg, DocumentDeserializer buf) { + msg.setDocumentUpdate(new DocumentUpdate(buf)); + msg.setOldTimestamp(buf.getLong(null)); + msg.setNewTimestamp(buf.getLong(null)); + decodeTasCondition(msg, buf); + } + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buffer) { + final LazyDecoder decoder = (obj, buf) -> { + decodeInto((UpdateDocumentMessage) obj, buf); + }; + + return new UpdateDocumentMessage(decoder, buffer); + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + UpdateDocumentMessage msg = (UpdateDocumentMessage)obj; + if (msg.getSerializedBuffer() != null) { + buf.put(null, msg.getSerializedBuffer()); + } else { + msg.getDocumentUpdate().serialize(buf); + buf.putLong(null, msg.getOldTimestamp()); + buf.putLong(null, msg.getNewTimestamp()); + encodeTasCondition(buf, (TestAndSetMessage) obj); + } + return true; + } + } + + public static class UpdateDocumentReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + UpdateDocumentReply rep = new UpdateDocumentReply(); + byte flag = buf.getByte(null); + rep.setWasFound(flag != 0); + rep.setHighestModificationTimestamp(buf.getLong(null)); + return rep; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + UpdateDocumentReply rep = (UpdateDocumentReply)obj; + buf.putByte(null, (byte)(rep.wasFound() ? 1 : 0)); + buf.putLong(null, rep.getHighestModificationTimestamp()); + return true; + } + } + + public static class VisitorInfoMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + VisitorInfoMessage msg = new VisitorInfoMessage(); + int size = buf.getInt(null); + for (int i = 0; i < size; i++) { + long reversed = buf.getLong(null); + long rawid = ((reversed >>> 56) & 0x00000000000000FFl) | ((reversed >>> 40) & 0x000000000000FF00l) | + ((reversed >>> 24) & 0x0000000000FF0000l) | ((reversed >>> 8) & 0x00000000FF000000l) | + ((reversed << 8) & 0x000000FF00000000l) | ((reversed << 24) & 0x0000FF0000000000l) | + ((reversed << 40) & 0x00FF000000000000l) | ((reversed << 56) & 0xFF00000000000000l); + msg.getFinishedBuckets().add(new BucketId(rawid)); + } + + msg.setErrorMessage(decodeString(buf)); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + VisitorInfoMessage msg = (VisitorInfoMessage)obj; + buf.putInt(null, msg.getFinishedBuckets().size()); + for (BucketId id : msg.getFinishedBuckets()) { + long rawid = id.getRawId(); + long reversed = ((rawid >>> 56) & 0x00000000000000FFl) | ((rawid >>> 40) & 0x000000000000FF00l) | + ((rawid >>> 24) & 0x0000000000FF0000l) | ((rawid >>> 8) & 0x00000000FF000000l) | + ((rawid << 8) & 0x000000FF00000000l) | ((rawid << 24) & 0x0000FF0000000000l) | + ((rawid << 40) & 0x00FF000000000000l) | ((rawid << 56) & 0xFF00000000000000l); + buf.putLong(null, reversed); + } + encodeString(msg.getErrorMessage(), buf); + return true; + } + } + + public static class VisitorInfoReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_VISITORINFO); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class WrongDistributionReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + WrongDistributionReply reply = new WrongDistributionReply(); + reply.setSystemState(decodeString(buf)); + return reply; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + WrongDistributionReply reply = (WrongDistributionReply)obj; + encodeString(reply.getSystemState(), buf); + return true; + } + } + static void decodeTasCondition(TestAndSetMessage msg, DocumentDeserializer buf) { + msg.setCondition(new TestAndSetCondition(decodeString(buf))); + } + + static void encodeTasCondition(DocumentSerializer buf, TestAndSetMessage msg) { + encodeString(msg.getCondition().getSelection(), buf); + } } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/VisitorDataQueueTest.java b/documentapi/src/test/java/com/yahoo/documentapi/VisitorDataQueueTest.java index fbe2abf2e20..7cd8cf05c8e 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/VisitorDataQueueTest.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/VisitorDataQueueTest.java @@ -10,7 +10,6 @@ import com.yahoo.document.DocumentTypeManagerConfigurer; import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; -import com.yahoo.vdslib.Entry; import org.junit.Before; import org.junit.Test; @@ -42,17 +41,15 @@ public class VisitorDataQueueTest { return new AckToken(new Object()); } - private static void assertNonNullDocumentListResponse(final VisitorResponse response) { + private static void assertNonNullDocumentOpResponse(final VisitorResponse response) { assertThat(response, notNullValue()); - assertThat(response, instanceOf(DocumentListVisitorResponse.class)); + assertThat(response, instanceOf(DocumentOpVisitorResponse.class)); } private static void assertResponseHasSinglePut(final VisitorResponse response, final DocumentPut expectedInstance) { - assertNonNullDocumentListResponse(response); - final DocumentListVisitorResponse visitorResponse = (DocumentListVisitorResponse)response; - assertThat(visitorResponse.getDocumentList().size(), equalTo(1)); - final Entry entry = visitorResponse.getDocumentList().get(0); - assertThat(entry.getDocumentOperation(), is(expectedInstance)); + assertNonNullDocumentOpResponse(response); + final DocumentOpVisitorResponse visitorResponse = (DocumentOpVisitorResponse)response; + assertThat(visitorResponse.getDocumentOperation(), is(expectedInstance)); } @Test @@ -78,13 +75,10 @@ public class VisitorDataQueueTest { } private static void assertResponseHasSingleRemove(final VisitorResponse response, final String docId) { - assertNonNullDocumentListResponse(response); - final DocumentListVisitorResponse visitorResponse = (DocumentListVisitorResponse)response; - assertThat(visitorResponse.getDocumentList().size(), equalTo(1)); - final Entry entry = visitorResponse.getDocumentList().get(0); - assertThat(entry.isRemoveEntry(), is(true)); - assertThat(entry.getDocumentOperation(), instanceOf(DocumentRemove.class)); - assertThat(entry.getDocumentOperation().getId(), equalTo(new DocumentId(docId))); + assertNonNullDocumentOpResponse(response); + final DocumentOpVisitorResponse visitorResponse = (DocumentOpVisitorResponse)response; + assertThat(visitorResponse.getDocumentOperation(), instanceOf(DocumentRemove.class)); + assertThat(visitorResponse.getDocumentOperation().getId(), equalTo(new DocumentId(docId))); } @Test diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java deleted file mode 100644 index 448c8fcb520..00000000000 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java +++ /dev/null @@ -1,921 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol.test; - -import com.yahoo.component.Version; -import com.yahoo.document.BucketId; -import com.yahoo.document.Document; -import com.yahoo.document.DocumentId; -import com.yahoo.document.DocumentPut; -import com.yahoo.document.DocumentType; -import com.yahoo.document.DocumentUpdate; -import com.yahoo.document.GlobalId; -import com.yahoo.document.TestAndSetCondition; -import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate; -import com.yahoo.document.idstring.IdString; -import com.yahoo.document.select.OrderingSpecification; -import com.yahoo.documentapi.messagebus.protocol.CreateVisitorMessage; -import com.yahoo.documentapi.messagebus.protocol.CreateVisitorReply; -import com.yahoo.documentapi.messagebus.protocol.DestroyVisitorMessage; -import com.yahoo.documentapi.messagebus.protocol.DocumentIgnoredReply; -import com.yahoo.documentapi.messagebus.protocol.DocumentListMessage; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.documentapi.messagebus.protocol.DocumentReply; -import com.yahoo.documentapi.messagebus.protocol.DocumentState; -import com.yahoo.documentapi.messagebus.protocol.DocumentSummaryMessage; -import com.yahoo.documentapi.messagebus.protocol.EmptyBucketsMessage; -import com.yahoo.documentapi.messagebus.protocol.GetBucketListMessage; -import com.yahoo.documentapi.messagebus.protocol.GetBucketListReply; -import com.yahoo.documentapi.messagebus.protocol.GetBucketStateMessage; -import com.yahoo.documentapi.messagebus.protocol.GetBucketStateReply; -import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; -import com.yahoo.documentapi.messagebus.protocol.MapVisitorMessage; -import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.QueryResultMessage; -import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentReply; -import com.yahoo.documentapi.messagebus.protocol.RemoveLocationMessage; -import com.yahoo.documentapi.messagebus.protocol.SearchResultMessage; -import com.yahoo.documentapi.messagebus.protocol.StatBucketMessage; -import com.yahoo.documentapi.messagebus.protocol.StatBucketReply; -import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentReply; -import com.yahoo.documentapi.messagebus.protocol.VisitorInfoMessage; -import com.yahoo.documentapi.messagebus.protocol.VisitorReply; -import com.yahoo.documentapi.messagebus.protocol.WriteDocumentReply; -import com.yahoo.documentapi.messagebus.protocol.WrongDistributionReply; -import com.yahoo.messagebus.Routable; -import com.yahoo.text.Utf8; -import com.yahoo.vdslib.SearchResult; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -/** - * @author Simon Thoresen Hult - * @author Vegard Sjonfjell - */ - -public class Messages52TestCase extends MessagesTestBase { - @Override - protected void registerTests(Map<Integer, RunnableTest> out) { - // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support - // version 5.2. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now. - out.put(DocumentProtocol.MESSAGE_CREATEVISITOR, new testCreateVisitorMessage()); - out.put(DocumentProtocol.MESSAGE_DESTROYVISITOR, new testDestroyVisitorMessage()); - out.put(DocumentProtocol.MESSAGE_DOCUMENTLIST, new testDocumentListMessage()); - out.put(DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, new testDocumentSummaryMessage()); - out.put(DocumentProtocol.MESSAGE_EMPTYBUCKETS, new testEmptyBucketsMessage()); - out.put(DocumentProtocol.MESSAGE_GETBUCKETLIST, new testGetBucketListMessage()); - out.put(DocumentProtocol.MESSAGE_GETBUCKETSTATE, new testGetBucketStateMessage()); - out.put(DocumentProtocol.MESSAGE_GETDOCUMENT, new testGetDocumentMessage()); - out.put(DocumentProtocol.MESSAGE_MAPVISITOR, new testMapVisitorMessage()); - out.put(DocumentProtocol.MESSAGE_PUTDOCUMENT, new testPutDocumentMessage()); - out.put(DocumentProtocol.MESSAGE_QUERYRESULT, new testQueryResultMessage()); - out.put(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, new testRemoveDocumentMessage()); - out.put(DocumentProtocol.MESSAGE_REMOVELOCATION, new testRemoveLocationMessage()); - out.put(DocumentProtocol.MESSAGE_SEARCHRESULT, new testSearchResultMessage()); - out.put(DocumentProtocol.MESSAGE_STATBUCKET, new testStatBucketMessage()); - out.put(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, new testUpdateDocumentMessage()); - out.put(DocumentProtocol.MESSAGE_VISITORINFO, new testVisitorInfoMessage()); - out.put(DocumentProtocol.REPLY_CREATEVISITOR, new testCreateVisitorReply()); - out.put(DocumentProtocol.REPLY_DESTROYVISITOR, new testDestroyVisitorReply()); - out.put(DocumentProtocol.REPLY_DOCUMENTIGNORED, new testDocumentIgnoredReply()); - out.put(DocumentProtocol.REPLY_DOCUMENTLIST, new testDocumentListReply()); - out.put(DocumentProtocol.REPLY_DOCUMENTSUMMARY, new testDocumentSummaryReply()); - out.put(DocumentProtocol.REPLY_EMPTYBUCKETS, new testEmptyBucketsReply()); - out.put(DocumentProtocol.REPLY_GETBUCKETLIST, new testGetBucketListReply()); - out.put(DocumentProtocol.REPLY_GETBUCKETSTATE, new testGetBucketStateReply()); - out.put(DocumentProtocol.REPLY_GETDOCUMENT, new testGetDocumentReply()); - out.put(DocumentProtocol.REPLY_MAPVISITOR, new testMapVisitorReply()); - out.put(DocumentProtocol.REPLY_PUTDOCUMENT, new testPutDocumentReply()); - out.put(DocumentProtocol.REPLY_QUERYRESULT, new testQueryResultReply()); - out.put(DocumentProtocol.REPLY_REMOVEDOCUMENT, new testRemoveDocumentReply()); - out.put(DocumentProtocol.REPLY_REMOVELOCATION, new testRemoveLocationReply()); - out.put(DocumentProtocol.REPLY_SEARCHRESULT, new testSearchResultReply()); - out.put(DocumentProtocol.REPLY_STATBUCKET, new testStatBucketReply()); - out.put(DocumentProtocol.REPLY_UPDATEDOCUMENT, new testUpdateDocumentReply()); - out.put(DocumentProtocol.REPLY_VISITORINFO, new testVisitorInfoReply()); - out.put(DocumentProtocol.REPLY_WRONGDISTRIBUTION, new testWrongDistributionReply()); - } - - @Override - protected Version version() { - return new Version(5, 115, 0); - } - - @Override - protected boolean shouldTestCoverage() { - return true; - } - - //////////////////////////////////////////////////////////////////////////////// - // - // Tests - // - //////////////////////////////////////////////////////////////////////////////// - - protected static int BASE_MESSAGE_LENGTH = 5; - - public class testRemoveLocationMessage implements RunnableTest { - - @Override - public void run() { - { - RemoveLocationMessage msg = new RemoveLocationMessage("id.group == \"mygroup\""); - assertEquals(BASE_MESSAGE_LENGTH + 29, serialize("RemoveLocationMessage", msg)); - - for (Language lang : LANGUAGES) { - msg = (RemoveLocationMessage)deserialize("RemoveLocationMessage", DocumentProtocol.MESSAGE_REMOVELOCATION, lang); - assertEquals("id.group == \"mygroup\"", msg.getDocumentSelection()); - } - } - } - } - - public class testGetBucketListMessage implements RunnableTest { - - @Override - public void run() { - GetBucketListMessage msg = new GetBucketListMessage(new BucketId(16, 123)); - msg.setLoadType(loadTypes.getNameMap().get("foo")); - assertEquals(BASE_MESSAGE_LENGTH + 12, serialize("GetBucketListMessage", msg)); - - for (Language lang : LANGUAGES) { - msg = (GetBucketListMessage)deserialize("GetBucketListMessage", DocumentProtocol.MESSAGE_GETBUCKETLIST, lang); - assertEquals(new BucketId(16, 123), msg.getBucketId()); - assertEquals("foo", msg.getLoadType().getName()); - } - } - } - - - public class testStatBucketMessage implements RunnableTest { - - @Override - public void run() { - StatBucketMessage msg = new StatBucketMessage(new BucketId(16, 123), "id.user=123"); - msg.setLoadType(null); - assertEquals(BASE_MESSAGE_LENGTH + 27, serialize("StatBucketMessage", msg)); - - for (Language lang : LANGUAGES) { - msg = (StatBucketMessage)deserialize("StatBucketMessage", DocumentProtocol.MESSAGE_STATBUCKET, lang); - assertEquals(new BucketId(16, 123), msg.getBucketId()); - assertEquals("id.user=123", msg.getDocumentSelection()); - assertEquals("default", msg.getLoadType().getName()); - } - } - } - - public class testGetBucketStateMessage implements RunnableTest { - - @Override - public void run() { - GetBucketStateMessage msg = new GetBucketStateMessage(new BucketId(16, 666)); - assertEquals(BASE_MESSAGE_LENGTH + 12, serialize("GetBucketStateMessage", msg)); - - for (Language lang : LANGUAGES) { - msg = (GetBucketStateMessage)deserialize("GetBucketStateMessage", DocumentProtocol.MESSAGE_GETBUCKETSTATE, lang); - assertEquals(16, msg.getBucketId().getUsedBits()); - assertEquals(4611686018427388570l, msg.getBucketId().getId()); - } - } - } - - public class testCreateVisitorMessage implements RunnableTest { - - @Override - public void run() { - CreateVisitorMessage msg = new CreateVisitorMessage("SomeLibrary", "myvisitor", "newyork", "london"); - msg.setDocumentSelection("true and false or true"); - msg.getParameters().put("myvar", Utf8.toBytes("somevalue")); - msg.getParameters().put("anothervar", Utf8.toBytes("34")); - msg.getBuckets().add(new BucketId(16, 1234)); - msg.setVisitRemoves(true); - msg.setFieldSet("foo bar"); - msg.setVisitorOrdering(OrderingSpecification.DESCENDING); - msg.setMaxBucketsPerVisitor(2); - assertEquals(BASE_MESSAGE_LENGTH + 178, serialize("CreateVisitorMessage", msg)); - - for (Language lang : LANGUAGES) { - msg = (CreateVisitorMessage)deserialize("CreateVisitorMessage", DocumentProtocol.MESSAGE_CREATEVISITOR, lang); - assertEquals("SomeLibrary", msg.getLibraryName()); - assertEquals("myvisitor", msg.getInstanceId()); - assertEquals("newyork", msg.getControlDestination()); - assertEquals("london", msg.getDataDestination()); - assertEquals("true and false or true", msg.getDocumentSelection()); - assertEquals(8, msg.getMaxPendingReplyCount()); - assertEquals(true, msg.getVisitRemoves()); - assertEquals("foo bar", msg.getFieldSet()); - assertEquals(false, msg.getVisitInconsistentBuckets()); - assertEquals(1, msg.getBuckets().size()); - assertEquals(new BucketId(16, 1234), msg.getBuckets().iterator().next()); - assertEquals("somevalue", Utf8.toString(msg.getParameters().get("myvar"))); - assertEquals("34", Utf8.toString(msg.getParameters().get("anothervar"))); - assertEquals(OrderingSpecification.DESCENDING, msg.getVisitorOrdering()); - assertEquals(2, msg.getMaxBucketsPerVisitor()); - } - - msg.getBuckets().clear(); - - assertEquals("CreateVisitorMessage(" + - "No buckets, " + - "selection 'true and false or true', " + - "bucket space 'default', " + - "library SomeLibrary, including removes, " + - "get fields: foo bar" + - ")", - msg.toString()); - - msg.getBuckets().add(new BucketId(16, 1234)); - - assertEquals("CreateVisitorMessage(" + - "Bucket BucketId(0x40000000000004d2), " + - "selection 'true and false or true', " + - "bucket space 'default', " + - "library SomeLibrary, including removes, " + - "get fields: foo bar" + - ")", - msg.toString()); - - msg.getBuckets().add(new BucketId(16, 1235)); - msg.getBuckets().add(new BucketId(16, 1236)); - msg.getBuckets().add(new BucketId(16, 1237)); - msg.getBuckets().add(new BucketId(16, 1238)); - msg.setFromTimestamp(10001); - msg.setToTimestamp(20002); - msg.setVisitInconsistentBuckets(true); - assertEquals("CreateVisitorMessage(" + - "5 buckets: BucketId(0x40000000000004d2) BucketId(0x40000000000004d3) BucketId(0x40000000000004d4) ..., " + - "time 10001-20002, " + - "selection 'true and false or true', " + - "bucket space 'default', " + - "library SomeLibrary, including removes, " + - "get fields: foo bar, " + - "visit inconsistent buckets" + - ")", - msg.toString()); - } - } - - public class testCreateVisitorReply implements RunnableTest { - - @Override - public void run() { - CreateVisitorReply reply = new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR); - reply.setLastBucket(new BucketId(16, 123)); - reply.getVisitorStatistics().setBucketsVisited(3); - reply.getVisitorStatistics().setDocumentsVisited(1000); - reply.getVisitorStatistics().setBytesVisited(1024000); - reply.getVisitorStatistics().setDocumentsReturned(123); - reply.getVisitorStatistics().setBytesReturned(512000); - reply.getVisitorStatistics().setSecondPassDocumentsReturned(456); - reply.getVisitorStatistics().setSecondPassBytesReturned(789100); - - assertEquals(65, serialize("CreateVisitorReply", reply)); - - for (Language lang : LANGUAGES) { - reply = (CreateVisitorReply)deserialize("CreateVisitorReply", DocumentProtocol.REPLY_CREATEVISITOR, lang); - assertNotNull(reply); - assertEquals(new BucketId(16, 123), reply.getLastBucket()); - assertEquals(3, reply.getVisitorStatistics().getBucketsVisited()); - assertEquals(1000, reply.getVisitorStatistics().getDocumentsVisited()); - assertEquals(1024000, reply.getVisitorStatistics().getBytesVisited()); - assertEquals(123, reply.getVisitorStatistics().getDocumentsReturned()); - assertEquals(512000, reply.getVisitorStatistics().getBytesReturned()); - assertEquals(456, reply.getVisitorStatistics().getSecondPassDocumentsReturned()); - assertEquals(789100, reply.getVisitorStatistics().getSecondPassBytesReturned()); - } - } - } - - public class testDestroyVisitorReply implements RunnableTest { - - @Override - public void run() { - testVisitorReply("DestroyVisitorReply", DocumentProtocol.REPLY_DESTROYVISITOR); - } - } - - public class testDocumentIgnoredReply implements RunnableTest { - - @Override - public void run() { - DocumentIgnoredReply reply = new DocumentIgnoredReply(); - assertEquals(BASE_MESSAGE_LENGTH, serialize("DocumentIgnoredReply", reply)); - - for (Language lang : LANGUAGES) { - reply = (DocumentIgnoredReply)deserialize("DocumentIgnoredReply", DocumentProtocol.REPLY_DOCUMENTIGNORED, lang); - } - } - } - - public class testDocumentListReply implements RunnableTest { - - @Override - public void run() { - testVisitorReply("DocumentListReply", DocumentProtocol.REPLY_DOCUMENTLIST); - } - } - - public class testDocumentSummaryReply implements RunnableTest { - - @Override - public void run() { - testVisitorReply("DocumentSummaryReply", DocumentProtocol.REPLY_DOCUMENTSUMMARY); - } - } - - public class testEmptyBucketsReply implements RunnableTest { - - @Override - public void run() { - testVisitorReply("EmptyBucketsReply", DocumentProtocol.REPLY_EMPTYBUCKETS); - } - } - - public class testDestroyVisitorMessage implements RunnableTest { - - @Override - public void run() { - DestroyVisitorMessage msg = new DestroyVisitorMessage("myvisitor"); - assertEquals(BASE_MESSAGE_LENGTH + 17, serialize("DestroyVisitorMessage", msg)); - - for (Language lang : LANGUAGES) { - msg = (DestroyVisitorMessage)deserialize("DestroyVisitorMessage", DocumentProtocol.MESSAGE_DESTROYVISITOR, lang); - assertEquals("myvisitor", msg.getInstanceId()); - } - } - } - - public class testDocumentListMessage implements RunnableTest { - - @Override - public void run() { - DocumentListMessage msg = (DocumentListMessage)deserialize("DocumentListMessage", DocumentProtocol.MESSAGE_DOCUMENTLIST, Language.CPP); - assertEquals("userdoc:scheme:1234:", msg.getDocuments().get(0).getDocument().getId().toString()); - assertEquals(1234, msg.getDocuments().get(0).getTimestamp()); - assertFalse(msg.getDocuments().get(0).isRemoveEntry()); - - assertEquals(BASE_MESSAGE_LENGTH + 63, serialize("DocumentListMessage", msg)); - msg = (DocumentListMessage)deserialize("DocumentListMessage", DocumentProtocol.MESSAGE_DOCUMENTLIST, Language.JAVA); - assertEquals("userdoc:scheme:1234:", msg.getDocuments().get(0).getDocument().getId().toString()); - assertEquals(1234, msg.getDocuments().get(0).getTimestamp()); - assertFalse(msg.getDocuments().get(0).isRemoveEntry()); - - } - } - - public class testEmptyBucketsMessage implements RunnableTest { - - @Override - public void run() { - List<BucketId> bids = new ArrayList<>(); - for (int i = 0; i < 13; ++i) { - bids.add(new BucketId(16, i)); - } - - EmptyBucketsMessage ebm = new EmptyBucketsMessage(bids); - assertEquals(BASE_MESSAGE_LENGTH + 112, serialize("EmptyBucketsMessage", ebm)); - for (Language lang : LANGUAGES) { - ebm = (EmptyBucketsMessage)deserialize("EmptyBucketsMessage", DocumentProtocol.MESSAGE_EMPTYBUCKETS, lang); - for (int i = 0; i < 13; ++i) { - assertEquals(new BucketId(16, i), ebm.getBucketIds().get(i)); - } - } - } - } - - public class testDocumentSummaryMessage implements RunnableTest { - - @Override - public void run() { - Routable routable = deserialize("DocumentSummaryMessage-1", DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, Language.CPP); - assertTrue(routable instanceof DocumentSummaryMessage); - - DocumentSummaryMessage msg = (DocumentSummaryMessage) routable; - assertEquals(0, msg.getResult().getSummaryCount()); - - routable = deserialize("DocumentSummaryMessage-2", DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, Language.CPP); - assertTrue(routable instanceof DocumentSummaryMessage); - - msg = (DocumentSummaryMessage) routable; - assertEquals(2, msg.getResult().getSummaryCount()); - com.yahoo.vdslib.DocumentSummary.Summary s = msg.getResult().getSummary(0); - assertEquals("doc1", s.getDocId()); - byte[] b = s.getSummary(); - assertEquals(8, b.length); - byte[] c = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '1'}; - for (int i = 0; i < b.length; i++) { - assertEquals(c[i], b[i]); - } - - s = msg.getResult().getSummary(1); - assertEquals("aoc17", s.getDocId()); - b = s.getSummary(); - assertEquals(9, b.length); - byte[] d = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5'}; - for (int i = 0; i < b.length; i++) { - assertEquals(d[i], b[i]); - } - routable = deserialize("DocumentSummaryMessage-3", DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, Language.CPP); - assertTrue(routable instanceof DocumentSummaryMessage); - - msg = (DocumentSummaryMessage) routable; - assertEquals(2, msg.getResult().getSummaryCount()); - - s = msg.getResult().getSummary(0); - assertEquals("aoc17", s.getDocId()); - b = s.getSummary(); - assertEquals(9, b.length); - byte[] e = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5'}; - for (int i = 0; i < b.length; i++) { - assertEquals(e[i], b[i]); - } - - s = msg.getResult().getSummary(1); - assertEquals("doc1", s.getDocId()); - b = s.getSummary(); - assertEquals(8, b.length); - byte[] f = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '1'}; - for (int i = 0; i < b.length; i++) { - assertEquals(f[i], b[i]); - } - } - } - - - public class testGetDocumentMessage implements RunnableTest { - - @Override - public void run() { - GetDocumentMessage msg = new GetDocumentMessage(new DocumentId("doc:scheme:"), "foo bar"); - assertEquals(BASE_MESSAGE_LENGTH + 27, serialize("GetDocumentMessage", msg)); - - for (Language lang : LANGUAGES) { - msg = (GetDocumentMessage)deserialize("GetDocumentMessage", DocumentProtocol.MESSAGE_GETDOCUMENT, lang); - assertEquals("doc:scheme:", msg.getDocumentId().toString()); - assertEquals("foo bar", msg.getFieldSet()); - } - } - } - - - public class testRemoveDocumentMessage implements RunnableTest { - - @Override - public void run() { - final RemoveDocumentMessage msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:")); - msg.setCondition(new TestAndSetCondition(CONDITION_STRING)); - - assertEquals(BASE_MESSAGE_LENGTH + 16 + serializedLength(msg.getCondition().getSelection()), serialize("RemoveDocumentMessage", msg)); - - for (Language lang : LANGUAGES) { - final RemoveDocumentMessage deserializedMsg = (RemoveDocumentMessage)deserialize("RemoveDocumentMessage", DocumentProtocol.MESSAGE_REMOVEDOCUMENT, lang); - assertEquals(deserializedMsg.getDocumentId().toString(), msg.getDocumentId().toString()); - } - } - } - - public class testMapVisitorMessage implements RunnableTest { - - @Override - public void run() { - MapVisitorMessage msg = (MapVisitorMessage)deserialize("MapVisitorMessage", DocumentProtocol.MESSAGE_MAPVISITOR, Language.CPP); - assertEquals("3", msg.getData().get("foo")); - assertEquals("5", msg.getData().get("bar")); - - assertEquals(BASE_MESSAGE_LENGTH + 32, serialize("MapVisitorMessage", msg)); - - msg = (MapVisitorMessage)deserialize("MapVisitorMessage", DocumentProtocol.MESSAGE_MAPVISITOR, Language.JAVA); - assertEquals("3", msg.getData().get("foo")); - assertEquals("5", msg.getData().get("bar")); - } - } - - - public class testVisitorInfoMessage implements RunnableTest { - - @Override - public void run() { - VisitorInfoMessage msg = new VisitorInfoMessage(); - msg.getFinishedBuckets().add(new BucketId(16, 1)); - msg.getFinishedBuckets().add(new BucketId(16, 2)); - msg.getFinishedBuckets().add(new BucketId(16, 4)); - msg.setErrorMessage("error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6"); - assertEquals(BASE_MESSAGE_LENGTH + 67, serialize("VisitorInfoMessage", msg)); - - for (Language lang : LANGUAGES) { - msg = (VisitorInfoMessage)deserialize("VisitorInfoMessage", DocumentProtocol.MESSAGE_VISITORINFO, lang); - assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 1))); - assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 2))); - assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 4))); - assertEquals("error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6", msg.getErrorMessage()); - } - } - } - - public class testSearchResultMessage implements RunnableTest { - - @Override - public void run() throws Exception { - Routable routable = deserialize("SearchResultMessage-1", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP); - assertTrue(routable instanceof SearchResultMessage); - - SearchResultMessage msg = (SearchResultMessage)routable; - assertEquals(0, msg.getResult().getHitCount()); - - routable = deserialize("SearchResultMessage-2", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP); - assertTrue(routable instanceof SearchResultMessage); - - msg = (SearchResultMessage)routable; - assertEquals(2, msg.getResult().getHitCount()); - com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0); - assertEquals(89.0, h.getRank(), 1E-6); - assertEquals("doc1", h.getDocId()); - h = msg.getResult().getHit(1); - assertEquals(109.0, h.getRank(), 1E-6); - assertEquals("doc17", h.getDocId()); - - routable = deserialize("SearchResultMessage-3", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP); - assertTrue(routable instanceof SearchResultMessage); - - msg = (SearchResultMessage)routable; - assertEquals(2, msg.getResult().getHitCount()); - h = msg.getResult().getHit(0); - assertEquals(109.0, h.getRank(), 1E-6); - assertEquals("doc17", h.getDocId()); - h = msg.getResult().getHit(1); - assertEquals(89.0, h.getRank(), 1E-6); - assertEquals("doc1", h.getDocId()); - - routable = deserialize("SearchResultMessage-4", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP); - assertTrue(routable instanceof SearchResultMessage); - - msg = (SearchResultMessage)routable; - assertEquals(3, msg.getResult().getHitCount()); - h = msg.getResult().getHit(0); - assertTrue(h instanceof SearchResult.HitWithSortBlob); - assertEquals(89.0, h.getRank(), 1E-6); - assertEquals("doc1", h.getDocId()); - byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); - assertEquals(9, b.length); - byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' }; - for (int i = 0; i < b.length; i++) { - assertEquals(e[i], b[i]); - } - h = msg.getResult().getHit(1); - assertTrue(h instanceof SearchResult.HitWithSortBlob); - assertEquals(109.0, h.getRank(), 1E-6); - assertEquals("doc17", h.getDocId()); - b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); - assertEquals(9, b.length); - byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' }; - for (int i = 0; i < b.length; i++) { - assertEquals(d[i], b[i]); - } - h = msg.getResult().getHit(2); - assertTrue(h instanceof SearchResult.HitWithSortBlob); - assertEquals(90.0, h.getRank(), 1E-6); - assertEquals("doc18", h.getDocId()); - b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); - assertEquals(9, b.length); - byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' }; - for (int i = 0; i < b.length; i++) { - assertEquals(c[i], b[i]); - } - } - } - - private static String CONDITION_STRING = "There's just one condition"; - - public class testPutDocumentMessage implements RunnableTest { - - @Override - public void run() { - PutDocumentMessage msg = new PutDocumentMessage(new DocumentPut(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:"))); - - msg.setTimestamp(666); - msg.setCondition(new TestAndSetCondition(CONDITION_STRING)); - - assertEquals(BASE_MESSAGE_LENGTH + 41 + serializedLength(msg.getCondition().getSelection()), serialize("PutDocumentMessage", msg)); - - for (Language lang : LANGUAGES) { - final PutDocumentMessage deserializedMsg = (PutDocumentMessage)deserialize("PutDocumentMessage", DocumentProtocol.MESSAGE_PUTDOCUMENT, lang); - assertEquals(msg.getDocumentPut().getDocument().getDataType().getName(), deserializedMsg.getDocumentPut().getDocument().getDataType().getName()); - assertEquals(msg.getDocumentPut().getDocument().getId().toString(), deserializedMsg.getDocumentPut().getDocument().getId().toString()); - assertEquals(msg.getTimestamp(), deserializedMsg.getTimestamp()); - assertEquals(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection()); - } - } - } - - public class testPutDocumentReply implements RunnableTest { - - @Override - public void run() { - WriteDocumentReply reply = new WriteDocumentReply(DocumentProtocol.REPLY_PUTDOCUMENT); - reply.setHighestModificationTimestamp(30); - - assertEquals(13, serialize("PutDocumentReply", reply)); - - for (Language lang : LANGUAGES) { - WriteDocumentReply obj = (WriteDocumentReply)deserialize("PutDocumentReply", DocumentProtocol.REPLY_PUTDOCUMENT, lang); - assertNotNull(obj); - assertEquals(30, obj.getHighestModificationTimestamp()); - } - } - } - - public class testUpdateDocumentMessage implements RunnableTest { - - @Override - public void run() { - DocumentType docType = protocol.getDocumentTypeManager().getDocumentType("testdoc"); - DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("doc:scheme:")); - update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0")); - - final UpdateDocumentMessage msg = new UpdateDocumentMessage(update); - msg.setNewTimestamp(777); - msg.setOldTimestamp(666); - msg.setCondition(new TestAndSetCondition(CONDITION_STRING)); - - assertEquals(BASE_MESSAGE_LENGTH + 89 + serializedLength(msg.getCondition().getSelection()), serialize("UpdateDocumentMessage", msg)); - - for (Language lang : LANGUAGES) { - final UpdateDocumentMessage deserializedMsg = (UpdateDocumentMessage) deserialize("UpdateDocumentMessage", DocumentProtocol.MESSAGE_UPDATEDOCUMENT, lang); - assertEquals(msg.getDocumentUpdate(), deserializedMsg.getDocumentUpdate()); - assertEquals(msg.getNewTimestamp(), deserializedMsg.getNewTimestamp()); - assertEquals(msg.getOldTimestamp(), deserializedMsg.getOldTimestamp()); - assertEquals(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection()); - } - } - } - - public class testUpdateDocumentReply implements RunnableTest { - - @Override - public void run() { - UpdateDocumentReply reply = new UpdateDocumentReply(); - reply.setHighestModificationTimestamp(30); - reply.setWasFound(false); - - assertEquals(14, serialize("UpdateDocumentReply", reply)); - - for (Language lang : LANGUAGES) { - UpdateDocumentReply obj = (UpdateDocumentReply)deserialize("UpdateDocumentReply", DocumentProtocol.REPLY_UPDATEDOCUMENT, lang); - assertNotNull(obj); - assertEquals(30, reply.getHighestModificationTimestamp()); - assertEquals(false, obj.wasFound()); - } - } - } - - public class testVisitorInfoReply implements RunnableTest { - - @Override - public void run() { - testVisitorReply("VisitorInfoReply", DocumentProtocol.REPLY_VISITORINFO); - } - } - - public class testWrongDistributionReply implements RunnableTest { - - @Override - public void run() { - WrongDistributionReply reply = new WrongDistributionReply("distributor:3 storage:2"); - assertEquals(32, serialize("WrongDistributionReply", reply)); - - for (Language lang : LANGUAGES) { - reply = (WrongDistributionReply)deserialize("WrongDistributionReply", DocumentProtocol.REPLY_WRONGDISTRIBUTION, lang); - assertEquals("distributor:3 storage:2", reply.getSystemState()); - } - } - } - - public class testRemoveDocumentReply implements RunnableTest { - - @Override - public void run() { - RemoveDocumentReply reply = new RemoveDocumentReply(); - reply.setHighestModificationTimestamp(30); - reply.setWasFound(false); - - assertEquals(14, serialize("RemoveDocumentReply", reply)); - - for (Language lang : LANGUAGES) { - RemoveDocumentReply obj = (RemoveDocumentReply)deserialize("RemoveDocumentReply", DocumentProtocol.REPLY_REMOVEDOCUMENT, lang); - assertNotNull(obj); - assertEquals(30, obj.getHighestModificationTimestamp()); - assertEquals(false, obj.wasFound()); - } - } - } - - public class testRemoveLocationReply implements RunnableTest { - - @Override - public void run() { - testDocumentReply("RemoveLocationReply", DocumentProtocol.REPLY_REMOVELOCATION); - } - } - - public class testSearchResultReply implements RunnableTest { - - @Override - public void run() { - testVisitorReply("SearchResultReply", DocumentProtocol.REPLY_SEARCHRESULT); - } - } - - public class testStatBucketReply implements RunnableTest { - - @Override - public void run() { - StatBucketReply msg = new StatBucketReply(); - msg.setResults("These are the votes of the Norwegian jury"); - - assertEquals(50, serialize("StatBucketReply", msg)); - - for (Language lang : LANGUAGES) { - msg = (StatBucketReply)deserialize("StatBucketReply", DocumentProtocol.REPLY_STATBUCKET, lang); - assertEquals("These are the votes of the Norwegian jury", msg.getResults()); - } - } - } - - public class testQueryResultReply implements RunnableTest { - - @Override - public void run() { - testVisitorReply("QueryResultReply", DocumentProtocol.REPLY_QUERYRESULT); - } - } - - public class testQueryResultMessage implements RunnableTest { - - @Override - public void run() throws Exception { - Routable routable = deserialize("QueryResultMessage-1", DocumentProtocol.MESSAGE_QUERYRESULT, Language.CPP); - assertTrue(routable instanceof QueryResultMessage); - - QueryResultMessage msg = (QueryResultMessage)routable; - assertEquals(0, msg.getResult().getHitCount()); - - routable = deserialize("QueryResultMessage-2", DocumentProtocol.MESSAGE_QUERYRESULT, Language.CPP); - assertTrue(routable instanceof QueryResultMessage); - - msg = (QueryResultMessage)routable; - assertEquals(2, msg.getResult().getHitCount()); - com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0); - assertEquals(89.0, h.getRank(), 1E-6); - assertEquals("doc1", h.getDocId()); - h = msg.getResult().getHit(1); - assertEquals(109.0, h.getRank(), 1E-6); - assertEquals("doc17", h.getDocId()); - - routable = deserialize("QueryResultMessage-3", DocumentProtocol.MESSAGE_QUERYRESULT, Language.CPP); - assertTrue(routable instanceof QueryResultMessage); - - msg = (QueryResultMessage)routable; - assertEquals(2, msg.getResult().getHitCount()); - h = msg.getResult().getHit(0); - assertEquals(109.0, h.getRank(), 1E-6); - assertEquals("doc17", h.getDocId()); - h = msg.getResult().getHit(1); - assertEquals(89.0, h.getRank(), 1E-6); - assertEquals("doc1", h.getDocId()); - - routable = deserialize("QueryResultMessage-4", DocumentProtocol.MESSAGE_QUERYRESULT, Language.CPP); - assertTrue(routable instanceof QueryResultMessage); - - msg = (QueryResultMessage)routable; - assertEquals(3, msg.getResult().getHitCount()); - h = msg.getResult().getHit(0); - assertTrue(h instanceof SearchResult.HitWithSortBlob); - assertEquals(89.0, h.getRank(), 1E-6); - assertEquals("doc1", h.getDocId()); - byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); - assertEquals(9, b.length); - byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' }; - for (int i = 0; i < b.length; i++) { - assertEquals(e[i], b[i]); - } - h = msg.getResult().getHit(1); - assertTrue(h instanceof SearchResult.HitWithSortBlob); - assertEquals(109.0, h.getRank(), 1E-6); - assertEquals("doc17", h.getDocId()); - b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); - assertEquals(9, b.length); - byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' }; - for (int i = 0; i < b.length; i++) { - assertEquals(d[i], b[i]); - } - h = msg.getResult().getHit(2); - assertTrue(h instanceof SearchResult.HitWithSortBlob); - assertEquals(90.0, h.getRank(), 1E-6); - assertEquals("doc18", h.getDocId()); - b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); - assertEquals(9, b.length); - byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' }; - for (int i = 0; i < b.length; i++) { - assertEquals(c[i], b[i]); - } - } - } - - public class testGetBucketListReply implements RunnableTest { - - public void run() { - GetBucketListReply reply = new GetBucketListReply(); - reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(16, 123), "foo")); - reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(17, 1123), "bar")); - reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(18, 11123), "zoink")); - - assertEquals(56, serialize("GetBucketListReply", reply)); - - for (Language lang : LANGUAGES) { - reply = (GetBucketListReply)deserialize("GetBucketListReply", DocumentProtocol.REPLY_GETBUCKETLIST, lang); - assertEquals(reply.getBuckets().get(0), new GetBucketListReply.BucketInfo(new BucketId(16, 123), "foo")); - assertEquals(reply.getBuckets().get(1), new GetBucketListReply.BucketInfo(new BucketId(17, 1123), "bar")); - assertEquals(reply.getBuckets().get(2), new GetBucketListReply.BucketInfo(new BucketId(18, 11123), "zoink")); - } - } - } - - public class testGetBucketStateReply implements RunnableTest { - - public void run() { - GlobalId foo = new GlobalId(IdString.createIdString("doc:scheme:foo")); - GlobalId bar = new GlobalId(IdString.createIdString("doc:scheme:bar")); - - GetBucketStateReply reply = new GetBucketStateReply(); - List<DocumentState> state = new ArrayList<>(2); - state.add(new DocumentState(foo, 777, false)); - state.add(new DocumentState(bar, 888, true)); - reply.setBucketState(state); - assertEquals(53, serialize("GetBucketStateReply", reply)); - - for (Language lang : LANGUAGES) { - reply = (GetBucketStateReply)deserialize("GetBucketStateReply", DocumentProtocol.REPLY_GETBUCKETSTATE, lang); - assertEquals(777, reply.getBucketState().get(0).getTimestamp()); - assertEquals(foo, reply.getBucketState().get(0).getGid()); - assertEquals(false, reply.getBucketState().get(0).isRemoveEntry()); - assertEquals(888, reply.getBucketState().get(1).getTimestamp()); - assertEquals(bar, reply.getBucketState().get(1).getGid()); - assertEquals(true, reply.getBucketState().get(1).isRemoveEntry()); - } - } - } - - public class testGetDocumentReply implements RunnableTest { - - public void run() { - GetDocumentReply reply = new GetDocumentReply(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:")); - assertEquals(43, serialize("GetDocumentReply", reply)); - - for (Language lang : LANGUAGES) { - reply = (GetDocumentReply)deserialize("GetDocumentReply", DocumentProtocol.REPLY_GETDOCUMENT, lang); - assertEquals("testdoc", reply.getDocument().getDataType().getName()); - assertEquals("doc:scheme:", reply.getDocument().getId().toString()); - } - } - } - - public class testMapVisitorReply implements RunnableTest { - - public void run() { - testVisitorReply("MapVisitorReply", DocumentProtocol.REPLY_MAPVISITOR); - } - } - - protected void testDocumentReply(String filename, int type) { - DocumentReply reply = new DocumentReply(type); - assertEquals(5, serialize(filename, reply)); - - for (Language lang : LANGUAGES) { - reply = (DocumentReply)deserialize(filename, type, lang); - assertNotNull(reply); - } - } - - protected void testVisitorReply(String filename, int type) { - VisitorReply reply = new VisitorReply(type); - assertEquals(5, serialize(filename, reply)); - - for (Language lang : LANGUAGES) { - reply = (VisitorReply)deserialize(filename, type, lang); - assertNotNull(reply); - } - } - - static int serializedLength(String str) { - return 4 + str.length(); - } -} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java index 4d914c03a2b..ac09acf646c 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java @@ -3,22 +3,111 @@ package com.yahoo.documentapi.messagebus.protocol.test; import com.yahoo.component.Version; import com.yahoo.document.BucketId; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.GlobalId; +import com.yahoo.document.TestAndSetCondition; +import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate; +import com.yahoo.document.idstring.IdString; import com.yahoo.document.select.OrderingSpecification; import com.yahoo.documentapi.messagebus.protocol.CreateVisitorMessage; +import com.yahoo.documentapi.messagebus.protocol.CreateVisitorReply; +import com.yahoo.documentapi.messagebus.protocol.DestroyVisitorMessage; +import com.yahoo.documentapi.messagebus.protocol.DocumentIgnoredReply; +import com.yahoo.documentapi.messagebus.protocol.DocumentListMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.messagebus.protocol.DocumentReply; +import com.yahoo.documentapi.messagebus.protocol.DocumentState; +import com.yahoo.documentapi.messagebus.protocol.DocumentSummaryMessage; +import com.yahoo.documentapi.messagebus.protocol.EmptyBucketsMessage; import com.yahoo.documentapi.messagebus.protocol.GetBucketListMessage; +import com.yahoo.documentapi.messagebus.protocol.GetBucketListReply; +import com.yahoo.documentapi.messagebus.protocol.GetBucketStateMessage; +import com.yahoo.documentapi.messagebus.protocol.GetBucketStateReply; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; +import com.yahoo.documentapi.messagebus.protocol.MapVisitorMessage; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.QueryResultMessage; +import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentReply; +import com.yahoo.documentapi.messagebus.protocol.RemoveLocationMessage; +import com.yahoo.documentapi.messagebus.protocol.SearchResultMessage; import com.yahoo.documentapi.messagebus.protocol.StatBucketMessage; +import com.yahoo.documentapi.messagebus.protocol.StatBucketReply; +import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentReply; +import com.yahoo.documentapi.messagebus.protocol.VisitorInfoMessage; +import com.yahoo.documentapi.messagebus.protocol.VisitorReply; +import com.yahoo.documentapi.messagebus.protocol.WriteDocumentReply; +import com.yahoo.documentapi.messagebus.protocol.WrongDistributionReply; +import com.yahoo.messagebus.Routable; import com.yahoo.text.Utf8; +import com.yahoo.vdslib.SearchResult; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; -public class Messages60TestCase extends Messages52TestCase { +/** + * @author Simon Thoresen Hult + * @author Vegard Sjonfjell + */ + +public class Messages60TestCase extends MessagesTestBase { + @Override + protected void registerTests(Map<Integer, RunnableTest> out) { + // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support + // version 6. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now. + out.put(DocumentProtocol.MESSAGE_CREATEVISITOR, new testCreateVisitorMessage()); + out.put(DocumentProtocol.MESSAGE_DESTROYVISITOR, new testDestroyVisitorMessage()); + out.put(DocumentProtocol.MESSAGE_DOCUMENTLIST, new testDocumentListMessage()); + out.put(DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, new testDocumentSummaryMessage()); + out.put(DocumentProtocol.MESSAGE_EMPTYBUCKETS, new testEmptyBucketsMessage()); + out.put(DocumentProtocol.MESSAGE_GETBUCKETLIST, new testGetBucketListMessage()); + out.put(DocumentProtocol.MESSAGE_GETBUCKETSTATE, new testGetBucketStateMessage()); + out.put(DocumentProtocol.MESSAGE_GETDOCUMENT, new testGetDocumentMessage()); + out.put(DocumentProtocol.MESSAGE_MAPVISITOR, new testMapVisitorMessage()); + out.put(DocumentProtocol.MESSAGE_PUTDOCUMENT, new testPutDocumentMessage()); + out.put(DocumentProtocol.MESSAGE_QUERYRESULT, new testQueryResultMessage()); + out.put(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, new testRemoveDocumentMessage()); + out.put(DocumentProtocol.MESSAGE_REMOVELOCATION, new testRemoveLocationMessage()); + out.put(DocumentProtocol.MESSAGE_SEARCHRESULT, new testSearchResultMessage()); + out.put(DocumentProtocol.MESSAGE_STATBUCKET, new testStatBucketMessage()); + out.put(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, new testUpdateDocumentMessage()); + out.put(DocumentProtocol.MESSAGE_VISITORINFO, new testVisitorInfoMessage()); + out.put(DocumentProtocol.REPLY_CREATEVISITOR, new testCreateVisitorReply()); + out.put(DocumentProtocol.REPLY_DESTROYVISITOR, new testDestroyVisitorReply()); + out.put(DocumentProtocol.REPLY_DOCUMENTIGNORED, new testDocumentIgnoredReply()); + out.put(DocumentProtocol.REPLY_DOCUMENTLIST, new testDocumentListReply()); + out.put(DocumentProtocol.REPLY_DOCUMENTSUMMARY, new testDocumentSummaryReply()); + out.put(DocumentProtocol.REPLY_EMPTYBUCKETS, new testEmptyBucketsReply()); + out.put(DocumentProtocol.REPLY_GETBUCKETLIST, new testGetBucketListReply()); + out.put(DocumentProtocol.REPLY_GETBUCKETSTATE, new testGetBucketStateReply()); + out.put(DocumentProtocol.REPLY_GETDOCUMENT, new testGetDocumentReply()); + out.put(DocumentProtocol.REPLY_MAPVISITOR, new testMapVisitorReply()); + out.put(DocumentProtocol.REPLY_PUTDOCUMENT, new testPutDocumentReply()); + out.put(DocumentProtocol.REPLY_QUERYRESULT, new testQueryResultReply()); + out.put(DocumentProtocol.REPLY_REMOVEDOCUMENT, new testRemoveDocumentReply()); + out.put(DocumentProtocol.REPLY_REMOVELOCATION, new testRemoveLocationReply()); + out.put(DocumentProtocol.REPLY_SEARCHRESULT, new testSearchResultReply()); + out.put(DocumentProtocol.REPLY_STATBUCKET, new testStatBucketReply()); + out.put(DocumentProtocol.REPLY_UPDATEDOCUMENT, new testUpdateDocumentReply()); + out.put(DocumentProtocol.REPLY_VISITORINFO, new testVisitorInfoReply()); + out.put(DocumentProtocol.REPLY_WRONGDISTRIBUTION, new testWrongDistributionReply()); + } @Override protected Version version() { - return new Version(6, 221); + return new Version(6, 221, 0); } @Override @@ -26,25 +115,90 @@ public class Messages60TestCase extends Messages52TestCase { return true; } - @Override - protected void registerTests(Map<Integer, MessagesTestBase.RunnableTest> out) { - super.registerTests(out); + //////////////////////////////////////////////////////////////////////////////// + // + // Tests + // + //////////////////////////////////////////////////////////////////////////////// - // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support - // version 6.0. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now. + protected static int BASE_MESSAGE_LENGTH = 5; + + public class testRemoveLocationMessage implements RunnableTest { + + @Override + public void run() { + { + RemoveLocationMessage msg = new RemoveLocationMessage("id.group == \"mygroup\""); + assertEquals(BASE_MESSAGE_LENGTH + 29, serialize("RemoveLocationMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (RemoveLocationMessage)deserialize("RemoveLocationMessage", DocumentProtocol.MESSAGE_REMOVELOCATION, lang); + assertEquals("id.group == \"mygroup\"", msg.getDocumentSelection()); + } + } + } + } + + public class testGetBucketListMessage implements RunnableTest { + + private static final String BUCKET_SPACE = "beartato"; + + @Override + public void run() { + GetBucketListMessage msg = new GetBucketListMessage(new BucketId(16, 123)); + msg.setLoadType(loadTypes.getNameMap().get("foo")); + msg.setBucketSpace(BUCKET_SPACE); + assertEquals(BASE_MESSAGE_LENGTH + 12 + serializedLength(BUCKET_SPACE), serialize("GetBucketListMessage", msg)); - out.put(DocumentProtocol.MESSAGE_CREATEVISITOR, new Messages60TestCase.testCreateVisitorMessage()); - out.put(DocumentProtocol.MESSAGE_STATBUCKET, new Messages60TestCase.testStatBucketMessage()); - out.put(DocumentProtocol.MESSAGE_GETBUCKETLIST, new Messages60TestCase.testGetBucketListMessage()); + for (Language lang : LANGUAGES) { + msg = (GetBucketListMessage)deserialize("GetBucketListMessage", DocumentProtocol.MESSAGE_GETBUCKETLIST, lang); + assertEquals(new BucketId(16, 123), msg.getBucketId()); + assertEquals("foo", msg.getLoadType().getName()); + assertEquals(BUCKET_SPACE, msg.getBucketSpace()); + } + } + } + + public class testStatBucketMessage implements RunnableTest { + + private static final String BUCKET_SPACE = "andrei"; + + @Override + public void run() { + StatBucketMessage msg = new StatBucketMessage(new BucketId(16, 123), "id.user=123"); + msg.setLoadType(null); + msg.setBucketSpace(BUCKET_SPACE); + assertEquals(BASE_MESSAGE_LENGTH + 27 + serializedLength(BUCKET_SPACE), serialize("StatBucketMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (StatBucketMessage)deserialize("StatBucketMessage", DocumentProtocol.MESSAGE_STATBUCKET, lang); + assertEquals(new BucketId(16, 123), msg.getBucketId()); + assertEquals("id.user=123", msg.getDocumentSelection()); + assertEquals("default", msg.getLoadType().getName()); + assertEquals(BUCKET_SPACE, msg.getBucketSpace()); + } + } + } + + public class testGetBucketStateMessage implements RunnableTest { + + @Override + public void run() { + GetBucketStateMessage msg = new GetBucketStateMessage(new BucketId(16, 666)); + assertEquals(BASE_MESSAGE_LENGTH + 12, serialize("GetBucketStateMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (GetBucketStateMessage)deserialize("GetBucketStateMessage", DocumentProtocol.MESSAGE_GETBUCKETSTATE, lang); + assertEquals(16, msg.getBucketId().getUsedBits()); + assertEquals(4611686018427388570l, msg.getBucketId().getId()); + } + } } public class testCreateVisitorMessage implements RunnableTest { private static final String BUCKET_SPACE = "bjarne"; - // FIXME there is a large amount of code duplication across version tests, presumably - // to ensure that fields survive across version boundaries, but it makes code very bloated. - // TODO head towards a blissful Protobuf future instead... @Override public void run() { CreateVisitorMessage msg = new CreateVisitorMessage("SomeLibrary", "myvisitor", "newyork", "london"); @@ -78,48 +232,701 @@ public class Messages60TestCase extends Messages52TestCase { assertEquals(2, msg.getMaxBucketsPerVisitor()); assertEquals(BUCKET_SPACE, msg.getBucketSpace()); } + + msg.getBuckets().clear(); + + assertEquals("CreateVisitorMessage(" + + "No buckets, " + + "selection 'true and false or true', " + + "bucket space 'bjarne', " + + "library SomeLibrary, including removes, " + + "get fields: foo bar" + + ")", + msg.toString()); + + msg.getBuckets().add(new BucketId(16, 1234)); + + assertEquals("CreateVisitorMessage(" + + "Bucket BucketId(0x40000000000004d2), " + + "selection 'true and false or true', " + + "bucket space 'bjarne', " + + "library SomeLibrary, including removes, " + + "get fields: foo bar" + + ")", + msg.toString()); + + msg.getBuckets().add(new BucketId(16, 1235)); + msg.getBuckets().add(new BucketId(16, 1236)); + msg.getBuckets().add(new BucketId(16, 1237)); + msg.getBuckets().add(new BucketId(16, 1238)); + msg.setFromTimestamp(10001); + msg.setToTimestamp(20002); + msg.setVisitInconsistentBuckets(true); + assertEquals("CreateVisitorMessage(" + + "5 buckets: BucketId(0x40000000000004d2) BucketId(0x40000000000004d3) BucketId(0x40000000000004d4) ..., " + + "time 10001-20002, " + + "selection 'true and false or true', " + + "bucket space 'bjarne', " + + "library SomeLibrary, including removes, " + + "get fields: foo bar, " + + "visit inconsistent buckets" + + ")", + msg.toString()); } } - public class testStatBucketMessage implements RunnableTest { + public class testCreateVisitorReply implements RunnableTest { - private static final String BUCKET_SPACE = "andrei"; + @Override + public void run() { + CreateVisitorReply reply = new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR); + reply.setLastBucket(new BucketId(16, 123)); + reply.getVisitorStatistics().setBucketsVisited(3); + reply.getVisitorStatistics().setDocumentsVisited(1000); + reply.getVisitorStatistics().setBytesVisited(1024000); + reply.getVisitorStatistics().setDocumentsReturned(123); + reply.getVisitorStatistics().setBytesReturned(512000); + reply.getVisitorStatistics().setSecondPassDocumentsReturned(456); + reply.getVisitorStatistics().setSecondPassBytesReturned(789100); + + assertEquals(65, serialize("CreateVisitorReply", reply)); + + for (Language lang : LANGUAGES) { + reply = (CreateVisitorReply)deserialize("CreateVisitorReply", DocumentProtocol.REPLY_CREATEVISITOR, lang); + assertNotNull(reply); + assertEquals(new BucketId(16, 123), reply.getLastBucket()); + assertEquals(3, reply.getVisitorStatistics().getBucketsVisited()); + assertEquals(1000, reply.getVisitorStatistics().getDocumentsVisited()); + assertEquals(1024000, reply.getVisitorStatistics().getBytesVisited()); + assertEquals(123, reply.getVisitorStatistics().getDocumentsReturned()); + assertEquals(512000, reply.getVisitorStatistics().getBytesReturned()); + assertEquals(456, reply.getVisitorStatistics().getSecondPassDocumentsReturned()); + assertEquals(789100, reply.getVisitorStatistics().getSecondPassBytesReturned()); + } + } + } + + public class testDestroyVisitorReply implements RunnableTest { @Override public void run() { - StatBucketMessage msg = new StatBucketMessage(new BucketId(16, 123), "id.user=123"); - msg.setLoadType(null); - msg.setBucketSpace(BUCKET_SPACE); - assertEquals(BASE_MESSAGE_LENGTH + 27 + serializedLength(BUCKET_SPACE), serialize("StatBucketMessage", msg)); + testVisitorReply("DestroyVisitorReply", DocumentProtocol.REPLY_DESTROYVISITOR); + } + } + + public class testDocumentIgnoredReply implements RunnableTest { + + @Override + public void run() { + DocumentIgnoredReply reply = new DocumentIgnoredReply(); + assertEquals(BASE_MESSAGE_LENGTH, serialize("DocumentIgnoredReply", reply)); for (Language lang : LANGUAGES) { - msg = (StatBucketMessage)deserialize("StatBucketMessage", DocumentProtocol.MESSAGE_STATBUCKET, lang); - assertEquals(new BucketId(16, 123), msg.getBucketId()); - assertEquals("id.user=123", msg.getDocumentSelection()); - assertEquals("default", msg.getLoadType().getName()); - assertEquals(BUCKET_SPACE, msg.getBucketSpace()); + reply = (DocumentIgnoredReply)deserialize("DocumentIgnoredReply", DocumentProtocol.REPLY_DOCUMENTIGNORED, lang); } } } - public class testGetBucketListMessage implements RunnableTest { + public class testDocumentListReply implements RunnableTest { - private static final String BUCKET_SPACE = "beartato"; + @Override + public void run() { + testVisitorReply("DocumentListReply", DocumentProtocol.REPLY_DOCUMENTLIST); + } + } + + public class testDocumentSummaryReply implements RunnableTest { @Override public void run() { - GetBucketListMessage msg = new GetBucketListMessage(new BucketId(16, 123)); - msg.setLoadType(loadTypes.getNameMap().get("foo")); - msg.setBucketSpace(BUCKET_SPACE); - assertEquals(BASE_MESSAGE_LENGTH + 12 + serializedLength(BUCKET_SPACE), serialize("GetBucketListMessage", msg)); + testVisitorReply("DocumentSummaryReply", DocumentProtocol.REPLY_DOCUMENTSUMMARY); + } + } + + public class testEmptyBucketsReply implements RunnableTest { + + @Override + public void run() { + testVisitorReply("EmptyBucketsReply", DocumentProtocol.REPLY_EMPTYBUCKETS); + } + } + + public class testDestroyVisitorMessage implements RunnableTest { + + @Override + public void run() { + DestroyVisitorMessage msg = new DestroyVisitorMessage("myvisitor"); + assertEquals(BASE_MESSAGE_LENGTH + 17, serialize("DestroyVisitorMessage", msg)); for (Language lang : LANGUAGES) { - msg = (GetBucketListMessage)deserialize("GetBucketListMessage", DocumentProtocol.MESSAGE_GETBUCKETLIST, lang); - assertEquals(new BucketId(16, 123), msg.getBucketId()); - assertEquals("foo", msg.getLoadType().getName()); - assertEquals(BUCKET_SPACE, msg.getBucketSpace()); + msg = (DestroyVisitorMessage)deserialize("DestroyVisitorMessage", DocumentProtocol.MESSAGE_DESTROYVISITOR, lang); + assertEquals("myvisitor", msg.getInstanceId()); } } } + public class testDocumentListMessage implements RunnableTest { + + @Override + public void run() { + DocumentListMessage msg = (DocumentListMessage)deserialize("DocumentListMessage", DocumentProtocol.MESSAGE_DOCUMENTLIST, Language.CPP); + assertEquals("userdoc:scheme:1234:", msg.getDocuments().get(0).getDocument().getId().toString()); + assertEquals(1234, msg.getDocuments().get(0).getTimestamp()); + assertFalse(msg.getDocuments().get(0).isRemoveEntry()); + + assertEquals(BASE_MESSAGE_LENGTH + 63, serialize("DocumentListMessage", msg)); + msg = (DocumentListMessage)deserialize("DocumentListMessage", DocumentProtocol.MESSAGE_DOCUMENTLIST, Language.JAVA); + assertEquals("userdoc:scheme:1234:", msg.getDocuments().get(0).getDocument().getId().toString()); + assertEquals(1234, msg.getDocuments().get(0).getTimestamp()); + assertFalse(msg.getDocuments().get(0).isRemoveEntry()); + + } + } + + public class testEmptyBucketsMessage implements RunnableTest { + + @Override + public void run() { + List<BucketId> bids = new ArrayList<>(); + for (int i = 0; i < 13; ++i) { + bids.add(new BucketId(16, i)); + } + + EmptyBucketsMessage ebm = new EmptyBucketsMessage(bids); + assertEquals(BASE_MESSAGE_LENGTH + 112, serialize("EmptyBucketsMessage", ebm)); + for (Language lang : LANGUAGES) { + ebm = (EmptyBucketsMessage)deserialize("EmptyBucketsMessage", DocumentProtocol.MESSAGE_EMPTYBUCKETS, lang); + for (int i = 0; i < 13; ++i) { + assertEquals(new BucketId(16, i), ebm.getBucketIds().get(i)); + } + } + } + } + + public class testDocumentSummaryMessage implements RunnableTest { + + @Override + public void run() { + Routable routable = deserialize("DocumentSummaryMessage-1", DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, Language.CPP); + assertTrue(routable instanceof DocumentSummaryMessage); + + DocumentSummaryMessage msg = (DocumentSummaryMessage) routable; + assertEquals(0, msg.getResult().getSummaryCount()); + + routable = deserialize("DocumentSummaryMessage-2", DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, Language.CPP); + assertTrue(routable instanceof DocumentSummaryMessage); + + msg = (DocumentSummaryMessage) routable; + assertEquals(2, msg.getResult().getSummaryCount()); + com.yahoo.vdslib.DocumentSummary.Summary s = msg.getResult().getSummary(0); + assertEquals("doc1", s.getDocId()); + byte[] b = s.getSummary(); + assertEquals(8, b.length); + byte[] c = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '1'}; + for (int i = 0; i < b.length; i++) { + assertEquals(c[i], b[i]); + } + + s = msg.getResult().getSummary(1); + assertEquals("aoc17", s.getDocId()); + b = s.getSummary(); + assertEquals(9, b.length); + byte[] d = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5'}; + for (int i = 0; i < b.length; i++) { + assertEquals(d[i], b[i]); + } + routable = deserialize("DocumentSummaryMessage-3", DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, Language.CPP); + assertTrue(routable instanceof DocumentSummaryMessage); + + msg = (DocumentSummaryMessage) routable; + assertEquals(2, msg.getResult().getSummaryCount()); + + s = msg.getResult().getSummary(0); + assertEquals("aoc17", s.getDocId()); + b = s.getSummary(); + assertEquals(9, b.length); + byte[] e = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5'}; + for (int i = 0; i < b.length; i++) { + assertEquals(e[i], b[i]); + } + + s = msg.getResult().getSummary(1); + assertEquals("doc1", s.getDocId()); + b = s.getSummary(); + assertEquals(8, b.length); + byte[] f = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '1'}; + for (int i = 0; i < b.length; i++) { + assertEquals(f[i], b[i]); + } + } + } + + + public class testGetDocumentMessage implements RunnableTest { + + @Override + public void run() { + GetDocumentMessage msg = new GetDocumentMessage(new DocumentId("doc:scheme:"), "foo bar"); + assertEquals(BASE_MESSAGE_LENGTH + 27, serialize("GetDocumentMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (GetDocumentMessage)deserialize("GetDocumentMessage", DocumentProtocol.MESSAGE_GETDOCUMENT, lang); + assertEquals("doc:scheme:", msg.getDocumentId().toString()); + assertEquals("foo bar", msg.getFieldSet()); + } + } + } + + + public class testRemoveDocumentMessage implements RunnableTest { + + @Override + public void run() { + final RemoveDocumentMessage msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:")); + msg.setCondition(new TestAndSetCondition(CONDITION_STRING)); + + assertEquals(BASE_MESSAGE_LENGTH + 16 + serializedLength(msg.getCondition().getSelection()), serialize("RemoveDocumentMessage", msg)); + + for (Language lang : LANGUAGES) { + final RemoveDocumentMessage deserializedMsg = (RemoveDocumentMessage)deserialize("RemoveDocumentMessage", DocumentProtocol.MESSAGE_REMOVEDOCUMENT, lang); + assertEquals(deserializedMsg.getDocumentId().toString(), msg.getDocumentId().toString()); + } + } + } + + public class testMapVisitorMessage implements RunnableTest { + + @Override + public void run() { + MapVisitorMessage msg = (MapVisitorMessage)deserialize("MapVisitorMessage", DocumentProtocol.MESSAGE_MAPVISITOR, Language.CPP); + assertEquals("3", msg.getData().get("foo")); + assertEquals("5", msg.getData().get("bar")); + + assertEquals(BASE_MESSAGE_LENGTH + 32, serialize("MapVisitorMessage", msg)); + + msg = (MapVisitorMessage)deserialize("MapVisitorMessage", DocumentProtocol.MESSAGE_MAPVISITOR, Language.JAVA); + assertEquals("3", msg.getData().get("foo")); + assertEquals("5", msg.getData().get("bar")); + } + } + + + public class testVisitorInfoMessage implements RunnableTest { + + @Override + public void run() { + VisitorInfoMessage msg = new VisitorInfoMessage(); + msg.getFinishedBuckets().add(new BucketId(16, 1)); + msg.getFinishedBuckets().add(new BucketId(16, 2)); + msg.getFinishedBuckets().add(new BucketId(16, 4)); + msg.setErrorMessage("error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6"); + assertEquals(BASE_MESSAGE_LENGTH + 67, serialize("VisitorInfoMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (VisitorInfoMessage)deserialize("VisitorInfoMessage", DocumentProtocol.MESSAGE_VISITORINFO, lang); + assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 1))); + assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 2))); + assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 4))); + assertEquals("error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6", msg.getErrorMessage()); + } + } + } + + public class testSearchResultMessage implements RunnableTest { + + @Override + public void run() throws Exception { + Routable routable = deserialize("SearchResultMessage-1", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP); + assertTrue(routable instanceof SearchResultMessage); + + SearchResultMessage msg = (SearchResultMessage)routable; + assertEquals(0, msg.getResult().getHitCount()); + + routable = deserialize("SearchResultMessage-2", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP); + assertTrue(routable instanceof SearchResultMessage); + + msg = (SearchResultMessage)routable; + assertEquals(2, msg.getResult().getHitCount()); + com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0); + assertEquals(89.0, h.getRank(), 1E-6); + assertEquals("doc1", h.getDocId()); + h = msg.getResult().getHit(1); + assertEquals(109.0, h.getRank(), 1E-6); + assertEquals("doc17", h.getDocId()); + + routable = deserialize("SearchResultMessage-3", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP); + assertTrue(routable instanceof SearchResultMessage); + + msg = (SearchResultMessage)routable; + assertEquals(2, msg.getResult().getHitCount()); + h = msg.getResult().getHit(0); + assertEquals(109.0, h.getRank(), 1E-6); + assertEquals("doc17", h.getDocId()); + h = msg.getResult().getHit(1); + assertEquals(89.0, h.getRank(), 1E-6); + assertEquals("doc1", h.getDocId()); + + routable = deserialize("SearchResultMessage-4", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP); + assertTrue(routable instanceof SearchResultMessage); + + msg = (SearchResultMessage)routable; + assertEquals(3, msg.getResult().getHitCount()); + h = msg.getResult().getHit(0); + assertTrue(h instanceof SearchResult.HitWithSortBlob); + assertEquals(89.0, h.getRank(), 1E-6); + assertEquals("doc1", h.getDocId()); + byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); + assertEquals(9, b.length); + byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' }; + for (int i = 0; i < b.length; i++) { + assertEquals(e[i], b[i]); + } + h = msg.getResult().getHit(1); + assertTrue(h instanceof SearchResult.HitWithSortBlob); + assertEquals(109.0, h.getRank(), 1E-6); + assertEquals("doc17", h.getDocId()); + b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); + assertEquals(9, b.length); + byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' }; + for (int i = 0; i < b.length; i++) { + assertEquals(d[i], b[i]); + } + h = msg.getResult().getHit(2); + assertTrue(h instanceof SearchResult.HitWithSortBlob); + assertEquals(90.0, h.getRank(), 1E-6); + assertEquals("doc18", h.getDocId()); + b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); + assertEquals(9, b.length); + byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' }; + for (int i = 0; i < b.length; i++) { + assertEquals(c[i], b[i]); + } + } + } + + private static String CONDITION_STRING = "There's just one condition"; + + public class testPutDocumentMessage implements RunnableTest { + + @Override + public void run() { + PutDocumentMessage msg = new PutDocumentMessage(new DocumentPut(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:"))); + + msg.setTimestamp(666); + msg.setCondition(new TestAndSetCondition(CONDITION_STRING)); + + assertEquals(BASE_MESSAGE_LENGTH + 41 + serializedLength(msg.getCondition().getSelection()), serialize("PutDocumentMessage", msg)); + + for (Language lang : LANGUAGES) { + final PutDocumentMessage deserializedMsg = (PutDocumentMessage)deserialize("PutDocumentMessage", DocumentProtocol.MESSAGE_PUTDOCUMENT, lang); + assertEquals(msg.getDocumentPut().getDocument().getDataType().getName(), deserializedMsg.getDocumentPut().getDocument().getDataType().getName()); + assertEquals(msg.getDocumentPut().getDocument().getId().toString(), deserializedMsg.getDocumentPut().getDocument().getId().toString()); + assertEquals(msg.getTimestamp(), deserializedMsg.getTimestamp()); + assertEquals(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection()); + } + } + } + + public class testPutDocumentReply implements RunnableTest { + + @Override + public void run() { + WriteDocumentReply reply = new WriteDocumentReply(DocumentProtocol.REPLY_PUTDOCUMENT); + reply.setHighestModificationTimestamp(30); + + assertEquals(13, serialize("PutDocumentReply", reply)); + + for (Language lang : LANGUAGES) { + WriteDocumentReply obj = (WriteDocumentReply)deserialize("PutDocumentReply", DocumentProtocol.REPLY_PUTDOCUMENT, lang); + assertNotNull(obj); + assertEquals(30, obj.getHighestModificationTimestamp()); + } + } + } + + public class testUpdateDocumentMessage implements RunnableTest { + + @Override + public void run() { + DocumentType docType = protocol.getDocumentTypeManager().getDocumentType("testdoc"); + DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("doc:scheme:")); + update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0")); + + final UpdateDocumentMessage msg = new UpdateDocumentMessage(update); + msg.setNewTimestamp(777); + msg.setOldTimestamp(666); + msg.setCondition(new TestAndSetCondition(CONDITION_STRING)); + + assertEquals(BASE_MESSAGE_LENGTH + 89 + serializedLength(msg.getCondition().getSelection()), serialize("UpdateDocumentMessage", msg)); + + for (Language lang : LANGUAGES) { + final UpdateDocumentMessage deserializedMsg = (UpdateDocumentMessage) deserialize("UpdateDocumentMessage", DocumentProtocol.MESSAGE_UPDATEDOCUMENT, lang); + assertEquals(msg.getDocumentUpdate(), deserializedMsg.getDocumentUpdate()); + assertEquals(msg.getNewTimestamp(), deserializedMsg.getNewTimestamp()); + assertEquals(msg.getOldTimestamp(), deserializedMsg.getOldTimestamp()); + assertEquals(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection()); + } + } + } + + public class testUpdateDocumentReply implements RunnableTest { + + @Override + public void run() { + UpdateDocumentReply reply = new UpdateDocumentReply(); + reply.setHighestModificationTimestamp(30); + reply.setWasFound(false); + + assertEquals(14, serialize("UpdateDocumentReply", reply)); + + for (Language lang : LANGUAGES) { + UpdateDocumentReply obj = (UpdateDocumentReply)deserialize("UpdateDocumentReply", DocumentProtocol.REPLY_UPDATEDOCUMENT, lang); + assertNotNull(obj); + assertEquals(30, reply.getHighestModificationTimestamp()); + assertEquals(false, obj.wasFound()); + } + } + } + + public class testVisitorInfoReply implements RunnableTest { + + @Override + public void run() { + testVisitorReply("VisitorInfoReply", DocumentProtocol.REPLY_VISITORINFO); + } + } + + public class testWrongDistributionReply implements RunnableTest { + + @Override + public void run() { + WrongDistributionReply reply = new WrongDistributionReply("distributor:3 storage:2"); + assertEquals(32, serialize("WrongDistributionReply", reply)); + + for (Language lang : LANGUAGES) { + reply = (WrongDistributionReply)deserialize("WrongDistributionReply", DocumentProtocol.REPLY_WRONGDISTRIBUTION, lang); + assertEquals("distributor:3 storage:2", reply.getSystemState()); + } + } + } + + public class testRemoveDocumentReply implements RunnableTest { + + @Override + public void run() { + RemoveDocumentReply reply = new RemoveDocumentReply(); + reply.setHighestModificationTimestamp(30); + reply.setWasFound(false); + + assertEquals(14, serialize("RemoveDocumentReply", reply)); + + for (Language lang : LANGUAGES) { + RemoveDocumentReply obj = (RemoveDocumentReply)deserialize("RemoveDocumentReply", DocumentProtocol.REPLY_REMOVEDOCUMENT, lang); + assertNotNull(obj); + assertEquals(30, obj.getHighestModificationTimestamp()); + assertEquals(false, obj.wasFound()); + } + } + } + + public class testRemoveLocationReply implements RunnableTest { + + @Override + public void run() { + testDocumentReply("RemoveLocationReply", DocumentProtocol.REPLY_REMOVELOCATION); + } + } + + public class testSearchResultReply implements RunnableTest { + + @Override + public void run() { + testVisitorReply("SearchResultReply", DocumentProtocol.REPLY_SEARCHRESULT); + } + } + + public class testStatBucketReply implements RunnableTest { + + @Override + public void run() { + StatBucketReply msg = new StatBucketReply(); + msg.setResults("These are the votes of the Norwegian jury"); + + assertEquals(50, serialize("StatBucketReply", msg)); + + for (Language lang : LANGUAGES) { + msg = (StatBucketReply)deserialize("StatBucketReply", DocumentProtocol.REPLY_STATBUCKET, lang); + assertEquals("These are the votes of the Norwegian jury", msg.getResults()); + } + } + } + + public class testQueryResultReply implements RunnableTest { + + @Override + public void run() { + testVisitorReply("QueryResultReply", DocumentProtocol.REPLY_QUERYRESULT); + } + } + + public class testQueryResultMessage implements RunnableTest { + + @Override + public void run() throws Exception { + Routable routable = deserialize("QueryResultMessage-1", DocumentProtocol.MESSAGE_QUERYRESULT, Language.CPP); + assertTrue(routable instanceof QueryResultMessage); + + QueryResultMessage msg = (QueryResultMessage)routable; + assertEquals(0, msg.getResult().getHitCount()); + + routable = deserialize("QueryResultMessage-2", DocumentProtocol.MESSAGE_QUERYRESULT, Language.CPP); + assertTrue(routable instanceof QueryResultMessage); + + msg = (QueryResultMessage)routable; + assertEquals(2, msg.getResult().getHitCount()); + com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0); + assertEquals(89.0, h.getRank(), 1E-6); + assertEquals("doc1", h.getDocId()); + h = msg.getResult().getHit(1); + assertEquals(109.0, h.getRank(), 1E-6); + assertEquals("doc17", h.getDocId()); + + routable = deserialize("QueryResultMessage-3", DocumentProtocol.MESSAGE_QUERYRESULT, Language.CPP); + assertTrue(routable instanceof QueryResultMessage); + + msg = (QueryResultMessage)routable; + assertEquals(2, msg.getResult().getHitCount()); + h = msg.getResult().getHit(0); + assertEquals(109.0, h.getRank(), 1E-6); + assertEquals("doc17", h.getDocId()); + h = msg.getResult().getHit(1); + assertEquals(89.0, h.getRank(), 1E-6); + assertEquals("doc1", h.getDocId()); + + routable = deserialize("QueryResultMessage-4", DocumentProtocol.MESSAGE_QUERYRESULT, Language.CPP); + assertTrue(routable instanceof QueryResultMessage); + + msg = (QueryResultMessage)routable; + assertEquals(3, msg.getResult().getHitCount()); + h = msg.getResult().getHit(0); + assertTrue(h instanceof SearchResult.HitWithSortBlob); + assertEquals(89.0, h.getRank(), 1E-6); + assertEquals("doc1", h.getDocId()); + byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); + assertEquals(9, b.length); + byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' }; + for (int i = 0; i < b.length; i++) { + assertEquals(e[i], b[i]); + } + h = msg.getResult().getHit(1); + assertTrue(h instanceof SearchResult.HitWithSortBlob); + assertEquals(109.0, h.getRank(), 1E-6); + assertEquals("doc17", h.getDocId()); + b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); + assertEquals(9, b.length); + byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' }; + for (int i = 0; i < b.length; i++) { + assertEquals(d[i], b[i]); + } + h = msg.getResult().getHit(2); + assertTrue(h instanceof SearchResult.HitWithSortBlob); + assertEquals(90.0, h.getRank(), 1E-6); + assertEquals("doc18", h.getDocId()); + b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); + assertEquals(9, b.length); + byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' }; + for (int i = 0; i < b.length; i++) { + assertEquals(c[i], b[i]); + } + } + } + + public class testGetBucketListReply implements RunnableTest { + + public void run() { + GetBucketListReply reply = new GetBucketListReply(); + reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(16, 123), "foo")); + reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(17, 1123), "bar")); + reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(18, 11123), "zoink")); + + assertEquals(56, serialize("GetBucketListReply", reply)); + + for (Language lang : LANGUAGES) { + reply = (GetBucketListReply)deserialize("GetBucketListReply", DocumentProtocol.REPLY_GETBUCKETLIST, lang); + assertEquals(reply.getBuckets().get(0), new GetBucketListReply.BucketInfo(new BucketId(16, 123), "foo")); + assertEquals(reply.getBuckets().get(1), new GetBucketListReply.BucketInfo(new BucketId(17, 1123), "bar")); + assertEquals(reply.getBuckets().get(2), new GetBucketListReply.BucketInfo(new BucketId(18, 11123), "zoink")); + } + } + } + + public class testGetBucketStateReply implements RunnableTest { + + public void run() { + GlobalId foo = new GlobalId(IdString.createIdString("doc:scheme:foo")); + GlobalId bar = new GlobalId(IdString.createIdString("doc:scheme:bar")); + + GetBucketStateReply reply = new GetBucketStateReply(); + List<DocumentState> state = new ArrayList<>(2); + state.add(new DocumentState(foo, 777, false)); + state.add(new DocumentState(bar, 888, true)); + reply.setBucketState(state); + assertEquals(53, serialize("GetBucketStateReply", reply)); + + for (Language lang : LANGUAGES) { + reply = (GetBucketStateReply)deserialize("GetBucketStateReply", DocumentProtocol.REPLY_GETBUCKETSTATE, lang); + assertEquals(777, reply.getBucketState().get(0).getTimestamp()); + assertEquals(foo, reply.getBucketState().get(0).getGid()); + assertEquals(false, reply.getBucketState().get(0).isRemoveEntry()); + assertEquals(888, reply.getBucketState().get(1).getTimestamp()); + assertEquals(bar, reply.getBucketState().get(1).getGid()); + assertEquals(true, reply.getBucketState().get(1).isRemoveEntry()); + } + } + } + + public class testGetDocumentReply implements RunnableTest { + + public void run() { + GetDocumentReply reply = new GetDocumentReply(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:")); + assertEquals(43, serialize("GetDocumentReply", reply)); + + for (Language lang : LANGUAGES) { + reply = (GetDocumentReply)deserialize("GetDocumentReply", DocumentProtocol.REPLY_GETDOCUMENT, lang); + assertEquals("testdoc", reply.getDocument().getDataType().getName()); + assertEquals("doc:scheme:", reply.getDocument().getId().toString()); + } + } + } + + public class testMapVisitorReply implements RunnableTest { + + public void run() { + testVisitorReply("MapVisitorReply", DocumentProtocol.REPLY_MAPVISITOR); + } + } + + protected void testDocumentReply(String filename, int type) { + DocumentReply reply = new DocumentReply(type); + assertEquals(5, serialize(filename, reply)); + + for (Language lang : LANGUAGES) { + reply = (DocumentReply)deserialize(filename, type, lang); + assertNotNull(reply); + } + } + + protected void testVisitorReply(String filename, int type) { + VisitorReply reply = new VisitorReply(type); + assertEquals(5, serialize(filename, reply)); + + for (Language lang : LANGUAGES) { + reply = (VisitorReply)deserialize(filename, type, lang); + assertNotNull(reply); + } + } + + static int serializedLength(String str) { + return 4 + str.length(); + } } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java index 59cbf5b65ae..d9fd307d9d9 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java @@ -8,7 +8,7 @@ import com.yahoo.document.serialization.DocumentSerializer; import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.messagebus.protocol.DocumentReply; -import com.yahoo.documentapi.messagebus.protocol.RoutableFactories52; +import com.yahoo.documentapi.messagebus.protocol.RoutableFactories60; import com.yahoo.jrt.ListenFailedException; import com.yahoo.jrt.slobrok.server.Slobrok; import com.yahoo.messagebus.DestinationSession; @@ -130,7 +130,7 @@ public class RoutableFactoryTestCase { assertFalse(reply.hasErrors()); } - private static class MyMessageFactory extends RoutableFactories52.DocumentMessageFactory { + private static class MyMessageFactory extends RoutableFactories60.DocumentMessageFactory { @Override protected DocumentMessage doDecode(DocumentDeserializer buf) { @@ -143,7 +143,7 @@ public class RoutableFactoryTestCase { } } - private static class MyReplyFactory extends RoutableFactories52.DocumentReplyFactory { + private static class MyReplyFactory extends RoutableFactories60.DocumentReplyFactory { @Override protected DocumentReply doDecode(DocumentDeserializer buf) { diff --git a/documentapi/src/tests/messagebus/messagebus_test.cpp b/documentapi/src/tests/messagebus/messagebus_test.cpp index 22a46951ecc..05275870a2e 100644 --- a/documentapi/src/tests/messagebus/messagebus_test.cpp +++ b/documentapi/src/tests/messagebus/messagebus_test.cpp @@ -70,10 +70,10 @@ void Test::testMessage() { LoadTypeSet set; DocumentProtocol protocol(set, _repo); - Blob blob = protocol.encode(vespalib::Version(5,115), upd1); + Blob blob = protocol.encode(vespalib::Version(6,221), upd1); EXPECT_TRUE(blob.size() > 0); - Routable::UP dec1 = protocol.decode(vespalib::Version(5,115), blob); + Routable::UP dec1 = protocol.decode(vespalib::Version(6,221), blob); EXPECT_TRUE(dec1.get() != NULL); EXPECT_TRUE(dec1->isReply() == false); EXPECT_TRUE(dec1->getType() == DocumentProtocol::MESSAGE_UPDATEDOCUMENT); diff --git a/documentapi/src/tests/messages/CMakeLists.txt b/documentapi/src/tests/messages/CMakeLists.txt index 576557968e8..87f8d4d01cd 100644 --- a/documentapi/src/tests/messages/CMakeLists.txt +++ b/documentapi/src/tests/messages/CMakeLists.txt @@ -1,17 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(documentapi_messages52_test_app TEST - SOURCES - testbase.cpp - messages52test.cpp - messages52app.cpp - DEPENDS - documentapi -) -vespa_add_test(NAME documentapi_messages52_test_app COMMAND documentapi_messages52_test_app) vespa_add_executable(documentapi_messages60_test_app TEST SOURCES testbase.cpp - messages52test.cpp messages60test.cpp messages60app.cpp DEPENDS diff --git a/documentapi/src/tests/messages/messages52app.cpp b/documentapi/src/tests/messages/messages52app.cpp deleted file mode 100644 index 4a925fd554f..00000000000 --- a/documentapi/src/tests/messages/messages52app.cpp +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "messages52test.h" - -TEST_APPHOOK(Messages52Test); diff --git a/documentapi/src/tests/messages/messages52test.cpp b/documentapi/src/tests/messages/messages52test.cpp deleted file mode 100644 index 21307326a75..00000000000 --- a/documentapi/src/tests/messages/messages52test.cpp +++ /dev/null @@ -1,1061 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// @author Vegard Sjonfjell - -#include "messages52test.h" -#include <vespa/document/bucket/bucketidfactory.h> -#include <vespa/document/datatype/documenttype.h> -#include <vespa/document/select/parser.h> -#include <vespa/document/repo/documenttyperepo.h> -#include <vespa/document/update/documentupdate.h> -#include <vespa/document/update/fieldpathupdates.h> -#include <vespa/documentapi/documentapi.h> -#include <vespa/document/bucket/fixed_bucket_spaces.h> - -using document::DataType; -using document::DocumentTypeRepo; - -/////////////////////////////////////////////////////////////////////////////// -// -// Setup -// -/////////////////////////////////////////////////////////////////////////////// - -Messages52Test::Messages52Test() -{ - // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support - // version 5.0. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now. - putTest(DocumentProtocol::MESSAGE_CREATEVISITOR, TEST_METHOD(Messages52Test::testCreateVisitorMessage)); - putTest(DocumentProtocol::MESSAGE_DESTROYVISITOR, TEST_METHOD(Messages52Test::testDestroyVisitorMessage)); - putTest(DocumentProtocol::MESSAGE_DOCUMENTLIST, TEST_METHOD(Messages52Test::testDocumentListMessage)); - putTest(DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, TEST_METHOD(Messages52Test::testDocumentSummaryMessage)); - putTest(DocumentProtocol::MESSAGE_EMPTYBUCKETS, TEST_METHOD(Messages52Test::testEmptyBucketsMessage)); - putTest(DocumentProtocol::MESSAGE_GETBUCKETLIST, TEST_METHOD(Messages52Test::testGetBucketListMessage)); - putTest(DocumentProtocol::MESSAGE_GETBUCKETSTATE, TEST_METHOD(Messages52Test::testGetBucketStateMessage)); - putTest(DocumentProtocol::MESSAGE_GETDOCUMENT, TEST_METHOD(Messages52Test::testGetDocumentMessage)); - putTest(DocumentProtocol::MESSAGE_MAPVISITOR, TEST_METHOD(Messages52Test::testMapVisitorMessage)); - putTest(DocumentProtocol::MESSAGE_PUTDOCUMENT, TEST_METHOD(Messages52Test::testPutDocumentMessage)); - putTest(DocumentProtocol::MESSAGE_QUERYRESULT, TEST_METHOD(Messages52Test::testQueryResultMessage)); - putTest(DocumentProtocol::MESSAGE_REMOVEDOCUMENT, TEST_METHOD(Messages52Test::testRemoveDocumentMessage)); - putTest(DocumentProtocol::MESSAGE_REMOVELOCATION, TEST_METHOD(Messages52Test::testRemoveLocationMessage)); - putTest(DocumentProtocol::MESSAGE_SEARCHRESULT, TEST_METHOD(Messages52Test::testSearchResultMessage)); - putTest(DocumentProtocol::MESSAGE_STATBUCKET, TEST_METHOD(Messages52Test::testStatBucketMessage)); - putTest(DocumentProtocol::MESSAGE_UPDATEDOCUMENT, TEST_METHOD(Messages52Test::testUpdateDocumentMessage)); - putTest(DocumentProtocol::MESSAGE_VISITORINFO, TEST_METHOD(Messages52Test::testVisitorInfoMessage)); - - putTest(DocumentProtocol::REPLY_CREATEVISITOR, TEST_METHOD(Messages52Test::testCreateVisitorReply)); - putTest(DocumentProtocol::REPLY_DESTROYVISITOR, TEST_METHOD(Messages52Test::testDestroyVisitorReply)); - putTest(DocumentProtocol::REPLY_DOCUMENTIGNORED, TEST_METHOD(Messages52Test::testDocumentIgnoredReply)); - putTest(DocumentProtocol::REPLY_DOCUMENTLIST, TEST_METHOD(Messages52Test::testDocumentListReply)); - putTest(DocumentProtocol::REPLY_DOCUMENTSUMMARY, TEST_METHOD(Messages52Test::testDocumentSummaryReply)); - putTest(DocumentProtocol::REPLY_EMPTYBUCKETS, TEST_METHOD(Messages52Test::testEmptyBucketsReply)); - putTest(DocumentProtocol::REPLY_GETBUCKETLIST, TEST_METHOD(Messages52Test::testGetBucketListReply)); - putTest(DocumentProtocol::REPLY_GETBUCKETSTATE, TEST_METHOD(Messages52Test::testGetBucketStateReply)); - putTest(DocumentProtocol::REPLY_GETDOCUMENT, TEST_METHOD(Messages52Test::testGetDocumentReply)); - putTest(DocumentProtocol::REPLY_MAPVISITOR, TEST_METHOD(Messages52Test::testMapVisitorReply)); - putTest(DocumentProtocol::REPLY_PUTDOCUMENT, TEST_METHOD(Messages52Test::testPutDocumentReply)); - putTest(DocumentProtocol::REPLY_QUERYRESULT, TEST_METHOD(Messages52Test::testQueryResultReply)); - putTest(DocumentProtocol::REPLY_REMOVEDOCUMENT, TEST_METHOD(Messages52Test::testRemoveDocumentReply)); - putTest(DocumentProtocol::REPLY_REMOVELOCATION, TEST_METHOD(Messages52Test::testRemoveLocationReply)); - putTest(DocumentProtocol::REPLY_SEARCHRESULT, TEST_METHOD(Messages52Test::testSearchResultReply)); - putTest(DocumentProtocol::REPLY_STATBUCKET, TEST_METHOD(Messages52Test::testStatBucketReply)); - putTest(DocumentProtocol::REPLY_UPDATEDOCUMENT, TEST_METHOD(Messages52Test::testUpdateDocumentReply)); - putTest(DocumentProtocol::REPLY_VISITORINFO, TEST_METHOD(Messages52Test::testVisitorInfoReply)); - putTest(DocumentProtocol::REPLY_WRONGDISTRIBUTION, TEST_METHOD(Messages52Test::testWrongDistributionReply)); -} - - - -/////////////////////////////////////////////////////////////////////////////// -// -// Tests -// -/////////////////////////////////////////////////////////////////////////////// - -static const int MESSAGE_BASE_LENGTH = 5; - -namespace { - -document::Document::SP -createDoc(const DocumentTypeRepo &repo, const string &type_name, const string &id) -{ - return document::Document::SP(new document::Document( - *repo.getDocumentType(type_name), - document::DocumentId(id))); -} - -} // namespace - -bool -Messages52Test::testGetBucketListMessage() -{ - GetBucketListMessage msg(document::BucketId(16, 123)); - msg.setLoadType(_loadTypes["foo"]); - EXPECT_EQUAL(string("foo"), msg.getLoadType().getName()); - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 12u, serialize("GetBucketListMessage", msg)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("GetBucketListMessage", DocumentProtocol::MESSAGE_GETBUCKETLIST, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - GetBucketListMessage &ref = static_cast<GetBucketListMessage&>(*obj); - EXPECT_EQUAL(string("foo"), ref.getLoadType().getName()); - EXPECT_EQUAL(document::BucketId(16, 123), ref.getBucketId()); - } - } - return true; -} - -bool -Messages52Test::testEmptyBucketsMessage() -{ - std::vector<document::BucketId> bids; - for (size_t i=0; i < 13; ++i) { - bids.push_back(document::BucketId(16, i)); - } - - EmptyBucketsMessage msg(bids); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 112u, serialize("EmptyBucketsMessage", msg)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("EmptyBucketsMessage", DocumentProtocol::MESSAGE_EMPTYBUCKETS, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - EmptyBucketsMessage &ref = static_cast<EmptyBucketsMessage&>(*obj); - for (size_t i=0; i < 13; ++i) { - EXPECT_EQUAL(document::BucketId(16, i), ref.getBucketIds()[i]); - } - } - } - return true; -} - - -bool -Messages52Test::testStatBucketMessage() -{ - StatBucketMessage msg(document::BucketId(16, 123), "id.user=123"); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 27u, serialize("StatBucketMessage", msg)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("StatBucketMessage", DocumentProtocol::MESSAGE_STATBUCKET, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - StatBucketMessage &ref = static_cast<StatBucketMessage&>(*obj); - EXPECT_EQUAL(document::BucketId(16, 123), ref.getBucketId()); - EXPECT_EQUAL("id.user=123", ref.getDocumentSelection()); - } - } - return true; -} - -bool -Messages52Test::testCreateVisitorMessage() { - CreateVisitorMessage tmp("SomeLibrary", "myvisitor", "newyork", "london"); - tmp.setDocumentSelection("true and false or true"); - tmp.getParameters().set("myvar", "somevalue"); - tmp.getParameters().set("anothervar", uint64_t(34)); - tmp.getBuckets().push_back(document::BucketId(16, 1234)); - tmp.setVisitRemoves(true); - tmp.setFieldSet("foo bar"); - tmp.setVisitorOrdering(document::OrderingSpecification::DESCENDING); - tmp.setMaxBucketsPerVisitor(2); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + (size_t)178, serialize("CreateVisitorMessage", tmp)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("CreateVisitorMessage", DocumentProtocol::MESSAGE_CREATEVISITOR, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - CreateVisitorMessage &ref = static_cast<CreateVisitorMessage&>(*obj); - - EXPECT_EQUAL(string("SomeLibrary"), ref.getLibraryName()); - EXPECT_EQUAL(string("myvisitor"), ref.getInstanceId()); - EXPECT_EQUAL(string("newyork"), ref.getControlDestination()); - EXPECT_EQUAL(string("london"), ref.getDataDestination()); - EXPECT_EQUAL(string("true and false or true"), ref.getDocumentSelection()); - EXPECT_EQUAL(string("foo bar"), ref.getFieldSet()); - EXPECT_EQUAL(uint32_t(8), ref.getMaximumPendingReplyCount()); - EXPECT_EQUAL(true, ref.visitRemoves()); - EXPECT_EQUAL(false, ref.visitHeadersOnly()); - EXPECT_EQUAL(false, ref.visitInconsistentBuckets()); - EXPECT_EQUAL(size_t(1), ref.getBuckets().size()); - EXPECT_EQUAL(document::BucketId(16, 1234), ref.getBuckets()[0]); - EXPECT_EQUAL(string("somevalue"), ref.getParameters().get("myvar")); - EXPECT_EQUAL(uint64_t(34), ref.getParameters().get("anothervar", uint64_t(1))); - EXPECT_EQUAL(document::OrderingSpecification::DESCENDING, ref.getVisitorOrdering()); - EXPECT_EQUAL(uint32_t(2), ref.getMaxBucketsPerVisitor()); - } - } - return true; -} - -bool -Messages52Test::testDestroyVisitorMessage() -{ - DestroyVisitorMessage tmp("myvisitor"); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + (size_t)17, serialize("DestroyVisitorMessage", tmp)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("DestroyVisitorMessage", DocumentProtocol::MESSAGE_DESTROYVISITOR, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - DestroyVisitorMessage &ref = static_cast<DestroyVisitorMessage&>(*obj); - EXPECT_EQUAL(string("myvisitor"), ref.getInstanceId()); - } - } - return true; -} - -bool -Messages52Test::testDocumentListMessage() -{ - document::Document::SP doc = - createDoc(getTypeRepo(), "testdoc", "userdoc:scheme:1234:"); - DocumentListMessage::Entry entry(1234, doc, false); - - DocumentListMessage tmp(document::BucketId(16, 1234)); - tmp.getDocuments().push_back(entry); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + (size_t)63, serialize("DocumentListMessage", tmp)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("DocumentListMessage", DocumentProtocol::MESSAGE_DOCUMENTLIST, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - DocumentListMessage &ref = static_cast<DocumentListMessage&>(*obj); - - EXPECT_EQUAL("userdoc:scheme:1234:", ref.getDocuments()[0].getDocument()->getId().toString()); - EXPECT_EQUAL(1234, ref.getDocuments()[0].getTimestamp()); - EXPECT_TRUE(!ref.getDocuments()[0].isRemoveEntry()); - } - } - return true; -} - - -bool -Messages52Test::testRemoveLocationMessage() -{ - { - document::BucketIdFactory factory; - document::select::Parser parser(getTypeRepo(), factory); - RemoveLocationMessage msg(factory, parser, "id.group == \"mygroup\""); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 29u, serialize("RemoveLocationMessage", msg)); - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("RemoveLocationMessage", DocumentProtocol::MESSAGE_REMOVELOCATION, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - RemoveLocationMessage &ref = static_cast<RemoveLocationMessage&>(*obj); - EXPECT_EQUAL(string("id.group == \"mygroup\""), ref.getDocumentSelection()); - // FIXME add to wire format, currently hardcoded. - EXPECT_EQUAL(string(document::FixedBucketSpaces::default_space_name()), ref.getBucketSpace()); - } - } - } - - return true; -} - - - -bool -Messages52Test::testDocumentSummaryMessage() -{ - DocumentSummaryMessage srm; - EXPECT_EQUAL(srm.hasSequenceId(), false); - EXPECT_EQUAL(srm.getSummaryCount(), size_t(0)); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(12), serialize("DocumentSummaryMessage-1", srm)); - - mbus::Routable::UP routable = deserialize("DocumentSummaryMessage-1", DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, LANG_CPP); - if (!EXPECT_TRUE(routable.get() != NULL)) { - return false; - } - DocumentSummaryMessage * dm = static_cast<DocumentSummaryMessage *>(routable.get()); - EXPECT_EQUAL(dm->getSummaryCount(), size_t(0)); - - srm.addSummary("doc1", "summary1", 8); - srm.addSummary("aoc17", "summary45", 9); - - const void *summary(NULL); - const char *docId(NULL); - size_t sz(0); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 52u, serialize("DocumentSummaryMessage-2", srm)); - routable = deserialize("DocumentSummaryMessage-2", DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, LANG_CPP); - if (!EXPECT_TRUE(routable.get() != NULL)) { - return false; - } - dm = static_cast<DocumentSummaryMessage *>(routable.get()); - EXPECT_EQUAL(dm->getSummaryCount(), size_t(2)); - dm->getSummary(0, docId, summary, sz); - EXPECT_EQUAL(sz, 8u); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - EXPECT_EQUAL(memcmp("summary1", summary, sz), 0); - dm->getSummary(1, docId, summary, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(strcmp("aoc17", docId), 0); - EXPECT_EQUAL(memcmp("summary45", summary, sz), 0); - - srm.sort(); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 52u, serialize("DocumentSummaryMessage-3", srm)); - routable = deserialize("DocumentSummaryMessage-3", DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, LANG_CPP); - if (!EXPECT_TRUE(routable.get() != NULL)) { - return false; - } - dm = static_cast<DocumentSummaryMessage *>(routable.get()); - EXPECT_EQUAL(dm->getSummaryCount(), size_t(2)); - dm->getSummary(0, docId, summary, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(strcmp("aoc17", docId), 0); - EXPECT_EQUAL(memcmp("summary45", summary, sz), 0); - dm->getSummary(1, docId, summary, sz); - EXPECT_EQUAL(sz, 8u); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - EXPECT_EQUAL(memcmp("summary1", summary, sz), 0); - return true; -} - -bool -Messages52Test::testGetDocumentMessage() -{ - GetDocumentMessage tmp(document::DocumentId("doc:scheme:"), "foo bar"); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + (size_t)27, serialize("GetDocumentMessage", tmp)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("GetDocumentMessage", DocumentProtocol::MESSAGE_GETDOCUMENT, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - GetDocumentMessage &ref = static_cast<GetDocumentMessage&>(*obj); - EXPECT_EQUAL(string("doc:scheme:"), ref.getDocumentId().toString()); - EXPECT_EQUAL(string("foo bar"), ref.getFieldSet()); - } - } - return true; -} - -bool -Messages52Test::testMapVisitorMessage() -{ - MapVisitorMessage tmp; - tmp.getData().set("foo", 3); - tmp.getData().set("bar", 5); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + (size_t)32, serialize("MapVisitorMessage", tmp)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("MapVisitorMessage", DocumentProtocol::MESSAGE_MAPVISITOR, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - MapVisitorMessage &ref = static_cast<MapVisitorMessage&>(*obj); - EXPECT_EQUAL(3, ref.getData().get("foo", 0)); - EXPECT_EQUAL(5, ref.getData().get("bar", 0)); - } - } - return true; -} - -bool -Messages52Test::testCreateVisitorReply() -{ - CreateVisitorReply reply(DocumentProtocol::REPLY_CREATEVISITOR); - reply.setLastBucket(document::BucketId(16, 123)); - vdslib::VisitorStatistics vs; - vs.setBucketsVisited(3); - vs.setDocumentsVisited(1000); - vs.setBytesVisited(1024000); - vs.setDocumentsReturned(123); - vs.setBytesReturned(512000); - vs.setSecondPassDocumentsReturned(456); - vs.setSecondPassBytesReturned(789100); - reply.setVisitorStatistics(vs); - - EXPECT_EQUAL(65u, serialize("CreateVisitorReply", reply)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("CreateVisitorReply", DocumentProtocol::REPLY_CREATEVISITOR, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - CreateVisitorReply &ref = static_cast<CreateVisitorReply&>(*obj); - - EXPECT_EQUAL(ref.getLastBucket(), document::BucketId(16, 123)); - EXPECT_EQUAL(ref.getVisitorStatistics().getBucketsVisited(), (uint32_t)3); - EXPECT_EQUAL(ref.getVisitorStatistics().getDocumentsVisited(), (uint64_t)1000); - EXPECT_EQUAL(ref.getVisitorStatistics().getBytesVisited(), (uint64_t)1024000); - EXPECT_EQUAL(ref.getVisitorStatistics().getDocumentsReturned(), (uint64_t)123); - EXPECT_EQUAL(ref.getVisitorStatistics().getBytesReturned(), (uint64_t)512000); - EXPECT_EQUAL(ref.getVisitorStatistics().getSecondPassDocumentsReturned(), (uint64_t)456); - EXPECT_EQUAL(ref.getVisitorStatistics().getSecondPassBytesReturned(), (uint64_t)789100); - } - } - return true; -} - -bool -Messages52Test::testPutDocumentMessage() -{ - auto doc = createDoc(getTypeRepo(), "testdoc", "doc:scheme:"); - PutDocumentMessage msg(doc); - - msg.setTimestamp(666); - msg.setCondition(TestAndSetCondition("There's just one condition")); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + - 41u + - serializedLength(msg.getCondition().getSelection()), - serialize("PutDocumentMessage", msg)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - auto routableUp = deserialize("PutDocumentMessage", DocumentProtocol::MESSAGE_PUTDOCUMENT, lang); - if (EXPECT_TRUE(routableUp.get() != nullptr)) { - auto & deserializedMsg = static_cast<PutDocumentMessage &>(*routableUp); - - EXPECT_EQUAL(msg.getDocument().getType().getName(), deserializedMsg.getDocument().getType().getName()); - EXPECT_EQUAL(msg.getDocument().getId().toString(), deserializedMsg.getDocument().getId().toString()); - EXPECT_EQUAL(msg.getTimestamp(), deserializedMsg.getTimestamp()); - EXPECT_EQUAL(67u, deserializedMsg.getApproxSize()); - EXPECT_EQUAL(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection()); - } - } - - return true; -} - -bool -Messages52Test::testGetBucketStateMessage() -{ - GetBucketStateMessage tmp; - tmp.setBucketId(document::BucketId(16, 666)); - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 12u, serialize("GetBucketStateMessage", tmp)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("GetBucketStateMessage", DocumentProtocol::MESSAGE_GETBUCKETSTATE, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - GetBucketStateMessage &ref = static_cast<GetBucketStateMessage&>(*obj); - - EXPECT_EQUAL(16u, ref.getBucketId().getUsedBits()); - EXPECT_EQUAL(4611686018427388570ull, ref.getBucketId().getId()); - } - } - return true; -} - -bool -Messages52Test::testPutDocumentReply() -{ - WriteDocumentReply reply(DocumentProtocol::REPLY_PUTDOCUMENT); - reply.setHighestModificationTimestamp(30); - - EXPECT_EQUAL(13u, serialize("PutDocumentReply", reply)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("PutDocumentReply", DocumentProtocol::REPLY_PUTDOCUMENT, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - WriteDocumentReply &ref = static_cast<WriteDocumentReply&>(*obj); - EXPECT_EQUAL(30u, ref.getHighestModificationTimestamp()); - } - } - return true; -} - -bool -Messages52Test::testUpdateDocumentReply() -{ - UpdateDocumentReply reply; - reply.setWasFound(false); - reply.setHighestModificationTimestamp(30); - - EXPECT_EQUAL(14u, serialize("UpdateDocumentReply", reply)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("UpdateDocumentReply", DocumentProtocol::REPLY_UPDATEDOCUMENT, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - UpdateDocumentReply &ref = static_cast<UpdateDocumentReply&>(*obj); - EXPECT_EQUAL(30u, ref.getHighestModificationTimestamp()); - EXPECT_EQUAL(false, ref.wasFound()); - } - } - return true; -} - -bool -Messages52Test::testRemoveDocumentMessage() -{ - RemoveDocumentMessage msg(document::DocumentId("doc:scheme:")); - - msg.setCondition(TestAndSetCondition("There's just one condition")); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(16) + serializedLength(msg.getCondition().getSelection()), serialize("RemoveDocumentMessage", msg)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - auto routablePtr = deserialize("RemoveDocumentMessage", DocumentProtocol::MESSAGE_REMOVEDOCUMENT, lang); - - if (EXPECT_TRUE(routablePtr.get() != nullptr)) { - auto & ref = static_cast<RemoveDocumentMessage &>(*routablePtr); - EXPECT_EQUAL(string("doc:scheme:"), ref.getDocumentId().toString()); - EXPECT_EQUAL(msg.getCondition().getSelection(), ref.getCondition().getSelection()); - } - } - return true; -} - -bool -Messages52Test::testRemoveDocumentReply() -{ - RemoveDocumentReply reply; - std::vector<uint64_t> ts; - reply.setWasFound(false); - reply.setHighestModificationTimestamp(30); - - EXPECT_EQUAL(14u, serialize("RemoveDocumentReply", reply)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("RemoveDocumentReply", DocumentProtocol::REPLY_REMOVEDOCUMENT, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - RemoveDocumentReply &ref = static_cast<RemoveDocumentReply&>(*obj); - EXPECT_EQUAL(30u, ref.getHighestModificationTimestamp()); - EXPECT_EQUAL(false, ref.wasFound()); - } - } - return true; -} - -bool -Messages52Test::testSearchResultMessage() -{ - SearchResultMessage srm; - EXPECT_EQUAL(srm.getSequenceId(), 0u); - EXPECT_EQUAL(srm.getHitCount(), 0u); - EXPECT_EQUAL(srm.getAggregatorList().getSerializedSize(), 4u); - EXPECT_EQUAL(srm.vdslib::SearchResult::getSerializedSize(), 20u); - EXPECT_EQUAL(srm.getSerializedSize(), 20u); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(24), serialize("SearchResultMessage-1", srm)); - - mbus::Routable::UP routable = deserialize("SearchResultMessage-1", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); - if (!EXPECT_TRUE(routable.get() != NULL)) { - return false; - } - SearchResultMessage * dm = static_cast<SearchResultMessage *>(routable.get()); - EXPECT_EQUAL(dm->getSequenceId(), size_t(0)); - EXPECT_EQUAL(dm->getHitCount(), size_t(0)); - - srm.addHit(0, "doc1", 89); - srm.addHit(1, "doc17", 109); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 55u, serialize("SearchResultMessage-2", srm)); - routable = deserialize("SearchResultMessage-2", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); - if (!EXPECT_TRUE(routable.get() != NULL)) { - return false; - } - dm = static_cast<SearchResultMessage *>(routable.get()); - EXPECT_EQUAL(dm->getHitCount(), size_t(2)); - const char *docId; - SearchResultMessage::RankType rank; - dm->getHit(0, docId, rank); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - dm->getHit(1, docId, rank); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); - EXPECT_EQUAL(strcmp("doc17", docId), 0); - - srm.sort(); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 55u, serialize("SearchResultMessage-3", srm)); - routable = deserialize("SearchResultMessage-3", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); - if (!EXPECT_TRUE(routable.get() != NULL)) { - return false; - } - dm = static_cast<SearchResultMessage *>(routable.get()); - EXPECT_EQUAL(dm->getHitCount(), size_t(2)); - dm->getHit(0, docId, rank); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); - EXPECT_EQUAL(strcmp("doc17", docId), 0); - dm->getHit(1, docId, rank); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - - SearchResultMessage srm2; - srm2.addHit(0, "doc1", 89, "sortdata2", 9); - srm2.addHit(1, "doc17", 109, "sortdata1", 9); - srm2.addHit(2, "doc18", 90, "sortdata3", 9); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 108u, serialize("SearchResultMessage-4", srm2)); - routable = deserialize("SearchResultMessage-4", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); - if (!EXPECT_TRUE(routable.get() != NULL)) { - return false; - } - dm = static_cast<SearchResultMessage *>(routable.get()); - EXPECT_EQUAL(dm->getHitCount(), size_t(3)); - dm->getHit(0, docId, rank); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - dm->getHit(1, docId, rank); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); - EXPECT_EQUAL(strcmp("doc17", docId), 0); - dm->getHit(2, docId, rank); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(90)); - EXPECT_EQUAL(strcmp("doc18", docId), 0); - - srm2.sort(); - const void *buf; - size_t sz; - srm2.getHit(0, docId, rank); - srm2.getSortBlob(0, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata1", buf, sz), 0); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); - EXPECT_EQUAL(strcmp("doc17", docId), 0); - srm2.getHit(1, docId, rank); - srm2.getSortBlob(1, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata2", buf, sz), 0); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - srm2.getHit(2, docId, rank); - srm2.getSortBlob(2, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata3", buf, sz), 0); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(90)); - EXPECT_EQUAL(strcmp("doc18", docId), 0); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 108u, serialize("SearchResultMessage-5", srm2)); - routable = deserialize("SearchResultMessage-5", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); - if (!EXPECT_TRUE(routable.get() != NULL)) { - return false; - } - dm = static_cast<SearchResultMessage *>(routable.get()); - EXPECT_EQUAL(dm->getHitCount(), size_t(3)); - dm->getHit(0, docId, rank); - dm->getSortBlob(0, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata1", buf, sz), 0); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); - EXPECT_EQUAL(strcmp("doc17", docId), 0); - dm->getHit(1, docId, rank); - dm->getSortBlob(1, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata2", buf, sz), 0); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - dm->getHit(2, docId, rank); - dm->getSortBlob(2, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata3", buf, sz), 0); - EXPECT_EQUAL(rank, SearchResultMessage::RankType(90)); - EXPECT_EQUAL(strcmp("doc18", docId), 0); - return true; -} - -bool -Messages52Test::testUpdateDocumentMessage() -{ - const DocumentTypeRepo & repo = getTypeRepo(); - const document::DocumentType & docType = *repo.getDocumentType("testdoc"); - - auto docUpdate = std::make_shared<document::DocumentUpdate>(repo, docType, document::DocumentId("doc:scheme:")); - - docUpdate->addFieldPathUpdate(document::FieldPathUpdate::CP( - new document::RemoveFieldPathUpdate("intfield", "testdoc.intfield > 0"))); - - UpdateDocumentMessage msg(docUpdate); - msg.setOldTimestamp(666u); - msg.setNewTimestamp(777u); - msg.setCondition(TestAndSetCondition("There's just one condition")); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 89u + serializedLength(msg.getCondition().getSelection()), serialize("UpdateDocumentMessage", msg)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - auto routableUp = deserialize("UpdateDocumentMessage", DocumentProtocol::MESSAGE_UPDATEDOCUMENT, lang); - - if (EXPECT_TRUE(routableUp.get() != nullptr)) { - auto & deserializedMsg = static_cast<UpdateDocumentMessage &>(*routableUp); - EXPECT_EQUAL(msg.getDocumentUpdate(), deserializedMsg.getDocumentUpdate()); - EXPECT_EQUAL(msg.getOldTimestamp(), deserializedMsg.getOldTimestamp()); - EXPECT_EQUAL(msg.getNewTimestamp(), deserializedMsg.getNewTimestamp()); - EXPECT_EQUAL(115u, deserializedMsg.getApproxSize()); - EXPECT_EQUAL(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection()); - } - } - return true; -} - -bool -Messages52Test::testQueryResultMessage() -{ - QueryResultMessage srm; - vdslib::SearchResult & sr(srm.getSearchResult()); - EXPECT_EQUAL(srm.getSequenceId(), 0u); - EXPECT_EQUAL(sr.getHitCount(), 0u); - EXPECT_EQUAL(sr.getAggregatorList().getSerializedSize(), 4u); - EXPECT_EQUAL(sr.getSerializedSize(), 20u); - EXPECT_EQUAL(srm.getApproxSize(), 28u); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(32), serialize("QueryResultMessage-1", srm)); - - mbus::Routable::UP routable = deserialize("QueryResultMessage-1", DocumentProtocol::MESSAGE_QUERYRESULT, LANG_CPP); - if (!EXPECT_TRUE(routable.get() != NULL)) { - return false; - } - QueryResultMessage * dm = static_cast<QueryResultMessage *>(routable.get()); - vdslib::SearchResult * dr(&dm->getSearchResult()); - EXPECT_EQUAL(dm->getSequenceId(), size_t(0)); - EXPECT_EQUAL(dr->getHitCount(), size_t(0)); - - sr.addHit(0, "doc1", 89); - sr.addHit(1, "doc17", 109); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 63u, serialize("QueryResultMessage-2", srm)); - routable = deserialize("QueryResultMessage-2", DocumentProtocol::MESSAGE_QUERYRESULT, LANG_CPP); - if (!EXPECT_TRUE(routable.get() != NULL)) { - return false; - } - dm = static_cast<QueryResultMessage *>(routable.get()); - dr = &dm->getSearchResult(); - EXPECT_EQUAL(dr->getHitCount(), size_t(2)); - const char *docId; - vdslib::SearchResult::RankType rank; - dr->getHit(0, docId, rank); - EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(89)); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - dr->getHit(1, docId, rank); - EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(109)); - EXPECT_EQUAL(strcmp("doc17", docId), 0); - - sr.sort(); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 63u, serialize("QueryResultMessage-3", srm)); - routable = deserialize("QueryResultMessage-3", DocumentProtocol::MESSAGE_QUERYRESULT, LANG_CPP); - if (!EXPECT_TRUE(routable.get() != NULL)) { - return false; - } - dm = static_cast<QueryResultMessage *>(routable.get()); - dr = &dm->getSearchResult(); - EXPECT_EQUAL(dr->getHitCount(), size_t(2)); - dr->getHit(0, docId, rank); - EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(109)); - EXPECT_EQUAL(strcmp("doc17", docId), 0); - dr->getHit(1, docId, rank); - EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(89)); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - - QueryResultMessage srm2; - vdslib::SearchResult & sr2(srm2.getSearchResult()); - sr2.addHit(0, "doc1", 89, "sortdata2", 9); - sr2.addHit(1, "doc17", 109, "sortdata1", 9); - sr2.addHit(2, "doc18", 90, "sortdata3", 9); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 116u, serialize("QueryResultMessage-4", srm2)); - routable = deserialize("QueryResultMessage-4", DocumentProtocol::MESSAGE_QUERYRESULT, LANG_CPP); - if (!EXPECT_TRUE(routable.get() != NULL)) { - return false; - } - dm = static_cast<QueryResultMessage *>(routable.get()); - dr = &dm->getSearchResult(); - EXPECT_EQUAL(dr->getHitCount(), size_t(3)); - dr->getHit(0, docId, rank); - EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(89)); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - dr->getHit(1, docId, rank); - EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(109)); - EXPECT_EQUAL(strcmp("doc17", docId), 0); - dr->getHit(2, docId, rank); - EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(90)); - EXPECT_EQUAL(strcmp("doc18", docId), 0); - - sr2.sort(); - const void *buf; - size_t sz; - sr2.getHit(0, docId, rank); - sr2.getSortBlob(0, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata1", buf, sz), 0); - EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(109)); - EXPECT_EQUAL(strcmp("doc17", docId), 0); - sr2.getHit(1, docId, rank); - sr2.getSortBlob(1, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata2", buf, sz), 0); - EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(89)); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - sr2.getHit(2, docId, rank); - sr2.getSortBlob(2, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata3", buf, sz), 0); - EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(90)); - EXPECT_EQUAL(strcmp("doc18", docId), 0); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 116u, serialize("QueryResultMessage-5", srm2)); - routable = deserialize("QueryResultMessage-5", DocumentProtocol::MESSAGE_QUERYRESULT, LANG_CPP); - if (!EXPECT_TRUE(routable.get() != NULL)) { - return false; - } - dm = static_cast<QueryResultMessage *>(routable.get()); - dr = &dm->getSearchResult(); - EXPECT_EQUAL(dr->getHitCount(), size_t(3)); - dr->getHit(0, docId, rank); - dr->getSortBlob(0, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata1", buf, sz), 0); - EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(109)); - EXPECT_EQUAL(strcmp("doc17", docId), 0); - dr->getHit(1, docId, rank); - dr->getSortBlob(1, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata2", buf, sz), 0); - EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(89)); - EXPECT_EQUAL(strcmp("doc1", docId), 0); - dr->getHit(2, docId, rank); - dr->getSortBlob(2, buf, sz); - EXPECT_EQUAL(sz, 9u); - EXPECT_EQUAL(memcmp("sortdata3", buf, sz), 0); - EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(90)); - EXPECT_EQUAL(strcmp("doc18", docId), 0); - return true; -} - -bool -Messages52Test::testQueryResultReply() -{ - return tryVisitorReply("QueryResultReply", DocumentProtocol::REPLY_QUERYRESULT); -} - -bool -Messages52Test::testVisitorInfoMessage() -{ - - VisitorInfoMessage tmp; - tmp.getFinishedBuckets().push_back(document::BucketId(16, 1)); - tmp.getFinishedBuckets().push_back(document::BucketId(16, 2)); - tmp.getFinishedBuckets().push_back(document::BucketId(16, 4)); - string utf8 = "error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6"; - tmp.setErrorMessage(utf8); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 67u, serialize("VisitorInfoMessage", tmp)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("VisitorInfoMessage", DocumentProtocol::MESSAGE_VISITORINFO, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - VisitorInfoMessage &ref = static_cast<VisitorInfoMessage&>(*obj); - EXPECT_EQUAL(document::BucketId(16, 1), ref.getFinishedBuckets()[0]); - EXPECT_EQUAL(document::BucketId(16, 2), ref.getFinishedBuckets()[1]); - EXPECT_EQUAL(document::BucketId(16, 4), ref.getFinishedBuckets()[2]); - EXPECT_EQUAL(utf8, ref.getErrorMessage()); - } - } - return true; -} - -bool -Messages52Test::testDestroyVisitorReply() -{ - return tryDocumentReply("DestroyVisitorReply", DocumentProtocol::REPLY_DESTROYVISITOR); -} - -bool -Messages52Test::testDocumentIgnoredReply() -{ - DocumentIgnoredReply tmp; - serialize("DocumentIgnoredReply", tmp); - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj( - deserialize("DocumentIgnoredReply", - DocumentProtocol::REPLY_DOCUMENTIGNORED, lang)); - EXPECT_TRUE(obj.get() != NULL); - } - return true; -} - -bool -Messages52Test::testDocumentListReply() -{ - return tryVisitorReply("DocumentListReply", DocumentProtocol::REPLY_DOCUMENTLIST); -} - -bool -Messages52Test::testDocumentSummaryReply() -{ - return tryVisitorReply("DocumentSummaryReply", DocumentProtocol::REPLY_DOCUMENTSUMMARY); -} - -bool -Messages52Test::testGetDocumentReply() -{ - document::Document::SP doc = - createDoc(getTypeRepo(), "testdoc", "doc:scheme:"); - GetDocumentReply tmp(doc); - - EXPECT_EQUAL((size_t)43, serialize("GetDocumentReply", tmp)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("GetDocumentReply", DocumentProtocol::REPLY_GETDOCUMENT, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - GetDocumentReply &ref = static_cast<GetDocumentReply&>(*obj); - - EXPECT_EQUAL(string("testdoc"), ref.getDocument().getType().getName()); - EXPECT_EQUAL(string("doc:scheme:"), ref.getDocument().getId().toString()); - } - } - return true; -} - -bool -Messages52Test::testMapVisitorReply() -{ - return tryVisitorReply("MapVisitorReply", DocumentProtocol::REPLY_MAPVISITOR); -} - -bool -Messages52Test::testSearchResultReply() -{ - return tryVisitorReply("SearchResultReply", DocumentProtocol::REPLY_SEARCHRESULT); -} - -bool -Messages52Test::testStatBucketReply() -{ - StatBucketReply msg; - msg.setResults("These are the votes of the Norwegian jury"); - - EXPECT_EQUAL(50u, serialize("StatBucketReply", msg)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("StatBucketReply", DocumentProtocol::REPLY_STATBUCKET, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - StatBucketReply &ref = static_cast<StatBucketReply&>(*obj); - EXPECT_EQUAL("These are the votes of the Norwegian jury", ref.getResults()); - } - } - return true; -} - -bool -Messages52Test::testVisitorInfoReply() -{ - return tryVisitorReply("VisitorInfoReply", DocumentProtocol::REPLY_VISITORINFO); -} - -bool -Messages52Test::testWrongDistributionReply() -{ - WrongDistributionReply tmp("distributor:3 storage:2"); - - serialize("WrongDistributionReply", tmp); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("WrongDistributionReply", DocumentProtocol::REPLY_WRONGDISTRIBUTION, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - WrongDistributionReply &ref = static_cast<WrongDistributionReply&>(*obj); - EXPECT_EQUAL(string("distributor:3 storage:2"), ref.getSystemState()); - } - } - return true; -} - -bool -Messages52Test::testGetBucketListReply() -{ - GetBucketListReply reply; - reply.getBuckets().push_back(GetBucketListReply::BucketInfo(document::BucketId(16, 123), "foo")); - reply.getBuckets().push_back(GetBucketListReply::BucketInfo(document::BucketId(17, 1123), "bar")); - reply.getBuckets().push_back(GetBucketListReply::BucketInfo(document::BucketId(18, 11123), "zoink")); - - EXPECT_EQUAL(56u, serialize("GetBucketListReply", reply)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("GetBucketListReply", DocumentProtocol::REPLY_GETBUCKETLIST, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - GetBucketListReply &ref = static_cast<GetBucketListReply&>(*obj); - - EXPECT_EQUAL(ref.getBuckets()[0], GetBucketListReply::BucketInfo(document::BucketId(16, 123), "foo")); - EXPECT_EQUAL(ref.getBuckets()[1], GetBucketListReply::BucketInfo(document::BucketId(17, 1123), "bar")); - EXPECT_EQUAL(ref.getBuckets()[2], GetBucketListReply::BucketInfo(document::BucketId(18, 11123), "zoink")); - } - } - return true; -} - -bool -Messages52Test::testGetBucketStateReply() -{ - document::GlobalId foo = document::DocumentId("doc:scheme:foo").getGlobalId(); - document::GlobalId bar = document::DocumentId("doc:scheme:bar").getGlobalId(); - - GetBucketStateReply reply; - reply.getBucketState().push_back(DocumentState(foo, 777, false)); - reply.getBucketState().push_back(DocumentState(bar, 888, true)); - EXPECT_EQUAL(53u, serialize("GetBucketStateReply", reply)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("GetBucketStateReply", DocumentProtocol::REPLY_GETBUCKETSTATE, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - GetBucketStateReply &ref = static_cast<GetBucketStateReply&>(*obj); - - EXPECT_EQUAL(777u, ref.getBucketState()[0].getTimestamp()); - EXPECT_EQUAL(foo, ref.getBucketState()[0].getGlobalId()); - EXPECT_EQUAL(false, ref.getBucketState()[0].isRemoveEntry()); - EXPECT_EQUAL(888u, ref.getBucketState()[1].getTimestamp()); - EXPECT_EQUAL(bar, ref.getBucketState()[1].getGlobalId()); - EXPECT_EQUAL(true, ref.getBucketState()[1].isRemoveEntry()); - } - } - return true; -} - -bool -Messages52Test::testEmptyBucketsReply() -{ - return tryVisitorReply("EmptyBucketsReply", DocumentProtocol::REPLY_EMPTYBUCKETS); -} - -bool -Messages52Test::testRemoveLocationReply() -{ - DocumentReply tmp(DocumentProtocol::REPLY_REMOVELOCATION); - - EXPECT_EQUAL((uint32_t)5, serialize("RemoveLocationReply", tmp)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("RemoveLocationReply", DocumentProtocol::REPLY_REMOVELOCATION, lang); - EXPECT_TRUE(obj.get() != NULL); - } - return true; -} - - - -//////////////////////////////////////////////////////////////////////////////// -// -// Utilities -// -//////////////////////////////////////////////////////////////////////////////// - -bool -Messages52Test::tryDocumentReply(const string &filename, uint32_t type) -{ - DocumentReply tmp(type); - - EXPECT_EQUAL((uint32_t)5, serialize(filename, tmp)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize(filename, type, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - DocumentReply *ref = dynamic_cast<DocumentReply*>(obj.get()); - EXPECT_TRUE(ref != NULL); - } - } - return true; -} - -bool -Messages52Test::tryVisitorReply(const string &filename, uint32_t type) -{ - VisitorReply tmp(type); - - EXPECT_EQUAL((uint32_t)5, serialize(filename, tmp)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize(filename, type, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - VisitorReply *ref = dynamic_cast<VisitorReply*>(obj.get()); - EXPECT_TRUE(ref != NULL); - } - } - return true; -} diff --git a/documentapi/src/tests/messages/messages52test.h b/documentapi/src/tests/messages/messages52test.h deleted file mode 100644 index 6b019d77bde..00000000000 --- a/documentapi/src/tests/messages/messages52test.h +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// @author Vegard Sjonfjell -#pragma once - -#include "testbase.h" - -class Messages52Test : public TestBase { -protected: - const vespalib::Version getVersion() const override { return vespalib::Version(5, 115); } - bool shouldTestCoverage() const override { return true; } - bool tryDocumentReply(const string &filename, uint32_t type); - bool tryVisitorReply(const string &filename, uint32_t type); - - static size_t serializedLength(const string & str) { return sizeof(int32_t) + str.size(); } - -public: - Messages52Test(); - - bool testCreateVisitorMessage(); - bool testCreateVisitorReply(); - bool testDestroyVisitorMessage(); - bool testDestroyVisitorReply(); - bool testDocumentIgnoredReply(); - bool testDocumentListMessage(); - bool testDocumentListReply(); - bool testDocumentSummaryMessage(); - bool testDocumentSummaryReply(); - bool testEmptyBucketsMessage(); - bool testEmptyBucketsReply(); - bool testGetBucketListMessage(); - bool testGetBucketListReply(); - bool testGetBucketStateMessage(); - bool testGetBucketStateReply(); - bool testGetDocumentMessage(); - bool testGetDocumentReply(); - bool testMapVisitorMessage(); - bool testMapVisitorReply(); - bool testPutDocumentMessage(); - bool testPutDocumentReply(); - bool testQueryResultMessage(); - bool testQueryResultReply(); - bool testRemoveDocumentMessage(); - bool testRemoveDocumentReply(); - bool testRemoveLocationMessage(); - bool testRemoveLocationReply(); - bool testSearchResultMessage(); - bool testSearchResultReply(); - bool testStatBucketMessage(); - bool testStatBucketReply(); - bool testUpdateDocumentMessage(); - bool testUpdateDocumentReply(); - bool testVisitorInfoMessage(); - bool testVisitorInfoReply(); - bool testWrongDistributionReply(); -}; - diff --git a/documentapi/src/tests/messages/messages60test.cpp b/documentapi/src/tests/messages/messages60test.cpp index e553744452b..2fdb4c12fd0 100644 --- a/documentapi/src/tests/messages/messages60test.cpp +++ b/documentapi/src/tests/messages/messages60test.cpp @@ -1,20 +1,159 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// @author Vegard Sjonfjell #include "messages60test.h" +#include <vespa/document/bucket/bucketidfactory.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/select/parser.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/update/documentupdate.h> +#include <vespa/document/update/fieldpathupdates.h> #include <vespa/documentapi/documentapi.h> +#include <vespa/document/bucket/fixed_bucket_spaces.h> -static constexpr int MESSAGE_BASE_LENGTH = 5; +using document::DataType; +using document::DocumentTypeRepo; -Messages60Test::Messages60Test() { +/////////////////////////////////////////////////////////////////////////////// +// +// Setup +// +/////////////////////////////////////////////////////////////////////////////// + +Messages60Test::Messages60Test() +{ // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support - // version 6.x. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now. + // version 5.0. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now. putTest(DocumentProtocol::MESSAGE_CREATEVISITOR, TEST_METHOD(Messages60Test::testCreateVisitorMessage)); - putTest(DocumentProtocol::MESSAGE_STATBUCKET, TEST_METHOD(Messages60Test::testStatBucketMessage)); + putTest(DocumentProtocol::MESSAGE_DESTROYVISITOR, TEST_METHOD(Messages60Test::testDestroyVisitorMessage)); + putTest(DocumentProtocol::MESSAGE_DOCUMENTLIST, TEST_METHOD(Messages60Test::testDocumentListMessage)); + putTest(DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, TEST_METHOD(Messages60Test::testDocumentSummaryMessage)); + putTest(DocumentProtocol::MESSAGE_EMPTYBUCKETS, TEST_METHOD(Messages60Test::testEmptyBucketsMessage)); putTest(DocumentProtocol::MESSAGE_GETBUCKETLIST, TEST_METHOD(Messages60Test::testGetBucketListMessage)); + putTest(DocumentProtocol::MESSAGE_GETBUCKETSTATE, TEST_METHOD(Messages60Test::testGetBucketStateMessage)); + putTest(DocumentProtocol::MESSAGE_GETDOCUMENT, TEST_METHOD(Messages60Test::testGetDocumentMessage)); + putTest(DocumentProtocol::MESSAGE_MAPVISITOR, TEST_METHOD(Messages60Test::testMapVisitorMessage)); + putTest(DocumentProtocol::MESSAGE_PUTDOCUMENT, TEST_METHOD(Messages60Test::testPutDocumentMessage)); + putTest(DocumentProtocol::MESSAGE_QUERYRESULT, TEST_METHOD(Messages60Test::testQueryResultMessage)); + putTest(DocumentProtocol::MESSAGE_REMOVEDOCUMENT, TEST_METHOD(Messages60Test::testRemoveDocumentMessage)); + putTest(DocumentProtocol::MESSAGE_REMOVELOCATION, TEST_METHOD(Messages60Test::testRemoveLocationMessage)); + putTest(DocumentProtocol::MESSAGE_SEARCHRESULT, TEST_METHOD(Messages60Test::testSearchResultMessage)); + putTest(DocumentProtocol::MESSAGE_STATBUCKET, TEST_METHOD(Messages60Test::testStatBucketMessage)); + putTest(DocumentProtocol::MESSAGE_UPDATEDOCUMENT, TEST_METHOD(Messages60Test::testUpdateDocumentMessage)); + putTest(DocumentProtocol::MESSAGE_VISITORINFO, TEST_METHOD(Messages60Test::testVisitorInfoMessage)); + + putTest(DocumentProtocol::REPLY_CREATEVISITOR, TEST_METHOD(Messages60Test::testCreateVisitorReply)); + putTest(DocumentProtocol::REPLY_DESTROYVISITOR, TEST_METHOD(Messages60Test::testDestroyVisitorReply)); + putTest(DocumentProtocol::REPLY_DOCUMENTIGNORED, TEST_METHOD(Messages60Test::testDocumentIgnoredReply)); + putTest(DocumentProtocol::REPLY_DOCUMENTLIST, TEST_METHOD(Messages60Test::testDocumentListReply)); + putTest(DocumentProtocol::REPLY_DOCUMENTSUMMARY, TEST_METHOD(Messages60Test::testDocumentSummaryReply)); + putTest(DocumentProtocol::REPLY_EMPTYBUCKETS, TEST_METHOD(Messages60Test::testEmptyBucketsReply)); + putTest(DocumentProtocol::REPLY_GETBUCKETLIST, TEST_METHOD(Messages60Test::testGetBucketListReply)); + putTest(DocumentProtocol::REPLY_GETBUCKETSTATE, TEST_METHOD(Messages60Test::testGetBucketStateReply)); + putTest(DocumentProtocol::REPLY_GETDOCUMENT, TEST_METHOD(Messages60Test::testGetDocumentReply)); + putTest(DocumentProtocol::REPLY_MAPVISITOR, TEST_METHOD(Messages60Test::testMapVisitorReply)); + putTest(DocumentProtocol::REPLY_PUTDOCUMENT, TEST_METHOD(Messages60Test::testPutDocumentReply)); + putTest(DocumentProtocol::REPLY_QUERYRESULT, TEST_METHOD(Messages60Test::testQueryResultReply)); + putTest(DocumentProtocol::REPLY_REMOVEDOCUMENT, TEST_METHOD(Messages60Test::testRemoveDocumentReply)); + putTest(DocumentProtocol::REPLY_REMOVELOCATION, TEST_METHOD(Messages60Test::testRemoveLocationReply)); + putTest(DocumentProtocol::REPLY_SEARCHRESULT, TEST_METHOD(Messages60Test::testSearchResultReply)); + putTest(DocumentProtocol::REPLY_STATBUCKET, TEST_METHOD(Messages60Test::testStatBucketReply)); + putTest(DocumentProtocol::REPLY_UPDATEDOCUMENT, TEST_METHOD(Messages60Test::testUpdateDocumentReply)); + putTest(DocumentProtocol::REPLY_VISITORINFO, TEST_METHOD(Messages60Test::testVisitorInfoReply)); + putTest(DocumentProtocol::REPLY_WRONGDISTRIBUTION, TEST_METHOD(Messages60Test::testWrongDistributionReply)); +} + + + +/////////////////////////////////////////////////////////////////////////////// +// +// Tests +// +/////////////////////////////////////////////////////////////////////////////// + +static const int MESSAGE_BASE_LENGTH = 5; + +namespace { + +document::Document::SP +createDoc(const DocumentTypeRepo &repo, const string &type_name, const string &id) +{ + return document::Document::SP(new document::Document( + *repo.getDocumentType(type_name), + document::DocumentId(id))); +} + +} // namespace + +bool +Messages60Test::testGetBucketListMessage() +{ + GetBucketListMessage msg(document::BucketId(16, 123)); + msg.setLoadType(_loadTypes["foo"]); + msg.setBucketSpace("beartato"); + EXPECT_EQUAL(string("foo"), msg.getLoadType().getName()); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 12u + serializedLength("beartato"), serialize("GetBucketListMessage", msg)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("GetBucketListMessage", DocumentProtocol::MESSAGE_GETBUCKETLIST, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + GetBucketListMessage &ref = static_cast<GetBucketListMessage&>(*obj); + EXPECT_EQUAL(string("foo"), ref.getLoadType().getName()); + EXPECT_EQUAL(document::BucketId(16, 123), ref.getBucketId()); + EXPECT_EQUAL("beartato", ref.getBucketSpace()); + } + } + return true; +} + +bool +Messages60Test::testEmptyBucketsMessage() +{ + std::vector<document::BucketId> bids; + for (size_t i=0; i < 13; ++i) { + bids.push_back(document::BucketId(16, i)); + } + + EmptyBucketsMessage msg(bids); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 112u, serialize("EmptyBucketsMessage", msg)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("EmptyBucketsMessage", DocumentProtocol::MESSAGE_EMPTYBUCKETS, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + EmptyBucketsMessage &ref = static_cast<EmptyBucketsMessage&>(*obj); + for (size_t i=0; i < 13; ++i) { + EXPECT_EQUAL(document::BucketId(16, i), ref.getBucketIds()[i]); + } + } + } + return true; } -// TODO code dupe with parent classes -bool Messages60Test::testCreateVisitorMessage() { + +bool +Messages60Test::testStatBucketMessage() +{ + StatBucketMessage msg(document::BucketId(16, 123), "id.user=123"); + msg.setBucketSpace("andrei"); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 27u + serializedLength("andrei"), serialize("StatBucketMessage", msg)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("StatBucketMessage", DocumentProtocol::MESSAGE_STATBUCKET, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + StatBucketMessage &ref = static_cast<StatBucketMessage&>(*obj); + EXPECT_EQUAL(document::BucketId(16, 123), ref.getBucketId()); + EXPECT_EQUAL("id.user=123", ref.getDocumentSelection()); + EXPECT_EQUAL("andrei", ref.getBucketSpace()); + } + } + return true; +} + +bool +Messages60Test::testCreateVisitorMessage() +{ CreateVisitorMessage tmp("SomeLibrary", "myvisitor", "newyork", "london"); tmp.setDocumentSelection("true and false or true"); tmp.getParameters().set("myvar", "somevalue"); @@ -41,7 +180,6 @@ bool Messages60Test::testCreateVisitorMessage() { EXPECT_EQUAL(string("foo bar"), ref.getFieldSet()); EXPECT_EQUAL(uint32_t(8), ref.getMaximumPendingReplyCount()); EXPECT_EQUAL(true, ref.visitRemoves()); - EXPECT_EQUAL(false, ref.visitHeadersOnly()); EXPECT_EQUAL(false, ref.visitInconsistentBuckets()); EXPECT_EQUAL(size_t(1), ref.getBuckets().size()); EXPECT_EQUAL(document::BucketId(16, 1234), ref.getBuckets()[0]); @@ -55,39 +193,875 @@ bool Messages60Test::testCreateVisitorMessage() { return true; } -bool Messages60Test::testStatBucketMessage() { - StatBucketMessage msg(document::BucketId(16, 123), "id.user=123"); - msg.setBucketSpace("andrei"); +bool +Messages60Test::testDestroyVisitorMessage() +{ + DestroyVisitorMessage tmp("myvisitor"); - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 27u + serializedLength("andrei"), serialize("StatBucketMessage", msg)); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + (size_t)17, serialize("DestroyVisitorMessage", tmp)); for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("StatBucketMessage", DocumentProtocol::MESSAGE_STATBUCKET, lang); + mbus::Routable::UP obj = deserialize("DestroyVisitorMessage", DocumentProtocol::MESSAGE_DESTROYVISITOR, lang); if (EXPECT_TRUE(obj.get() != NULL)) { - StatBucketMessage &ref = static_cast<StatBucketMessage&>(*obj); - EXPECT_EQUAL(document::BucketId(16, 123), ref.getBucketId()); - EXPECT_EQUAL("id.user=123", ref.getDocumentSelection()); - EXPECT_EQUAL("andrei", ref.getBucketSpace()); + DestroyVisitorMessage &ref = static_cast<DestroyVisitorMessage&>(*obj); + EXPECT_EQUAL(string("myvisitor"), ref.getInstanceId()); } } return true; } -bool Messages60Test::testGetBucketListMessage() { - GetBucketListMessage msg(document::BucketId(16, 123)); - msg.setLoadType(_loadTypes["foo"]); - msg.setBucketSpace("beartato"); - EXPECT_EQUAL(string("foo"), msg.getLoadType().getName()); - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 12u + serializedLength("beartato"), serialize("GetBucketListMessage", msg)); +bool +Messages60Test::testDocumentListMessage() +{ + document::Document::SP doc = + createDoc(getTypeRepo(), "testdoc", "userdoc:scheme:1234:"); + DocumentListMessage::Entry entry(1234, doc, false); + + DocumentListMessage tmp(document::BucketId(16, 1234)); + tmp.getDocuments().push_back(entry); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + (size_t)63, serialize("DocumentListMessage", tmp)); for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("GetBucketListMessage", DocumentProtocol::MESSAGE_GETBUCKETLIST, lang); + mbus::Routable::UP obj = deserialize("DocumentListMessage", DocumentProtocol::MESSAGE_DOCUMENTLIST, lang); if (EXPECT_TRUE(obj.get() != NULL)) { - GetBucketListMessage &ref = static_cast<GetBucketListMessage&>(*obj); - EXPECT_EQUAL(string("foo"), ref.getLoadType().getName()); - EXPECT_EQUAL(document::BucketId(16, 123), ref.getBucketId()); - EXPECT_EQUAL("beartato", ref.getBucketSpace()); + DocumentListMessage &ref = static_cast<DocumentListMessage&>(*obj); + + EXPECT_EQUAL("userdoc:scheme:1234:", ref.getDocuments()[0].getDocument()->getId().toString()); + EXPECT_EQUAL(1234, ref.getDocuments()[0].getTimestamp()); + EXPECT_TRUE(!ref.getDocuments()[0].isRemoveEntry()); + } + } + return true; +} + + +bool +Messages60Test::testRemoveLocationMessage() +{ + { + document::BucketIdFactory factory; + document::select::Parser parser(getTypeRepo(), factory); + RemoveLocationMessage msg(factory, parser, "id.group == \"mygroup\""); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 29u, serialize("RemoveLocationMessage", msg)); + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("RemoveLocationMessage", DocumentProtocol::MESSAGE_REMOVELOCATION, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + RemoveLocationMessage &ref = static_cast<RemoveLocationMessage&>(*obj); + EXPECT_EQUAL(string("id.group == \"mygroup\""), ref.getDocumentSelection()); + // FIXME add to wire format, currently hardcoded. + EXPECT_EQUAL(string(document::FixedBucketSpaces::default_space_name()), ref.getBucketSpace()); + } + } + } + + return true; +} + + + +bool +Messages60Test::testDocumentSummaryMessage() +{ + DocumentSummaryMessage srm; + EXPECT_EQUAL(srm.hasSequenceId(), false); + EXPECT_EQUAL(srm.getSummaryCount(), size_t(0)); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(12), serialize("DocumentSummaryMessage-1", srm)); + + mbus::Routable::UP routable = deserialize("DocumentSummaryMessage-1", DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, LANG_CPP); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + DocumentSummaryMessage * dm = static_cast<DocumentSummaryMessage *>(routable.get()); + EXPECT_EQUAL(dm->getSummaryCount(), size_t(0)); + + srm.addSummary("doc1", "summary1", 8); + srm.addSummary("aoc17", "summary45", 9); + + const void *summary(NULL); + const char *docId(NULL); + size_t sz(0); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 52u, serialize("DocumentSummaryMessage-2", srm)); + routable = deserialize("DocumentSummaryMessage-2", DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, LANG_CPP); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + dm = static_cast<DocumentSummaryMessage *>(routable.get()); + EXPECT_EQUAL(dm->getSummaryCount(), size_t(2)); + dm->getSummary(0, docId, summary, sz); + EXPECT_EQUAL(sz, 8u); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + EXPECT_EQUAL(memcmp("summary1", summary, sz), 0); + dm->getSummary(1, docId, summary, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(strcmp("aoc17", docId), 0); + EXPECT_EQUAL(memcmp("summary45", summary, sz), 0); + + srm.sort(); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 52u, serialize("DocumentSummaryMessage-3", srm)); + routable = deserialize("DocumentSummaryMessage-3", DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, LANG_CPP); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + dm = static_cast<DocumentSummaryMessage *>(routable.get()); + EXPECT_EQUAL(dm->getSummaryCount(), size_t(2)); + dm->getSummary(0, docId, summary, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(strcmp("aoc17", docId), 0); + EXPECT_EQUAL(memcmp("summary45", summary, sz), 0); + dm->getSummary(1, docId, summary, sz); + EXPECT_EQUAL(sz, 8u); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + EXPECT_EQUAL(memcmp("summary1", summary, sz), 0); + return true; +} + +bool +Messages60Test::testGetDocumentMessage() +{ + GetDocumentMessage tmp(document::DocumentId("doc:scheme:"), "foo bar"); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + (size_t)27, serialize("GetDocumentMessage", tmp)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("GetDocumentMessage", DocumentProtocol::MESSAGE_GETDOCUMENT, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + GetDocumentMessage &ref = static_cast<GetDocumentMessage&>(*obj); + EXPECT_EQUAL(string("doc:scheme:"), ref.getDocumentId().toString()); + EXPECT_EQUAL(string("foo bar"), ref.getFieldSet()); + } + } + return true; +} + +bool +Messages60Test::testMapVisitorMessage() +{ + MapVisitorMessage tmp; + tmp.getData().set("foo", 3); + tmp.getData().set("bar", 5); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + (size_t)32, serialize("MapVisitorMessage", tmp)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("MapVisitorMessage", DocumentProtocol::MESSAGE_MAPVISITOR, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + MapVisitorMessage &ref = static_cast<MapVisitorMessage&>(*obj); + EXPECT_EQUAL(3, ref.getData().get("foo", 0)); + EXPECT_EQUAL(5, ref.getData().get("bar", 0)); + } + } + return true; +} + +bool +Messages60Test::testCreateVisitorReply() +{ + CreateVisitorReply reply(DocumentProtocol::REPLY_CREATEVISITOR); + reply.setLastBucket(document::BucketId(16, 123)); + vdslib::VisitorStatistics vs; + vs.setBucketsVisited(3); + vs.setDocumentsVisited(1000); + vs.setBytesVisited(1024000); + vs.setDocumentsReturned(123); + vs.setBytesReturned(512000); + vs.setSecondPassDocumentsReturned(456); + vs.setSecondPassBytesReturned(789100); + reply.setVisitorStatistics(vs); + + EXPECT_EQUAL(65u, serialize("CreateVisitorReply", reply)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("CreateVisitorReply", DocumentProtocol::REPLY_CREATEVISITOR, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + CreateVisitorReply &ref = static_cast<CreateVisitorReply&>(*obj); + + EXPECT_EQUAL(ref.getLastBucket(), document::BucketId(16, 123)); + EXPECT_EQUAL(ref.getVisitorStatistics().getBucketsVisited(), (uint32_t)3); + EXPECT_EQUAL(ref.getVisitorStatistics().getDocumentsVisited(), (uint64_t)1000); + EXPECT_EQUAL(ref.getVisitorStatistics().getBytesVisited(), (uint64_t)1024000); + EXPECT_EQUAL(ref.getVisitorStatistics().getDocumentsReturned(), (uint64_t)123); + EXPECT_EQUAL(ref.getVisitorStatistics().getBytesReturned(), (uint64_t)512000); + EXPECT_EQUAL(ref.getVisitorStatistics().getSecondPassDocumentsReturned(), (uint64_t)456); + EXPECT_EQUAL(ref.getVisitorStatistics().getSecondPassBytesReturned(), (uint64_t)789100); + } + } + return true; +} + +bool +Messages60Test::testPutDocumentMessage() +{ + auto doc = createDoc(getTypeRepo(), "testdoc", "doc:scheme:"); + PutDocumentMessage msg(doc); + + msg.setTimestamp(666); + msg.setCondition(TestAndSetCondition("There's just one condition")); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + + 41u + + serializedLength(msg.getCondition().getSelection()), + serialize("PutDocumentMessage", msg)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + auto routableUp = deserialize("PutDocumentMessage", DocumentProtocol::MESSAGE_PUTDOCUMENT, lang); + if (EXPECT_TRUE(routableUp.get() != nullptr)) { + auto & deserializedMsg = static_cast<PutDocumentMessage &>(*routableUp); + + EXPECT_EQUAL(msg.getDocument().getType().getName(), deserializedMsg.getDocument().getType().getName()); + EXPECT_EQUAL(msg.getDocument().getId().toString(), deserializedMsg.getDocument().getId().toString()); + EXPECT_EQUAL(msg.getTimestamp(), deserializedMsg.getTimestamp()); + EXPECT_EQUAL(67u, deserializedMsg.getApproxSize()); + EXPECT_EQUAL(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection()); + } + } + + return true; +} + +bool +Messages60Test::testGetBucketStateMessage() +{ + GetBucketStateMessage tmp; + tmp.setBucketId(document::BucketId(16, 666)); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 12u, serialize("GetBucketStateMessage", tmp)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("GetBucketStateMessage", DocumentProtocol::MESSAGE_GETBUCKETSTATE, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + GetBucketStateMessage &ref = static_cast<GetBucketStateMessage&>(*obj); + + EXPECT_EQUAL(16u, ref.getBucketId().getUsedBits()); + EXPECT_EQUAL(4611686018427388570ull, ref.getBucketId().getId()); + } + } + return true; +} + +bool +Messages60Test::testPutDocumentReply() +{ + WriteDocumentReply reply(DocumentProtocol::REPLY_PUTDOCUMENT); + reply.setHighestModificationTimestamp(30); + + EXPECT_EQUAL(13u, serialize("PutDocumentReply", reply)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("PutDocumentReply", DocumentProtocol::REPLY_PUTDOCUMENT, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + WriteDocumentReply &ref = static_cast<WriteDocumentReply&>(*obj); + EXPECT_EQUAL(30u, ref.getHighestModificationTimestamp()); + } + } + return true; +} + +bool +Messages60Test::testUpdateDocumentReply() +{ + UpdateDocumentReply reply; + reply.setWasFound(false); + reply.setHighestModificationTimestamp(30); + + EXPECT_EQUAL(14u, serialize("UpdateDocumentReply", reply)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("UpdateDocumentReply", DocumentProtocol::REPLY_UPDATEDOCUMENT, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + UpdateDocumentReply &ref = static_cast<UpdateDocumentReply&>(*obj); + EXPECT_EQUAL(30u, ref.getHighestModificationTimestamp()); + EXPECT_EQUAL(false, ref.wasFound()); + } + } + return true; +} + +bool +Messages60Test::testRemoveDocumentMessage() +{ + RemoveDocumentMessage msg(document::DocumentId("doc:scheme:")); + + msg.setCondition(TestAndSetCondition("There's just one condition")); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(16) + serializedLength(msg.getCondition().getSelection()), serialize("RemoveDocumentMessage", msg)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + auto routablePtr = deserialize("RemoveDocumentMessage", DocumentProtocol::MESSAGE_REMOVEDOCUMENT, lang); + + if (EXPECT_TRUE(routablePtr.get() != nullptr)) { + auto & ref = static_cast<RemoveDocumentMessage &>(*routablePtr); + EXPECT_EQUAL(string("doc:scheme:"), ref.getDocumentId().toString()); + EXPECT_EQUAL(msg.getCondition().getSelection(), ref.getCondition().getSelection()); + } + } + return true; +} + +bool +Messages60Test::testRemoveDocumentReply() +{ + RemoveDocumentReply reply; + std::vector<uint64_t> ts; + reply.setWasFound(false); + reply.setHighestModificationTimestamp(30); + + EXPECT_EQUAL(14u, serialize("RemoveDocumentReply", reply)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("RemoveDocumentReply", DocumentProtocol::REPLY_REMOVEDOCUMENT, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + RemoveDocumentReply &ref = static_cast<RemoveDocumentReply&>(*obj); + EXPECT_EQUAL(30u, ref.getHighestModificationTimestamp()); + EXPECT_EQUAL(false, ref.wasFound()); } } return true; -}
\ No newline at end of file +} + +bool +Messages60Test::testSearchResultMessage() +{ + SearchResultMessage srm; + EXPECT_EQUAL(srm.getSequenceId(), 0u); + EXPECT_EQUAL(srm.getHitCount(), 0u); + EXPECT_EQUAL(srm.getAggregatorList().getSerializedSize(), 4u); + EXPECT_EQUAL(srm.vdslib::SearchResult::getSerializedSize(), 20u); + EXPECT_EQUAL(srm.getSerializedSize(), 20u); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(24), serialize("SearchResultMessage-1", srm)); + + mbus::Routable::UP routable = deserialize("SearchResultMessage-1", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + SearchResultMessage * dm = static_cast<SearchResultMessage *>(routable.get()); + EXPECT_EQUAL(dm->getSequenceId(), size_t(0)); + EXPECT_EQUAL(dm->getHitCount(), size_t(0)); + + srm.addHit(0, "doc1", 89); + srm.addHit(1, "doc17", 109); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 55u, serialize("SearchResultMessage-2", srm)); + routable = deserialize("SearchResultMessage-2", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + dm = static_cast<SearchResultMessage *>(routable.get()); + EXPECT_EQUAL(dm->getHitCount(), size_t(2)); + const char *docId; + SearchResultMessage::RankType rank; + dm->getHit(0, docId, rank); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + dm->getHit(1, docId, rank); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); + EXPECT_EQUAL(strcmp("doc17", docId), 0); + + srm.sort(); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 55u, serialize("SearchResultMessage-3", srm)); + routable = deserialize("SearchResultMessage-3", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + dm = static_cast<SearchResultMessage *>(routable.get()); + EXPECT_EQUAL(dm->getHitCount(), size_t(2)); + dm->getHit(0, docId, rank); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); + EXPECT_EQUAL(strcmp("doc17", docId), 0); + dm->getHit(1, docId, rank); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + + SearchResultMessage srm2; + srm2.addHit(0, "doc1", 89, "sortdata2", 9); + srm2.addHit(1, "doc17", 109, "sortdata1", 9); + srm2.addHit(2, "doc18", 90, "sortdata3", 9); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 108u, serialize("SearchResultMessage-4", srm2)); + routable = deserialize("SearchResultMessage-4", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + dm = static_cast<SearchResultMessage *>(routable.get()); + EXPECT_EQUAL(dm->getHitCount(), size_t(3)); + dm->getHit(0, docId, rank); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + dm->getHit(1, docId, rank); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); + EXPECT_EQUAL(strcmp("doc17", docId), 0); + dm->getHit(2, docId, rank); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(90)); + EXPECT_EQUAL(strcmp("doc18", docId), 0); + + srm2.sort(); + const void *buf; + size_t sz; + srm2.getHit(0, docId, rank); + srm2.getSortBlob(0, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata1", buf, sz), 0); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); + EXPECT_EQUAL(strcmp("doc17", docId), 0); + srm2.getHit(1, docId, rank); + srm2.getSortBlob(1, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata2", buf, sz), 0); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + srm2.getHit(2, docId, rank); + srm2.getSortBlob(2, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata3", buf, sz), 0); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(90)); + EXPECT_EQUAL(strcmp("doc18", docId), 0); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 108u, serialize("SearchResultMessage-5", srm2)); + routable = deserialize("SearchResultMessage-5", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + dm = static_cast<SearchResultMessage *>(routable.get()); + EXPECT_EQUAL(dm->getHitCount(), size_t(3)); + dm->getHit(0, docId, rank); + dm->getSortBlob(0, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata1", buf, sz), 0); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); + EXPECT_EQUAL(strcmp("doc17", docId), 0); + dm->getHit(1, docId, rank); + dm->getSortBlob(1, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata2", buf, sz), 0); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + dm->getHit(2, docId, rank); + dm->getSortBlob(2, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata3", buf, sz), 0); + EXPECT_EQUAL(rank, SearchResultMessage::RankType(90)); + EXPECT_EQUAL(strcmp("doc18", docId), 0); + return true; +} + +bool +Messages60Test::testUpdateDocumentMessage() +{ + const DocumentTypeRepo & repo = getTypeRepo(); + const document::DocumentType & docType = *repo.getDocumentType("testdoc"); + + auto docUpdate = std::make_shared<document::DocumentUpdate>(repo, docType, document::DocumentId("doc:scheme:")); + + docUpdate->addFieldPathUpdate(document::FieldPathUpdate::CP( + new document::RemoveFieldPathUpdate("intfield", "testdoc.intfield > 0"))); + + UpdateDocumentMessage msg(docUpdate); + msg.setOldTimestamp(666u); + msg.setNewTimestamp(777u); + msg.setCondition(TestAndSetCondition("There's just one condition")); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 89u + serializedLength(msg.getCondition().getSelection()), serialize("UpdateDocumentMessage", msg)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + auto routableUp = deserialize("UpdateDocumentMessage", DocumentProtocol::MESSAGE_UPDATEDOCUMENT, lang); + + if (EXPECT_TRUE(routableUp.get() != nullptr)) { + auto & deserializedMsg = static_cast<UpdateDocumentMessage &>(*routableUp); + EXPECT_EQUAL(msg.getDocumentUpdate(), deserializedMsg.getDocumentUpdate()); + EXPECT_EQUAL(msg.getOldTimestamp(), deserializedMsg.getOldTimestamp()); + EXPECT_EQUAL(msg.getNewTimestamp(), deserializedMsg.getNewTimestamp()); + EXPECT_EQUAL(115u, deserializedMsg.getApproxSize()); + EXPECT_EQUAL(msg.getCondition().getSelection(), deserializedMsg.getCondition().getSelection()); + } + } + return true; +} + +bool +Messages60Test::testQueryResultMessage() +{ + QueryResultMessage srm; + vdslib::SearchResult & sr(srm.getSearchResult()); + EXPECT_EQUAL(srm.getSequenceId(), 0u); + EXPECT_EQUAL(sr.getHitCount(), 0u); + EXPECT_EQUAL(sr.getAggregatorList().getSerializedSize(), 4u); + EXPECT_EQUAL(sr.getSerializedSize(), 20u); + EXPECT_EQUAL(srm.getApproxSize(), 28u); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(32), serialize("QueryResultMessage-1", srm)); + + mbus::Routable::UP routable = deserialize("QueryResultMessage-1", DocumentProtocol::MESSAGE_QUERYRESULT, LANG_CPP); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + QueryResultMessage * dm = static_cast<QueryResultMessage *>(routable.get()); + vdslib::SearchResult * dr(&dm->getSearchResult()); + EXPECT_EQUAL(dm->getSequenceId(), size_t(0)); + EXPECT_EQUAL(dr->getHitCount(), size_t(0)); + + sr.addHit(0, "doc1", 89); + sr.addHit(1, "doc17", 109); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 63u, serialize("QueryResultMessage-2", srm)); + routable = deserialize("QueryResultMessage-2", DocumentProtocol::MESSAGE_QUERYRESULT, LANG_CPP); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + dm = static_cast<QueryResultMessage *>(routable.get()); + dr = &dm->getSearchResult(); + EXPECT_EQUAL(dr->getHitCount(), size_t(2)); + const char *docId; + vdslib::SearchResult::RankType rank; + dr->getHit(0, docId, rank); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(89)); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + dr->getHit(1, docId, rank); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(109)); + EXPECT_EQUAL(strcmp("doc17", docId), 0); + + sr.sort(); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 63u, serialize("QueryResultMessage-3", srm)); + routable = deserialize("QueryResultMessage-3", DocumentProtocol::MESSAGE_QUERYRESULT, LANG_CPP); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + dm = static_cast<QueryResultMessage *>(routable.get()); + dr = &dm->getSearchResult(); + EXPECT_EQUAL(dr->getHitCount(), size_t(2)); + dr->getHit(0, docId, rank); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(109)); + EXPECT_EQUAL(strcmp("doc17", docId), 0); + dr->getHit(1, docId, rank); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(89)); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + + QueryResultMessage srm2; + vdslib::SearchResult & sr2(srm2.getSearchResult()); + sr2.addHit(0, "doc1", 89, "sortdata2", 9); + sr2.addHit(1, "doc17", 109, "sortdata1", 9); + sr2.addHit(2, "doc18", 90, "sortdata3", 9); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 116u, serialize("QueryResultMessage-4", srm2)); + routable = deserialize("QueryResultMessage-4", DocumentProtocol::MESSAGE_QUERYRESULT, LANG_CPP); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + dm = static_cast<QueryResultMessage *>(routable.get()); + dr = &dm->getSearchResult(); + EXPECT_EQUAL(dr->getHitCount(), size_t(3)); + dr->getHit(0, docId, rank); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(89)); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + dr->getHit(1, docId, rank); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(109)); + EXPECT_EQUAL(strcmp("doc17", docId), 0); + dr->getHit(2, docId, rank); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(90)); + EXPECT_EQUAL(strcmp("doc18", docId), 0); + + sr2.sort(); + const void *buf; + size_t sz; + sr2.getHit(0, docId, rank); + sr2.getSortBlob(0, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata1", buf, sz), 0); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(109)); + EXPECT_EQUAL(strcmp("doc17", docId), 0); + sr2.getHit(1, docId, rank); + sr2.getSortBlob(1, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata2", buf, sz), 0); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(89)); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + sr2.getHit(2, docId, rank); + sr2.getSortBlob(2, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata3", buf, sz), 0); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(90)); + EXPECT_EQUAL(strcmp("doc18", docId), 0); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 116u, serialize("QueryResultMessage-5", srm2)); + routable = deserialize("QueryResultMessage-5", DocumentProtocol::MESSAGE_QUERYRESULT, LANG_CPP); + if (!EXPECT_TRUE(routable.get() != NULL)) { + return false; + } + dm = static_cast<QueryResultMessage *>(routable.get()); + dr = &dm->getSearchResult(); + EXPECT_EQUAL(dr->getHitCount(), size_t(3)); + dr->getHit(0, docId, rank); + dr->getSortBlob(0, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata1", buf, sz), 0); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(109)); + EXPECT_EQUAL(strcmp("doc17", docId), 0); + dr->getHit(1, docId, rank); + dr->getSortBlob(1, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata2", buf, sz), 0); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(89)); + EXPECT_EQUAL(strcmp("doc1", docId), 0); + dr->getHit(2, docId, rank); + dr->getSortBlob(2, buf, sz); + EXPECT_EQUAL(sz, 9u); + EXPECT_EQUAL(memcmp("sortdata3", buf, sz), 0); + EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(90)); + EXPECT_EQUAL(strcmp("doc18", docId), 0); + return true; +} + +bool +Messages60Test::testQueryResultReply() +{ + return tryVisitorReply("QueryResultReply", DocumentProtocol::REPLY_QUERYRESULT); +} + +bool +Messages60Test::testVisitorInfoMessage() +{ + + VisitorInfoMessage tmp; + tmp.getFinishedBuckets().push_back(document::BucketId(16, 1)); + tmp.getFinishedBuckets().push_back(document::BucketId(16, 2)); + tmp.getFinishedBuckets().push_back(document::BucketId(16, 4)); + string utf8 = "error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6"; + tmp.setErrorMessage(utf8); + + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 67u, serialize("VisitorInfoMessage", tmp)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("VisitorInfoMessage", DocumentProtocol::MESSAGE_VISITORINFO, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + VisitorInfoMessage &ref = static_cast<VisitorInfoMessage&>(*obj); + EXPECT_EQUAL(document::BucketId(16, 1), ref.getFinishedBuckets()[0]); + EXPECT_EQUAL(document::BucketId(16, 2), ref.getFinishedBuckets()[1]); + EXPECT_EQUAL(document::BucketId(16, 4), ref.getFinishedBuckets()[2]); + EXPECT_EQUAL(utf8, ref.getErrorMessage()); + } + } + return true; +} + +bool +Messages60Test::testDestroyVisitorReply() +{ + return tryDocumentReply("DestroyVisitorReply", DocumentProtocol::REPLY_DESTROYVISITOR); +} + +bool +Messages60Test::testDocumentIgnoredReply() +{ + DocumentIgnoredReply tmp; + serialize("DocumentIgnoredReply", tmp); + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj( + deserialize("DocumentIgnoredReply", + DocumentProtocol::REPLY_DOCUMENTIGNORED, lang)); + EXPECT_TRUE(obj.get() != NULL); + } + return true; +} + +bool +Messages60Test::testDocumentListReply() +{ + return tryVisitorReply("DocumentListReply", DocumentProtocol::REPLY_DOCUMENTLIST); +} + +bool +Messages60Test::testDocumentSummaryReply() +{ + return tryVisitorReply("DocumentSummaryReply", DocumentProtocol::REPLY_DOCUMENTSUMMARY); +} + +bool +Messages60Test::testGetDocumentReply() +{ + document::Document::SP doc = + createDoc(getTypeRepo(), "testdoc", "doc:scheme:"); + GetDocumentReply tmp(doc); + + EXPECT_EQUAL((size_t)43, serialize("GetDocumentReply", tmp)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("GetDocumentReply", DocumentProtocol::REPLY_GETDOCUMENT, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + GetDocumentReply &ref = static_cast<GetDocumentReply&>(*obj); + + EXPECT_EQUAL(string("testdoc"), ref.getDocument().getType().getName()); + EXPECT_EQUAL(string("doc:scheme:"), ref.getDocument().getId().toString()); + } + } + return true; +} + +bool +Messages60Test::testMapVisitorReply() +{ + return tryVisitorReply("MapVisitorReply", DocumentProtocol::REPLY_MAPVISITOR); +} + +bool +Messages60Test::testSearchResultReply() +{ + return tryVisitorReply("SearchResultReply", DocumentProtocol::REPLY_SEARCHRESULT); +} + +bool +Messages60Test::testStatBucketReply() +{ + StatBucketReply msg; + msg.setResults("These are the votes of the Norwegian jury"); + + EXPECT_EQUAL(50u, serialize("StatBucketReply", msg)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("StatBucketReply", DocumentProtocol::REPLY_STATBUCKET, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + StatBucketReply &ref = static_cast<StatBucketReply&>(*obj); + EXPECT_EQUAL("These are the votes of the Norwegian jury", ref.getResults()); + } + } + return true; +} + +bool +Messages60Test::testVisitorInfoReply() +{ + return tryVisitorReply("VisitorInfoReply", DocumentProtocol::REPLY_VISITORINFO); +} + +bool +Messages60Test::testWrongDistributionReply() +{ + WrongDistributionReply tmp("distributor:3 storage:2"); + + serialize("WrongDistributionReply", tmp); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("WrongDistributionReply", DocumentProtocol::REPLY_WRONGDISTRIBUTION, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + WrongDistributionReply &ref = static_cast<WrongDistributionReply&>(*obj); + EXPECT_EQUAL(string("distributor:3 storage:2"), ref.getSystemState()); + } + } + return true; +} + +bool +Messages60Test::testGetBucketListReply() +{ + GetBucketListReply reply; + reply.getBuckets().push_back(GetBucketListReply::BucketInfo(document::BucketId(16, 123), "foo")); + reply.getBuckets().push_back(GetBucketListReply::BucketInfo(document::BucketId(17, 1123), "bar")); + reply.getBuckets().push_back(GetBucketListReply::BucketInfo(document::BucketId(18, 11123), "zoink")); + + EXPECT_EQUAL(56u, serialize("GetBucketListReply", reply)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("GetBucketListReply", DocumentProtocol::REPLY_GETBUCKETLIST, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + GetBucketListReply &ref = static_cast<GetBucketListReply&>(*obj); + + EXPECT_EQUAL(ref.getBuckets()[0], GetBucketListReply::BucketInfo(document::BucketId(16, 123), "foo")); + EXPECT_EQUAL(ref.getBuckets()[1], GetBucketListReply::BucketInfo(document::BucketId(17, 1123), "bar")); + EXPECT_EQUAL(ref.getBuckets()[2], GetBucketListReply::BucketInfo(document::BucketId(18, 11123), "zoink")); + } + } + return true; +} + +bool +Messages60Test::testGetBucketStateReply() +{ + document::GlobalId foo = document::DocumentId("doc:scheme:foo").getGlobalId(); + document::GlobalId bar = document::DocumentId("doc:scheme:bar").getGlobalId(); + + GetBucketStateReply reply; + reply.getBucketState().push_back(DocumentState(foo, 777, false)); + reply.getBucketState().push_back(DocumentState(bar, 888, true)); + EXPECT_EQUAL(53u, serialize("GetBucketStateReply", reply)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("GetBucketStateReply", DocumentProtocol::REPLY_GETBUCKETSTATE, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + GetBucketStateReply &ref = static_cast<GetBucketStateReply&>(*obj); + + EXPECT_EQUAL(777u, ref.getBucketState()[0].getTimestamp()); + EXPECT_EQUAL(foo, ref.getBucketState()[0].getGlobalId()); + EXPECT_EQUAL(false, ref.getBucketState()[0].isRemoveEntry()); + EXPECT_EQUAL(888u, ref.getBucketState()[1].getTimestamp()); + EXPECT_EQUAL(bar, ref.getBucketState()[1].getGlobalId()); + EXPECT_EQUAL(true, ref.getBucketState()[1].isRemoveEntry()); + } + } + return true; +} + +bool +Messages60Test::testEmptyBucketsReply() +{ + return tryVisitorReply("EmptyBucketsReply", DocumentProtocol::REPLY_EMPTYBUCKETS); +} + +bool +Messages60Test::testRemoveLocationReply() +{ + DocumentReply tmp(DocumentProtocol::REPLY_REMOVELOCATION); + + EXPECT_EQUAL((uint32_t)5, serialize("RemoveLocationReply", tmp)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize("RemoveLocationReply", DocumentProtocol::REPLY_REMOVELOCATION, lang); + EXPECT_TRUE(obj.get() != NULL); + } + return true; +} + + + +//////////////////////////////////////////////////////////////////////////////// +// +// Utilities +// +//////////////////////////////////////////////////////////////////////////////// + +bool +Messages60Test::tryDocumentReply(const string &filename, uint32_t type) +{ + DocumentReply tmp(type); + + EXPECT_EQUAL((uint32_t)5, serialize(filename, tmp)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize(filename, type, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + DocumentReply *ref = dynamic_cast<DocumentReply*>(obj.get()); + EXPECT_TRUE(ref != NULL); + } + } + return true; +} + +bool +Messages60Test::tryVisitorReply(const string &filename, uint32_t type) +{ + VisitorReply tmp(type); + + EXPECT_EQUAL((uint32_t)5, serialize(filename, tmp)); + + for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { + mbus::Routable::UP obj = deserialize(filename, type, lang); + if (EXPECT_TRUE(obj.get() != NULL)) { + VisitorReply *ref = dynamic_cast<VisitorReply*>(obj.get()); + EXPECT_TRUE(ref != NULL); + } + } + return true; +} diff --git a/documentapi/src/tests/messages/messages60test.h b/documentapi/src/tests/messages/messages60test.h index 78224e5a808..87954310536 100644 --- a/documentapi/src/tests/messages/messages60test.h +++ b/documentapi/src/tests/messages/messages60test.h @@ -1,14 +1,56 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// @author Vegard Sjonfjell #pragma once -#include "messages52test.h" +#include "testbase.h" -class Messages60Test : public Messages52Test { +class Messages60Test : public TestBase { protected: const vespalib::Version getVersion() const override { return vespalib::Version(6, 221); } + bool shouldTestCoverage() const override { return true; } + bool tryDocumentReply(const string &filename, uint32_t type); + bool tryVisitorReply(const string &filename, uint32_t type); + + static size_t serializedLength(const string & str) { return sizeof(int32_t) + str.size(); } + public: Messages60Test(); + bool testCreateVisitorMessage(); - bool testStatBucketMessage(); + bool testCreateVisitorReply(); + bool testDestroyVisitorMessage(); + bool testDestroyVisitorReply(); + bool testDocumentIgnoredReply(); + bool testDocumentListMessage(); + bool testDocumentListReply(); + bool testDocumentSummaryMessage(); + bool testDocumentSummaryReply(); + bool testEmptyBucketsMessage(); + bool testEmptyBucketsReply(); bool testGetBucketListMessage(); + bool testGetBucketListReply(); + bool testGetBucketStateMessage(); + bool testGetBucketStateReply(); + bool testGetDocumentMessage(); + bool testGetDocumentReply(); + bool testMapVisitorMessage(); + bool testMapVisitorReply(); + bool testPutDocumentMessage(); + bool testPutDocumentReply(); + bool testQueryResultMessage(); + bool testQueryResultReply(); + bool testRemoveDocumentMessage(); + bool testRemoveDocumentReply(); + bool testRemoveLocationMessage(); + bool testRemoveLocationReply(); + bool testSearchResultMessage(); + bool testSearchResultReply(); + bool testStatBucketMessage(); + bool testStatBucketReply(); + bool testUpdateDocumentMessage(); + bool testUpdateDocumentReply(); + bool testVisitorInfoMessage(); + bool testVisitorInfoReply(); + bool testWrongDistributionReply(); }; + diff --git a/documentapi/src/tests/routablefactory/routablefactory.cpp b/documentapi/src/tests/routablefactory/routablefactory.cpp index 13ebd45fdc6..3f94d120d66 100644 --- a/documentapi/src/tests/routablefactory/routablefactory.cpp +++ b/documentapi/src/tests/routablefactory/routablefactory.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/document/repo/documenttyperepo.h> #include <vespa/documentapi/messagebus/documentprotocol.h> -#include <vespa/documentapi/messagebus/routablefactories52.h> +#include <vespa/documentapi/messagebus/routablefactories60.h> #include <vespa/messagebus/testlib/receptor.h> #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> @@ -48,7 +48,7 @@ public: } }; -class MyMessageFactory : public RoutableFactories52::DocumentMessageFactory { +class MyMessageFactory : public RoutableFactories60::DocumentMessageFactory { protected: DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override { (void)buf; @@ -62,7 +62,7 @@ protected: } }; -class MyReplyFactory : public RoutableFactories52::DocumentReplyFactory { +class MyReplyFactory : public RoutableFactories60::DocumentReplyFactory { protected: DocumentReply::UP doDecode(document::ByteBuffer &buf) const override { (void)buf; diff --git a/documentapi/src/vespa/documentapi/messagebus/CMakeLists.txt b/documentapi/src/vespa/documentapi/messagebus/CMakeLists.txt index 6f65b3c67a9..13852c7b1d7 100644 --- a/documentapi/src/vespa/documentapi/messagebus/CMakeLists.txt +++ b/documentapi/src/vespa/documentapi/messagebus/CMakeLists.txt @@ -3,7 +3,6 @@ vespa_add_library(documentapi_documentapimessagebus OBJECT SOURCES documentprotocol.cpp replymerger.cpp - routablefactories52.cpp routablefactories60.cpp routablerepository.cpp routingpolicyfactories.cpp diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp index b70a76690f9..e8cc7a03e94 100644 --- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp @@ -1,6 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "routablefactories52.h" #include "routablefactories60.h" #include "routingpolicyfactories.h" #include "routablerepository.h" @@ -44,54 +43,47 @@ DocumentProtocol::DocumentProtocol(const LoadTypeSet& loadTypes, putRoutingPolicyFactory("LoadBalancer", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::LoadBalancerPolicyFactory())); // Prepare version specifications to use when adding routable factories. - vespalib::VersionSpecification version52(5, 115); vespalib::VersionSpecification version6(6, 221); - std::vector<vespalib::VersionSpecification> from52 = { version52, version6 }; std::vector<vespalib::VersionSpecification> from6 = { version6 }; - // Add 5.2 serialization - putRoutableFactory(MESSAGE_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories52::CreateVisitorMessageFactory(*_repo)), from52); - putRoutableFactory(MESSAGE_DESTROYVISITOR, IRoutableFactory::SP(new RoutableFactories52::DestroyVisitorMessageFactory()), from52); - putRoutableFactory(MESSAGE_DOCUMENTLIST, IRoutableFactory::SP(new RoutableFactories52::DocumentListMessageFactory(*_repo)), from52); - putRoutableFactory(MESSAGE_DOCUMENTSUMMARY, IRoutableFactory::SP(new RoutableFactories52::DocumentSummaryMessageFactory()), from52); - putRoutableFactory(MESSAGE_EMPTYBUCKETS, IRoutableFactory::SP(new RoutableFactories52::EmptyBucketsMessageFactory()), from52); - putRoutableFactory(MESSAGE_GETBUCKETLIST, IRoutableFactory::SP(new RoutableFactories52::GetBucketListMessageFactory()), from52); - putRoutableFactory(MESSAGE_GETBUCKETSTATE, IRoutableFactory::SP(new RoutableFactories52::GetBucketStateMessageFactory()), from52); - putRoutableFactory(MESSAGE_GETDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::GetDocumentMessageFactory()), from52); - putRoutableFactory(MESSAGE_MAPVISITOR, IRoutableFactory::SP(new RoutableFactories52::MapVisitorMessageFactory(*_repo)), from52); - putRoutableFactory(MESSAGE_PUTDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::PutDocumentMessageFactory(*_repo)), from52); - putRoutableFactory(MESSAGE_QUERYRESULT, IRoutableFactory::SP(new RoutableFactories52::QueryResultMessageFactory()), from52); - putRoutableFactory(MESSAGE_REMOVEDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::RemoveDocumentMessageFactory()), from52); - putRoutableFactory(MESSAGE_REMOVELOCATION, IRoutableFactory::SP(new RoutableFactories52::RemoveLocationMessageFactory(*_repo)), from52); - putRoutableFactory(MESSAGE_SEARCHRESULT, IRoutableFactory::SP(new RoutableFactories52::SearchResultMessageFactory()), from52); - putRoutableFactory(MESSAGE_STATBUCKET, IRoutableFactory::SP(new RoutableFactories52::StatBucketMessageFactory()), from52); - putRoutableFactory(MESSAGE_UPDATEDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::UpdateDocumentMessageFactory(*_repo)), from52); - putRoutableFactory(MESSAGE_VISITORINFO, IRoutableFactory::SP(new RoutableFactories52::VisitorInfoMessageFactory()), from52); - putRoutableFactory(REPLY_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories52::CreateVisitorReplyFactory()), from52); - putRoutableFactory(REPLY_DESTROYVISITOR, IRoutableFactory::SP(new RoutableFactories52::DestroyVisitorReplyFactory()), from52); - putRoutableFactory(REPLY_DOCUMENTIGNORED, IRoutableFactory::SP(new RoutableFactories52::DocumentIgnoredReplyFactory()), from52); - putRoutableFactory(REPLY_DOCUMENTLIST, IRoutableFactory::SP(new RoutableFactories52::DocumentListReplyFactory()), from52); - putRoutableFactory(REPLY_DOCUMENTSUMMARY, IRoutableFactory::SP(new RoutableFactories52::DocumentSummaryReplyFactory()), from52); - putRoutableFactory(REPLY_EMPTYBUCKETS, IRoutableFactory::SP(new RoutableFactories52::EmptyBucketsReplyFactory()), from52); - putRoutableFactory(REPLY_GETBUCKETLIST, IRoutableFactory::SP(new RoutableFactories52::GetBucketListReplyFactory()), from52); - putRoutableFactory(REPLY_GETBUCKETSTATE, IRoutableFactory::SP(new RoutableFactories52::GetBucketStateReplyFactory()), from52); - putRoutableFactory(REPLY_GETDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::GetDocumentReplyFactory(*_repo)), from52); - putRoutableFactory(REPLY_MAPVISITOR, IRoutableFactory::SP(new RoutableFactories52::MapVisitorReplyFactory()), from52); - putRoutableFactory(REPLY_PUTDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::PutDocumentReplyFactory()), from52); - putRoutableFactory(REPLY_QUERYRESULT, IRoutableFactory::SP(new RoutableFactories52::QueryResultReplyFactory()), from52); - putRoutableFactory(REPLY_REMOVEDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::RemoveDocumentReplyFactory()), from52); - putRoutableFactory(REPLY_REMOVELOCATION, IRoutableFactory::SP(new RoutableFactories52::RemoveLocationReplyFactory()), from52); - putRoutableFactory(REPLY_SEARCHRESULT, IRoutableFactory::SP(new RoutableFactories52::SearchResultReplyFactory()), from52); - putRoutableFactory(REPLY_STATBUCKET, IRoutableFactory::SP(new RoutableFactories52::StatBucketReplyFactory()), from52); - putRoutableFactory(REPLY_UPDATEDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::UpdateDocumentReplyFactory()), from52); - putRoutableFactory(REPLY_VISITORINFO, IRoutableFactory::SP(new RoutableFactories52::VisitorInfoReplyFactory()), from52); - putRoutableFactory(REPLY_WRONGDISTRIBUTION, IRoutableFactory::SP(new RoutableFactories52::WrongDistributionReplyFactory()), from52); - - // Add 6.x serialization (TODO finalize version) + // Add 6.x serialization putRoutableFactory(MESSAGE_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories60::CreateVisitorMessageFactory(*_repo)), from6); - putRoutableFactory(MESSAGE_STATBUCKET, IRoutableFactory::SP(new RoutableFactories60::StatBucketMessageFactory()), from6); + putRoutableFactory(MESSAGE_DESTROYVISITOR, IRoutableFactory::SP(new RoutableFactories60::DestroyVisitorMessageFactory()), from6); + putRoutableFactory(MESSAGE_DOCUMENTLIST, IRoutableFactory::SP(new RoutableFactories60::DocumentListMessageFactory(*_repo)), from6); + putRoutableFactory(MESSAGE_DOCUMENTSUMMARY, IRoutableFactory::SP(new RoutableFactories60::DocumentSummaryMessageFactory()), from6); + putRoutableFactory(MESSAGE_EMPTYBUCKETS, IRoutableFactory::SP(new RoutableFactories60::EmptyBucketsMessageFactory()), from6); putRoutableFactory(MESSAGE_GETBUCKETLIST, IRoutableFactory::SP(new RoutableFactories60::GetBucketListMessageFactory()), from6); + putRoutableFactory(MESSAGE_GETBUCKETSTATE, IRoutableFactory::SP(new RoutableFactories60::GetBucketStateMessageFactory()), from6); + putRoutableFactory(MESSAGE_GETDOCUMENT, IRoutableFactory::SP(new RoutableFactories60::GetDocumentMessageFactory()), from6); + putRoutableFactory(MESSAGE_MAPVISITOR, IRoutableFactory::SP(new RoutableFactories60::MapVisitorMessageFactory(*_repo)), from6); + putRoutableFactory(MESSAGE_PUTDOCUMENT, IRoutableFactory::SP(new RoutableFactories60::PutDocumentMessageFactory(*_repo)), from6); + putRoutableFactory(MESSAGE_QUERYRESULT, IRoutableFactory::SP(new RoutableFactories60::QueryResultMessageFactory()), from6); + putRoutableFactory(MESSAGE_REMOVEDOCUMENT, IRoutableFactory::SP(new RoutableFactories60::RemoveDocumentMessageFactory()), from6); + putRoutableFactory(MESSAGE_REMOVELOCATION, IRoutableFactory::SP(new RoutableFactories60::RemoveLocationMessageFactory(*_repo)), from6); + putRoutableFactory(MESSAGE_SEARCHRESULT, IRoutableFactory::SP(new RoutableFactories60::SearchResultMessageFactory()), from6); + putRoutableFactory(MESSAGE_STATBUCKET, IRoutableFactory::SP(new RoutableFactories60::StatBucketMessageFactory()), from6); + putRoutableFactory(MESSAGE_UPDATEDOCUMENT, IRoutableFactory::SP(new RoutableFactories60::UpdateDocumentMessageFactory(*_repo)), from6); + putRoutableFactory(MESSAGE_VISITORINFO, IRoutableFactory::SP(new RoutableFactories60::VisitorInfoMessageFactory()), from6); + putRoutableFactory(REPLY_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories60::CreateVisitorReplyFactory()), from6); + putRoutableFactory(REPLY_DESTROYVISITOR, IRoutableFactory::SP(new RoutableFactories60::DestroyVisitorReplyFactory()), from6); + putRoutableFactory(REPLY_DOCUMENTIGNORED, IRoutableFactory::SP(new RoutableFactories60::DocumentIgnoredReplyFactory()), from6); + putRoutableFactory(REPLY_DOCUMENTLIST, IRoutableFactory::SP(new RoutableFactories60::DocumentListReplyFactory()), from6); + putRoutableFactory(REPLY_DOCUMENTSUMMARY, IRoutableFactory::SP(new RoutableFactories60::DocumentSummaryReplyFactory()), from6); + putRoutableFactory(REPLY_EMPTYBUCKETS, IRoutableFactory::SP(new RoutableFactories60::EmptyBucketsReplyFactory()), from6); + putRoutableFactory(REPLY_GETBUCKETLIST, IRoutableFactory::SP(new RoutableFactories60::GetBucketListReplyFactory()), from6); + putRoutableFactory(REPLY_GETBUCKETSTATE, IRoutableFactory::SP(new RoutableFactories60::GetBucketStateReplyFactory()), from6); + putRoutableFactory(REPLY_GETDOCUMENT, IRoutableFactory::SP(new RoutableFactories60::GetDocumentReplyFactory(*_repo)), from6); + putRoutableFactory(REPLY_MAPVISITOR, IRoutableFactory::SP(new RoutableFactories60::MapVisitorReplyFactory()), from6); + putRoutableFactory(REPLY_PUTDOCUMENT, IRoutableFactory::SP(new RoutableFactories60::PutDocumentReplyFactory()), from6); + putRoutableFactory(REPLY_QUERYRESULT, IRoutableFactory::SP(new RoutableFactories60::QueryResultReplyFactory()), from6); + putRoutableFactory(REPLY_REMOVEDOCUMENT, IRoutableFactory::SP(new RoutableFactories60::RemoveDocumentReplyFactory()), from6); + putRoutableFactory(REPLY_REMOVELOCATION, IRoutableFactory::SP(new RoutableFactories60::RemoveLocationReplyFactory()), from6); + putRoutableFactory(REPLY_SEARCHRESULT, IRoutableFactory::SP(new RoutableFactories60::SearchResultReplyFactory()), from6); + putRoutableFactory(REPLY_STATBUCKET, IRoutableFactory::SP(new RoutableFactories60::StatBucketReplyFactory()), from6); + putRoutableFactory(REPLY_UPDATEDOCUMENT, IRoutableFactory::SP(new RoutableFactories60::UpdateDocumentReplyFactory()), from6); + putRoutableFactory(REPLY_VISITORINFO, IRoutableFactory::SP(new RoutableFactories60::VisitorInfoReplyFactory()), from6); + putRoutableFactory(REPLY_WRONGDISTRIBUTION, IRoutableFactory::SP(new RoutableFactories60::WrongDistributionReplyFactory()), from6); } DocumentProtocol::~DocumentProtocol() = default; diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h index 2e6f094bf47..e748e1cbc40 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h +++ b/documentapi/src/vespa/documentapi/messagebus/messages/visitor.h @@ -89,14 +89,6 @@ public: bool visitRemoves() const { return _visitRemoves; } void setVisitRemoves(bool val) { _visitRemoves = val; } - void setVisitHeadersOnly(bool visitHeadersOnly_) { - _fieldSet = (visitHeadersOnly_ ? "[header]" : "[all]"); - } - - bool visitHeadersOnly() const { - return (_fieldSet == "[header]"); - } - const string & getFieldSet() const { return _fieldSet; } void setFieldSet(vespalib::stringref fieldSet) { _fieldSet = fieldSet; } diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories51.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories51.cpp deleted file mode 100644 index 01d0c5b6c0e..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories51.cpp +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "routablefactories51.h" -#include <vespa/document/bucket/fixed_bucket_spaces.h> -#include <vespa/document/document.h> -#include <vespa/documentapi/documentapi.h> -#include <vespa/documentapi/loadtypes/loadtypeset.h> -#include <vespa/vespalib/objects/nbostream.h> - -using document::FixedBucketSpaces; -using vespalib::nbostream; - -namespace documentapi { - -DocumentMessage::UP -RoutableFactories51::CreateVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const -{ - DocumentMessage::UP ret(new CreateVisitorMessage()); - CreateVisitorMessage &msg = static_cast<CreateVisitorMessage&>(*ret); - - msg.setLibraryName(decodeString(buf)); - msg.setInstanceId(decodeString(buf)); - msg.setControlDestination(decodeString(buf)); - msg.setDataDestination(decodeString(buf)); - msg.setDocumentSelection(decodeString(buf)); - msg.setMaximumPendingReplyCount(decodeInt(buf)); - - int32_t len = decodeInt(buf); - msg.getBuckets().reserve(len); - for (int32_t i = 0; i < len; i++) { - int64_t val; - buf.getLong(val); // NOT using getLongNetwork - msg.getBuckets().push_back(document::BucketId(val)); - } - - msg.setFromTimestamp(decodeLong(buf)); - msg.setToTimestamp(decodeLong(buf)); - msg.setVisitRemoves(decodeBoolean(buf)); - msg.setFieldSet(decodeString(buf)); - msg.setVisitInconsistentBuckets(decodeBoolean(buf)); - msg.getParameters().deserialize(_repo, buf); - msg.setVisitorDispatcherVersion(50); - msg.setVisitorOrdering((document::OrderingSpecification::Order)decodeInt(buf)); - msg.setMaxBucketsPerVisitor(decodeInt(buf)); - msg.setBucketSpace(decodeBucketSpace(buf)); - - return ret; -} - -bool -RoutableFactories51::CreateVisitorMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const -{ - const CreateVisitorMessage &msg = static_cast<const CreateVisitorMessage&>(obj); - - buf.putString(msg.getLibraryName()); - buf.putString(msg.getInstanceId()); - buf.putString(msg.getControlDestination()); - buf.putString(msg.getDataDestination()); - buf.putString(msg.getDocumentSelection()); - buf.putInt(msg.getMaximumPendingReplyCount()); - buf.putInt(msg.getBuckets().size()); - - const std::vector<document::BucketId> &buckets = msg.getBuckets(); - for (std::vector<document::BucketId>::const_iterator it = buckets.begin(); - it != buckets.end(); ++it) - { - uint64_t val = it->getRawId(); - buf.putBytes((const char*)&val, 8); - } - - buf.putLong(msg.getFromTimestamp()); - buf.putLong(msg.getToTimestamp()); - buf.putBoolean(msg.visitRemoves()); - buf.putString(msg.getFieldSet()); - buf.putBoolean(msg.visitInconsistentBuckets()); - - int len = msg.getParameters().getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - msg.getParameters().serialize(dbuf); - - buf.putInt(msg.getVisitorOrdering()); - buf.putInt(msg.getMaxBucketsPerVisitor()); - return encodeBucketSpace(msg.getBucketSpace(), buf); -} - -bool RoutableFactories51::CreateVisitorMessageFactory::encodeBucketSpace( - vespalib::stringref bucketSpace, - vespalib::GrowableByteBuffer& buf) const { - (void) buf; - return (bucketSpace == FixedBucketSpaces::default_space_name()); -} - -string RoutableFactories51::CreateVisitorMessageFactory::decodeBucketSpace(document::ByteBuffer&) const { - return FixedBucketSpaces::default_space_name(); -} - -DocumentMessage::UP -RoutableFactories51::GetDocumentMessageFactory::doDecode(document::ByteBuffer &buf) const -{ - return DocumentMessage::UP( - new GetDocumentMessage(decodeDocumentId(buf), - decodeString(buf))); -} - -bool -RoutableFactories51::GetDocumentMessageFactory::doEncode(const DocumentMessage &obj, - vespalib::GrowableByteBuffer &buf) const -{ - const GetDocumentMessage &msg = static_cast<const GetDocumentMessage&>(obj); - - encodeDocumentId(msg.getDocumentId(), buf); - buf.putString(msg.getFieldSet()); - return true; -} - -DocumentReply::UP -RoutableFactories51::DocumentIgnoredReplyFactory::doDecode(document::ByteBuffer& buf) const -{ - (void) buf; - return DocumentReply::UP(new DocumentIgnoredReply()); -} - -bool -RoutableFactories51::DocumentIgnoredReplyFactory::doEncode( - const DocumentReply& obj, - vespalib::GrowableByteBuffer& buf) const -{ - (void) obj; - (void) buf; - return true; -} - -} diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories51.h b/documentapi/src/vespa/documentapi/messagebus/routablefactories51.h deleted file mode 100644 index 913b05f109b..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories51.h +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "routablefactories50.h" - -namespace document { class DocumentTypeRepo; } - -namespace documentapi { -/** - * This class encapsulates all the {@link RoutableFactory} classes needed to implement factories for the document - * routable. When adding new factories to this class, please KEEP THE THEM ORDERED alphabetically like they are now. - */ -class RoutableFactories51 : public RoutableFactories50 { -public: - RoutableFactories51() = delete; - - class DocumentIgnoredReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; - - //////////////////////////////////////////////////////////////////////////////// - // - // Factories - // - //////////////////////////////////////////////////////////////////////////////// - - class CreateVisitorMessageFactory : public DocumentMessageFactory { - const document::DocumentTypeRepo &_repo; - protected: - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - - virtual bool encodeBucketSpace(vespalib::stringref bucketSpace, vespalib::GrowableByteBuffer& buf) const; - virtual string decodeBucketSpace(document::ByteBuffer&) const; - public: - CreateVisitorMessageFactory(const document::DocumentTypeRepo &r) : _repo(r) {} - }; - - class GetDocumentMessageFactory : public DocumentMessageFactory { - protected: - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - }; - - /////////////////////////////////////////////////////////////////////////// - // - // Utilities - // - /////////////////////////////////////////////////////////////////////////// - protected: - /** - * This is a complement for the vespalib::GrowableByteBuffer.putString() method. - * - * @param in The byte buffer to read from. - * @return The decoded string. - */ - static string decodeString(document::ByteBuffer &in) - { return RoutableFactories50::decodeString(in); } - - /** - * This is a complement for the vespalib::GrowableByteBuffer.putBoolean() method. - * - * @param in The byte buffer to read from. - * @return The decoded bool. - */ - static bool decodeBoolean(document::ByteBuffer &in) - { return RoutableFactories50::decodeBoolean(in); } - - /** - * Convenience method to decode a 32-bit int from the given byte buffer. - * - * @param in The byte buffer to read from. - * @return The decoded int. - */ - static int32_t decodeInt(document::ByteBuffer &in) - { return RoutableFactories50::decodeInt(in); } - - /** - * Convenience method to decode a 64-bit int from the given byte buffer. - * - * @param in The byte buffer to read from. - * @return The decoded int. - */ - static int64_t decodeLong(document::ByteBuffer &in) - { return RoutableFactories50::decodeLong(in); } - - - /** - * Convenience method to decode a document id from the given byte buffer. - * - * @param in The byte buffer to read from. - * @return The decoded document id. - */ - static document::DocumentId decodeDocumentId(document::ByteBuffer &in) - { return RoutableFactories50::decodeDocumentId(in); } - - /** - * Convenience method to encode a document id to the given byte buffer. - * - * @param id The document id to encode. - * @param out The byte buffer to write to. - */ - static void encodeDocumentId(const document::DocumentId &id, - vespalib::GrowableByteBuffer &out) - { return RoutableFactories50::encodeDocumentId(id, out); } -}; - -} - diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories52.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories52.cpp deleted file mode 100644 index 6a49d812957..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories52.cpp +++ /dev/null @@ -1,949 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// @author Vegard Sjonfjell - -#include "routablefactories52.h" -#include <vespa/document/bucket/bucketidfactory.h> -#include <vespa/document/bucket/fixed_bucket_spaces.h> -#include <vespa/document/select/parser.h> -#include <vespa/document/update/documentupdate.h> -#include <vespa/documentapi/documentapi.h> -#include <vespa/documentapi/loadtypes/loadtypeset.h> -#include <vespa/vespalib/objects/nbostream.h> - -using document::FixedBucketSpaces; -using vespalib::nbostream; -using std::make_shared; -using std::make_unique; - -namespace documentapi { - -bool -RoutableFactories52::DocumentMessageFactory::encode(const mbus::Routable &obj, vespalib::GrowableByteBuffer &out) const -{ - const DocumentMessage &msg = static_cast<const DocumentMessage&>(obj); - out.putByte(msg.getPriority()); - out.putInt(msg.getLoadType().getId()); - return doEncode(msg, out); -} - -mbus::Routable::UP -RoutableFactories52::DocumentMessageFactory::decode(document::ByteBuffer &in, const LoadTypeSet& loadTypes) const -{ - uint8_t pri; - in.getByte(pri); - uint32_t loadClass = decodeInt(in); - - DocumentMessage::UP msg = doDecode(in); - if (msg) { - msg->setPriority((Priority::Value)pri); - msg->setLoadType(loadTypes[loadClass]); - } - - return mbus::Routable::UP(msg.release()); -} - -bool -RoutableFactories52::DocumentReplyFactory::encode(const mbus::Routable &obj, vespalib::GrowableByteBuffer &out) const -{ - const DocumentReply &msg = static_cast<const DocumentReply&>(obj); - out.putByte(msg.getPriority()); - return doEncode(msg, out); -} - -mbus::Routable::UP -RoutableFactories52::DocumentReplyFactory::decode(document::ByteBuffer &in, const LoadTypeSet&) const -{ - uint8_t pri; - in.getByte(pri); - DocumentReply::UP reply = doDecode(in); - if (reply) { - reply->setPriority((Priority::Value)pri); - } - return mbus::Routable::UP(reply.release()); -} - -//////////////////////////////////////////////////////////////////////////////// -// -// Factories -// -//////////////////////////////////////////////////////////////////////////////// - -DocumentMessage::UP -RoutableFactories52::CreateVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const -{ - auto msg = std::make_unique<CreateVisitorMessage>(); - - msg->setLibraryName(decodeString(buf)); - msg->setInstanceId(decodeString(buf)); - msg->setControlDestination(decodeString(buf)); - msg->setDataDestination(decodeString(buf)); - msg->setDocumentSelection(decodeString(buf)); - msg->setMaximumPendingReplyCount(decodeInt(buf)); - - int32_t len = decodeInt(buf); - msg->getBuckets().reserve(len); - for (int32_t i = 0; i < len; i++) { - int64_t val; - buf.getLong(val); // NOT using getLongNetwork - msg->getBuckets().push_back(document::BucketId(val)); - } - - msg->setFromTimestamp(decodeLong(buf)); - msg->setToTimestamp(decodeLong(buf)); - msg->setVisitRemoves(decodeBoolean(buf)); - msg->setFieldSet(decodeString(buf)); - msg->setVisitInconsistentBuckets(decodeBoolean(buf)); - msg->getParameters().deserialize(_repo, buf); - msg->setVisitorDispatcherVersion(50); - msg->setVisitorOrdering((document::OrderingSpecification::Order)decodeInt(buf)); - msg->setMaxBucketsPerVisitor(decodeInt(buf)); - msg->setBucketSpace(decodeBucketSpace(buf)); - - return msg; -} - -bool -RoutableFactories52::CreateVisitorMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const -{ - const CreateVisitorMessage &msg = static_cast<const CreateVisitorMessage&>(obj); - - buf.putString(msg.getLibraryName()); - buf.putString(msg.getInstanceId()); - buf.putString(msg.getControlDestination()); - buf.putString(msg.getDataDestination()); - buf.putString(msg.getDocumentSelection()); - buf.putInt(msg.getMaximumPendingReplyCount()); - buf.putInt(msg.getBuckets().size()); - - for (const auto & bucketId : msg.getBuckets()) { - uint64_t val = bucketId.getRawId(); - buf.putBytes((const char*)&val, 8); - } - - buf.putLong(msg.getFromTimestamp()); - buf.putLong(msg.getToTimestamp()); - buf.putBoolean(msg.visitRemoves()); - buf.putString(msg.getFieldSet()); - buf.putBoolean(msg.visitInconsistentBuckets()); - - int len = msg.getParameters().getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - msg.getParameters().serialize(dbuf); - - buf.putInt(msg.getVisitorOrdering()); - buf.putInt(msg.getMaxBucketsPerVisitor()); - return encodeBucketSpace(msg.getBucketSpace(), buf); -} - -bool RoutableFactories52::CreateVisitorMessageFactory::encodeBucketSpace( - vespalib::stringref bucketSpace, - vespalib::GrowableByteBuffer& buf) const { - (void) buf; - return (bucketSpace == FixedBucketSpaces::default_space_name()); -} - -string RoutableFactories52::CreateVisitorMessageFactory::decodeBucketSpace(document::ByteBuffer&) const { - return FixedBucketSpaces::default_space_name(); -} - -DocumentMessage::UP -RoutableFactories52::DestroyVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const -{ - auto msg = std::make_unique<DestroyVisitorMessage>(); - msg->setInstanceId(decodeString(buf)); - return msg; -} - -bool -RoutableFactories52::DestroyVisitorMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const -{ - const DestroyVisitorMessage &msg = static_cast<const DestroyVisitorMessage&>(obj); - buf.putString(msg.getInstanceId()); - return true; -} - -DocumentReply::UP -RoutableFactories52::CreateVisitorReplyFactory::doDecode(document::ByteBuffer &buf) const -{ - auto reply = std::make_unique<CreateVisitorReply>(DocumentProtocol::REPLY_CREATEVISITOR); - reply->setLastBucket(document::BucketId((uint64_t)decodeLong(buf))); - vdslib::VisitorStatistics vs; - vs.setBucketsVisited(decodeInt(buf)); - vs.setDocumentsVisited(decodeLong(buf)); - vs.setBytesVisited(decodeLong(buf)); - vs.setDocumentsReturned(decodeLong(buf)); - vs.setBytesReturned(decodeLong(buf)); - vs.setSecondPassDocumentsReturned(decodeLong(buf)); - vs.setSecondPassBytesReturned(decodeLong(buf)); - reply->setVisitorStatistics(vs); - - return reply; -} - -bool -RoutableFactories52::CreateVisitorReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const -{ - const CreateVisitorReply &reply = static_cast<const CreateVisitorReply&>(obj); - buf.putLong(reply.getLastBucket().getRawId()); - buf.putInt(reply.getVisitorStatistics().getBucketsVisited()); - buf.putLong(reply.getVisitorStatistics().getDocumentsVisited()); - buf.putLong(reply.getVisitorStatistics().getBytesVisited()); - buf.putLong(reply.getVisitorStatistics().getDocumentsReturned()); - buf.putLong(reply.getVisitorStatistics().getBytesReturned()); - buf.putLong(reply.getVisitorStatistics().getSecondPassDocumentsReturned()); - buf.putLong(reply.getVisitorStatistics().getSecondPassBytesReturned()); - return true; -} - -DocumentReply::UP -RoutableFactories52::DestroyVisitorReplyFactory::doDecode(document::ByteBuffer &) const -{ - return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_DESTROYVISITOR); -} - -bool -RoutableFactories52::DestroyVisitorReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const -{ - return true; -} - -DocumentReply::UP -RoutableFactories52::DocumentIgnoredReplyFactory::doDecode(document::ByteBuffer& buf) const -{ - (void) buf; - return DocumentReply::UP(new DocumentIgnoredReply()); -} - -bool -RoutableFactories52::DocumentIgnoredReplyFactory::doEncode( - const DocumentReply& obj, - vespalib::GrowableByteBuffer& buf) const -{ - (void) obj; - (void) buf; - return true; -} - -DocumentMessage::UP -RoutableFactories52::DocumentListMessageFactory::doDecode(document::ByteBuffer &buf) const -{ - auto msg = std::make_unique<DocumentListMessage>(); - msg->setBucketId(document::BucketId(decodeLong(buf))); - - int32_t len = decodeInt(buf); - for (int32_t i = 0; i < len; i++) { - DocumentListMessage::Entry entry(_repo, buf); - msg->getDocuments().push_back(entry); - } - - return msg; -} - -bool -RoutableFactories52::DocumentListMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const -{ - const DocumentListMessage &msg = static_cast<const DocumentListMessage&>(obj); - - buf.putLong(msg.getBucketId().getRawId()); - buf.putInt(msg.getDocuments().size()); - for (const auto & document : msg.getDocuments()) { - int len = document.getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - document.serialize(dbuf); - } - - return true; -} - -DocumentReply::UP -RoutableFactories52::DocumentListReplyFactory::doDecode(document::ByteBuffer &) const -{ - return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_DOCUMENTLIST); -} - -bool -RoutableFactories52::DocumentListReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const -{ - return true; -} - -DocumentMessage::UP -RoutableFactories52::DocumentSummaryMessageFactory::doDecode(document::ByteBuffer &buf) const -{ - auto msg = std::make_unique<DocumentSummaryMessage>(); - - msg->deserialize(buf); - - return msg; -} - -bool -RoutableFactories52::DocumentSummaryMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const -{ - const DocumentSummaryMessage &msg = static_cast<const DocumentSummaryMessage&>(obj); - - int32_t len = msg.getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - msg.serialize(dbuf); - - return true; -} - -DocumentReply::UP -RoutableFactories52::DocumentSummaryReplyFactory::doDecode(document::ByteBuffer &) const -{ - return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_DOCUMENTSUMMARY); -} - -bool -RoutableFactories52::DocumentSummaryReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const -{ - return true; -} - -DocumentMessage::UP -RoutableFactories52::EmptyBucketsMessageFactory::doDecode(document::ByteBuffer &buf) const -{ - auto msg = std::make_unique<EmptyBucketsMessage>(); - - int32_t len = decodeInt(buf); - std::vector<document::BucketId> buckets(len); - for (int32_t i = 0; i < len; ++i) { - buckets[i] = document::BucketId(decodeLong(buf)); - } - msg->getBucketIds().swap(buckets); - - return msg; -} - -bool -RoutableFactories52::EmptyBucketsMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const -{ - const EmptyBucketsMessage &msg = static_cast<const EmptyBucketsMessage&>(obj); - - buf.putInt(msg.getBucketIds().size()); - for (const auto & bucketId : msg.getBucketIds()) { - buf.putLong(bucketId.getRawId()); - } - - return true; -} - -DocumentReply::UP -RoutableFactories52::EmptyBucketsReplyFactory::doDecode(document::ByteBuffer &) const -{ - return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_EMPTYBUCKETS); -} - -bool -RoutableFactories52::EmptyBucketsReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const -{ - return true; -} - -bool RoutableFactories52::GetBucketListMessageFactory::encodeBucketSpace(vespalib::stringref bucketSpace, - vespalib::GrowableByteBuffer& ) const -{ - return (bucketSpace == FixedBucketSpaces::default_space_name()); -} - -string RoutableFactories52::GetBucketListMessageFactory::decodeBucketSpace(document::ByteBuffer&) const { - return FixedBucketSpaces::default_space_name(); -} - -DocumentMessage::UP -RoutableFactories52::GetBucketListMessageFactory::doDecode(document::ByteBuffer &buf) const -{ - document::BucketId bucketId(decodeLong(buf)); - auto msg = std::make_unique<GetBucketListMessage>(bucketId); - msg->setBucketSpace(decodeBucketSpace(buf)); - return msg; -} - -bool -RoutableFactories52::GetBucketListMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const -{ - const GetBucketListMessage &msg = static_cast<const GetBucketListMessage&>(obj); - buf.putLong(msg.getBucketId().getRawId()); - return encodeBucketSpace(msg.getBucketSpace(), buf); -} - -DocumentReply::UP -RoutableFactories52::GetBucketListReplyFactory::doDecode(document::ByteBuffer &buf) const -{ - auto reply = std::make_unique<GetBucketListReply>(); - - int32_t len = decodeInt(buf); - reply->getBuckets().reserve(len); - for (int32_t i = 0; i < len; i++) { - GetBucketListReply::BucketInfo info; - info._bucket = document::BucketId((uint64_t)decodeLong(buf)); - info._bucketInformation = decodeString(buf); - reply->getBuckets().push_back(info); - } - - return reply; -} - -bool -RoutableFactories52::GetBucketListReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const -{ - const GetBucketListReply &reply = static_cast<const GetBucketListReply&>(obj); - - const std::vector<GetBucketListReply::BucketInfo> &buckets = reply.getBuckets(); - buf.putInt(buckets.size()); - for (const auto & bucketInfo : buckets) { - buf.putLong(bucketInfo._bucket.getRawId()); - buf.putString(bucketInfo._bucketInformation); - } - - return true; -} - -DocumentMessage::UP -RoutableFactories52::GetBucketStateMessageFactory::doDecode(document::ByteBuffer &buf) const -{ - auto msg = std::make_unique<GetBucketStateMessage>(); - - msg->setBucketId(document::BucketId((uint64_t)decodeLong(buf))); - - return msg; -} - -bool -RoutableFactories52::GetBucketStateMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const -{ - const GetBucketStateMessage &msg = static_cast<const GetBucketStateMessage&>(obj); - buf.putLong(msg.getBucketId().getRawId()); - return true; -} - -DocumentReply::UP -RoutableFactories52::GetBucketStateReplyFactory::doDecode(document::ByteBuffer &buf) const -{ - auto reply = std::make_unique<GetBucketStateReply>(); - - int32_t len = decodeInt(buf); - reply->getBucketState().reserve(len); - for (int32_t i = 0; i < len; i++) { - reply->getBucketState().emplace_back(buf); - } - - return reply; -} - -bool -RoutableFactories52::GetBucketStateReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const -{ - const GetBucketStateReply &reply = static_cast<const GetBucketStateReply&>(obj); - - buf.putInt(reply.getBucketState().size()); - for (const auto & state : reply.getBucketState()) { - state.serialize(buf); - } - - return true; -} - -DocumentMessage::UP -RoutableFactories52::GetDocumentMessageFactory::doDecode(document::ByteBuffer &buf) const -{ - return DocumentMessage::UP( - new GetDocumentMessage(decodeDocumentId(buf), - decodeString(buf))); -} - -bool -RoutableFactories52::GetDocumentMessageFactory::doEncode(const DocumentMessage &obj, - vespalib::GrowableByteBuffer &buf) const -{ - const GetDocumentMessage &msg = static_cast<const GetDocumentMessage&>(obj); - - encodeDocumentId(msg.getDocumentId(), buf); - buf.putString(msg.getFieldSet()); - return true; -} - -DocumentReply::UP -RoutableFactories52::GetDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const -{ - auto reply = std::make_unique<GetDocumentReply>(); - - bool hasDocument = decodeBoolean(buf); - document::Document * document = nullptr; - if (hasDocument) { - auto doc = std::make_shared<document::Document>(_repo, buf); - document = doc.get(); - reply->setDocument(std::move(doc)); - } - int64_t lastModified = decodeLong(buf); - reply->setLastModified(lastModified); - if (hasDocument) { - document->setLastModified(lastModified); - } - - return reply; -} - -bool -RoutableFactories52::GetDocumentReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const -{ - const GetDocumentReply &reply = static_cast<const GetDocumentReply&>(obj); - - buf.putByte(reply.hasDocument() ? 1 : 0); - if (reply.hasDocument()) { - nbostream stream; - reply.getDocument().serialize(stream); - buf.putBytes(stream.peek(), stream.size()); - } - buf.putLong(reply.getLastModified()); - - return true; -} - -DocumentMessage::UP -RoutableFactories52::MapVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const -{ - auto msg = std::make_unique<MapVisitorMessage>(); - msg->getData().deserialize(_repo, buf); - return msg; -} - -bool -RoutableFactories52::MapVisitorMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const -{ - const MapVisitorMessage &msg = static_cast<const MapVisitorMessage&>(obj); - - int32_t len = msg.getData().getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - msg.getData().serialize(dbuf); - - return true; -} - -DocumentReply::UP -RoutableFactories52::MapVisitorReplyFactory::doDecode(document::ByteBuffer &) const -{ - return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_MAPVISITOR); -} - -bool -RoutableFactories52::MapVisitorReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const -{ - return true; -} - -void -RoutableFactories52::PutDocumentMessageFactory::decodeInto(PutDocumentMessage & msg, document::ByteBuffer & buf) const { - msg.setDocument(make_shared<document::Document>(_repo, buf)); - msg.setTimestamp(static_cast<uint64_t>(decodeLong(buf))); - decodeTasCondition(msg, buf); -} - -bool -RoutableFactories52::PutDocumentMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const -{ - auto & msg = static_cast<const PutDocumentMessage &>(obj); - nbostream stream; - - msg.getDocument().serialize(stream); - buf.putBytes(stream.peek(), stream.size()); - buf.putLong(static_cast<int64_t>(msg.getTimestamp())); - encodeTasCondition(buf, msg); - - return true; -} - -DocumentReply::UP -RoutableFactories52::PutDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const -{ - auto reply = make_unique<WriteDocumentReply>(DocumentProtocol::REPLY_PUTDOCUMENT); - reply->setHighestModificationTimestamp(decodeLong(buf)); - - // Doing an explicit move here to force converting result to an rvalue. - // This is done automatically in GCC >= 5. - return std::move(reply); -} - -bool -RoutableFactories52::PutDocumentReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const -{ - const WriteDocumentReply& reply = (const WriteDocumentReply&)obj; - buf.putLong(reply.getHighestModificationTimestamp()); - return true; -} - -void -RoutableFactories52::RemoveDocumentMessageFactory::decodeInto(RemoveDocumentMessage & msg, document::ByteBuffer & buf) const { - msg.setDocumentId(decodeDocumentId(buf)); - decodeTasCondition(msg, buf); -} - -bool -RoutableFactories52::RemoveDocumentMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const -{ - const RemoveDocumentMessage &msg = static_cast<const RemoveDocumentMessage&>(obj); - encodeDocumentId(msg.getDocumentId(), buf); - encodeTasCondition(buf, msg); - return true; -} - -DocumentReply::UP -RoutableFactories52::RemoveDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const -{ - auto reply = std::make_unique<RemoveDocumentReply>(); - reply->setWasFound(decodeBoolean(buf)); - reply->setHighestModificationTimestamp(decodeLong(buf)); - return reply; -} - -bool -RoutableFactories52::RemoveDocumentReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const -{ - const RemoveDocumentReply &reply = static_cast<const RemoveDocumentReply&>(obj); - buf.putBoolean(reply.getWasFound()); - buf.putLong(reply.getHighestModificationTimestamp()); - return true; -} - -DocumentMessage::UP -RoutableFactories52::RemoveLocationMessageFactory::doDecode(document::ByteBuffer &buf) const -{ - string selection = decodeString(buf); - - document::BucketIdFactory factory; - document::select::Parser parser(_repo, factory); - - auto msg = std::make_unique<RemoveLocationMessage>(factory, parser, selection); - // FIXME bucket space not part of wire format, implicitly limiting to only default space for now. - msg->setBucketSpace(document::FixedBucketSpaces::default_space_name()); - return msg; -} - -bool -RoutableFactories52::RemoveLocationMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const -{ - const RemoveLocationMessage &msg = static_cast<const RemoveLocationMessage&>(obj); - buf.putString(msg.getDocumentSelection()); - return true; -} - -DocumentReply::UP -RoutableFactories52::RemoveLocationReplyFactory::doDecode(document::ByteBuffer &) const -{ - return std::make_unique<DocumentReply>(DocumentProtocol::REPLY_REMOVELOCATION); -} - -bool -RoutableFactories52::RemoveLocationReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const -{ - return true; -} - -DocumentMessage::UP -RoutableFactories52::SearchResultMessageFactory::doDecode(document::ByteBuffer &buf) const -{ - auto msg = std::make_unique<SearchResultMessage>(); - msg->deserialize(buf); - return msg; -} - -bool -RoutableFactories52::SearchResultMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const -{ - const SearchResultMessage &msg = static_cast<const SearchResultMessage&>(obj); - - int len = msg.getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - msg.serialize(dbuf); - - return true; -} - -DocumentMessage::UP -RoutableFactories52::QueryResultMessageFactory::doDecode(document::ByteBuffer &buf) const -{ - auto msg = std::make_unique<QueryResultMessage>(); - msg->getSearchResult().deserialize(buf); - msg->getDocumentSummary().deserialize(buf); - - return msg; -} - -bool -RoutableFactories52::QueryResultMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const -{ - const QueryResultMessage &msg = static_cast<const QueryResultMessage&>(obj); - - int len = msg.getSearchResult().getSerializedSize() + msg.getDocumentSummary().getSerializedSize(); - char *tmp = buf.allocate(len); - document::ByteBuffer dbuf(tmp, len); - msg.getSearchResult().serialize(dbuf); - msg.getDocumentSummary().serialize(dbuf); - - return true; -} - -DocumentReply::UP -RoutableFactories52::SearchResultReplyFactory::doDecode(document::ByteBuffer &) const -{ - return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_SEARCHRESULT); -} - -bool -RoutableFactories52::SearchResultReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const -{ - return true; -} - -DocumentReply::UP -RoutableFactories52::QueryResultReplyFactory::doDecode(document::ByteBuffer &) const -{ - return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_QUERYRESULT); -} - -bool -RoutableFactories52::QueryResultReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const -{ - return true; -} - -bool RoutableFactories52::StatBucketMessageFactory::encodeBucketSpace(vespalib::stringref bucketSpace, - vespalib::GrowableByteBuffer& ) const -{ - return (bucketSpace == FixedBucketSpaces::default_space_name()); -} - -string RoutableFactories52::StatBucketMessageFactory::decodeBucketSpace(document::ByteBuffer&) const { - return FixedBucketSpaces::default_space_name(); -} - -DocumentMessage::UP -RoutableFactories52::StatBucketMessageFactory::doDecode(document::ByteBuffer &buf) const -{ - auto msg = std::make_unique<StatBucketMessage>(); - - msg->setBucketId(document::BucketId(decodeLong(buf))); - msg->setDocumentSelection(decodeString(buf)); - msg->setBucketSpace(decodeBucketSpace(buf)); - - return msg; -} - -bool -RoutableFactories52::StatBucketMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const -{ - const StatBucketMessage &msg = static_cast<const StatBucketMessage&>(obj); - - buf.putLong(msg.getBucketId().getRawId()); - buf.putString(msg.getDocumentSelection()); - return encodeBucketSpace(msg.getBucketSpace(), buf); -} - -DocumentReply::UP -RoutableFactories52::StatBucketReplyFactory::doDecode(document::ByteBuffer &buf) const -{ - auto reply = std::make_unique<StatBucketReply>(); - reply->setResults(decodeString(buf)); - return reply; -} - -bool -RoutableFactories52::StatBucketReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const -{ - const StatBucketReply &reply = static_cast<const StatBucketReply&>(obj); - buf.putString(reply.getResults()); - return true; -} - -DocumentMessage::UP -RoutableFactories52::StatDocumentMessageFactory::doDecode(document::ByteBuffer &) const -{ - return DocumentMessage::UP(); // TODO: remove message type -} - -bool -RoutableFactories52::StatDocumentMessageFactory::doEncode(const DocumentMessage &, vespalib::GrowableByteBuffer &) const -{ - return false; -} - -DocumentReply::UP -RoutableFactories52::StatDocumentReplyFactory::doDecode(document::ByteBuffer &) const -{ - return DocumentReply::UP(); // TODO: remove reply type -} - -bool -RoutableFactories52::StatDocumentReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const -{ - return false; -} - -void -RoutableFactories52::UpdateDocumentMessageFactory::decodeInto(UpdateDocumentMessage & msg, document::ByteBuffer & buf) const { - msg.setDocumentUpdate(document::DocumentUpdate::createHEAD(_repo, buf)); - msg.setOldTimestamp(static_cast<uint64_t>(decodeLong(buf))); - msg.setNewTimestamp(static_cast<uint64_t>(decodeLong(buf))); - decodeTasCondition(msg, buf); -} - -bool -RoutableFactories52::UpdateDocumentMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const -{ - const UpdateDocumentMessage &msg = static_cast<const UpdateDocumentMessage&>(obj); - - vespalib::nbostream stream; - msg.getDocumentUpdate().serializeHEAD(stream); - buf.putBytes(stream.peek(), stream.size()); - buf.putLong((int64_t)msg.getOldTimestamp()); - buf.putLong((int64_t)msg.getNewTimestamp()); - encodeTasCondition(buf, msg); - - return true; -} - -DocumentReply::UP -RoutableFactories52::UpdateDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const -{ - auto reply = std::make_unique<UpdateDocumentReply>(); - reply->setWasFound(decodeBoolean(buf)); - reply->setHighestModificationTimestamp(decodeLong(buf)); - return reply; -} - -bool -RoutableFactories52::UpdateDocumentReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const -{ - const UpdateDocumentReply &reply = static_cast<const UpdateDocumentReply&>(obj); - buf.putBoolean(reply.getWasFound()); - buf.putLong(reply.getHighestModificationTimestamp()); - return true; -} - -DocumentMessage::UP -RoutableFactories52::VisitorInfoMessageFactory::doDecode(document::ByteBuffer &buf) const -{ - auto msg = std::make_unique<VisitorInfoMessage>(); - - int32_t len = decodeInt(buf); - msg->getFinishedBuckets().reserve(len); - for (int32_t i = 0; i < len; i++) { - int64_t val; - buf.getLong(val); // NOT using getLongNetwork - msg->getFinishedBuckets().emplace_back(val); - } - msg->setErrorMessage(decodeString(buf)); - - return msg; -} - -bool -RoutableFactories52::VisitorInfoMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const -{ - const VisitorInfoMessage &msg = static_cast<const VisitorInfoMessage&>(obj); - - buf.putInt(msg.getFinishedBuckets().size()); - for (const auto & bucketId : msg.getFinishedBuckets()) { - uint64_t val = bucketId.getRawId(); - buf.putBytes((const char*)&val, 8); - } - buf.putString(msg.getErrorMessage()); - - return true; -} - -DocumentReply::UP -RoutableFactories52::VisitorInfoReplyFactory::doDecode(document::ByteBuffer &) const -{ - return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_VISITORINFO); -} - -bool -RoutableFactories52::VisitorInfoReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const -{ - return true; -} - -DocumentReply::UP -RoutableFactories52::WrongDistributionReplyFactory::doDecode(document::ByteBuffer &buf) const -{ - auto reply = std::make_unique<WrongDistributionReply>(); - reply->setSystemState(decodeString(buf)); - return reply; -} - -bool -RoutableFactories52::WrongDistributionReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const -{ - const WrongDistributionReply &reply = static_cast<const WrongDistributionReply&>(obj); - buf.putString(reply.getSystemState()); - return true; -} - -string -RoutableFactories52::decodeString(document::ByteBuffer &in) -{ - int32_t len = decodeInt(in); - string ret = string(in.getBufferAtPos(), len); - in.incPos(len); - return ret; -} - -bool -RoutableFactories52::decodeBoolean(document::ByteBuffer &in) -{ - char ret; - in.getBytes(&ret, 1); - return (bool)ret; -} - -int32_t -RoutableFactories52::decodeInt(document::ByteBuffer &in) -{ - int32_t ret; - in.getIntNetwork(ret); - return ret; -} - -int64_t -RoutableFactories52::decodeLong(document::ByteBuffer &in) -{ - int64_t ret; - in.getLongNetwork(ret); - return ret; -} - -document::DocumentId -RoutableFactories52::decodeDocumentId(document::ByteBuffer &in) -{ - nbostream stream(in.getBufferAtPos(), in.getRemaining()); - document::DocumentId ret(stream); - in.incPos(stream.rp()); - return ret; -} - -void -RoutableFactories52::encodeDocumentId(const document::DocumentId &id, vespalib::GrowableByteBuffer &out) -{ - string str = id.toString(); - out.putBytes(str.c_str(), str.size() + 1); -} - -void RoutableFactories52::decodeTasCondition(DocumentMessage & docMsg, document::ByteBuffer & buf) { - auto & msg = static_cast<TestAndSetMessage &>(docMsg); - msg.setCondition(TestAndSetCondition(decodeString(buf))); -} - -void RoutableFactories52::encodeTasCondition(vespalib::GrowableByteBuffer & buf, const DocumentMessage & docMsg) { - auto & msg = static_cast<const TestAndSetMessage &>(docMsg); - buf.putString(msg.getCondition().getSelection()); -} - - -} diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories52.h b/documentapi/src/vespa/documentapi/messagebus/routablefactories52.h deleted file mode 100644 index 9c614bde487..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories52.h +++ /dev/null @@ -1,421 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// @author Vegard Sjonfjell -#pragma once - -#include <vespa/document/util/bytebuffer.h> -#include <vespa/vespalib/util/growablebytebuffer.h> -#include "iroutablefactory.h" -#include <vespa/documentapi/messagebus/messages/putdocumentmessage.h> -#include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> -#include <vespa/documentapi/messagebus/messages/updatedocumentmessage.h> - -namespace document { class DocumentTypeRepo; } - -/** - * Utility class for invoking setApproxSize on a DocumentMessage with the delta - * between the read position of a ByteBuffer at construction and destruction - * time. The assumption being made is that the in-memory footprint of a message - * is reasonably close to its wire-serialized form. - */ -class ScopedApproxSizeSetter { -public: - ScopedApproxSizeSetter(documentapi::DocumentMessage& msg, - const document::ByteBuffer& buf) - : _msg(msg), - _buf(buf), - _posBefore(_buf.getPos()) - { - } - - ~ScopedApproxSizeSetter() { - _msg.setApproxSize(static_cast<uint32_t>(_buf.getPos() - _posBefore)); - } - -private: - documentapi::DocumentMessage& _msg; - const document::ByteBuffer& _buf; - const size_t _posBefore; -}; - -namespace documentapi { - -template<typename MessageType, typename FactoryType> -DocumentMessage::UP -decodeMessage(const FactoryType * self, document::ByteBuffer & buf) { - auto msg = std::make_unique<MessageType>(); - ScopedApproxSizeSetter sizeSetter(*msg, buf); - self->decodeInto(*msg, buf); - return msg; -} - -/** - * This class encapsulates all the {@link RoutableFactory} classes needed to implement factories for the document - * routable. When adding new factories to this class, please KEEP THE THEM ORDERED alphabetically like they are now. - */ -class RoutableFactories52 { -public: - RoutableFactories52() = delete; - - /** - * Implements the shared factory logic required for {@link DocumentMessage} objects, and it offers a more - * convenient interface for implementing {@link RoutableFactory}. - */ - class DocumentMessageFactory : public IRoutableFactory { - protected: - /** - * This method encodes the given message into the given byte buffer. You are guaranteed to only receive messages of - * the type that this factory was registered for. - * - * This method is NOT exception safe. Return false to signal failure. - * - * @param msg The message to encode. - * @param buf The byte buffer to write to. - * @return True if the message was encoded. - */ - virtual bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const = 0; - - /** - * This method decodes a message from the given byte buffer. You are guaranteed to only receive byte buffers - * generated by a previous call to {@link #doEncode(DocumentMessage, GrowableByteBuffer)}. - * - * This method is NOT exception safe. Return null to signal failure. - * - * @param buf The byte buffer to read from. - * @return The decoded message. - */ - virtual DocumentMessage::UP doDecode(document::ByteBuffer &buf) const = 0; - - public: - bool encode(const mbus::Routable &obj, vespalib::GrowableByteBuffer &out) const override; - mbus::Routable::UP decode(document::ByteBuffer &in, const LoadTypeSet& loadTypes) const override; - }; - - /** - * Implements the shared factory logic required for {@link DocumentReply} objects, and it offers a more - * convenient interface for implementing {@link RoutableFactory}. - */ - class DocumentReplyFactory : public IRoutableFactory { - protected: - /** - * This method encodes the given reply into the given byte buffer. You are guaranteed to only receive - * replies of the type that this factory was registered for. - * - * This method is NOT exception safe. Return false to signal failure. - * - * @param reply The reply to encode. - * @param buf The byte buffer to write to. - * @return True if the message was encoded. - */ - virtual bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const = 0; - - /** - * This method decodes a reply from the given byte buffer. You are guaranteed to only receive byte buffers - * generated by a previous call to {@link #doEncode(DocumentReply, GrowableByteBuffer)}. - * - * This method is NOT exception safe. Return null to signal failure. - * - * @param buf The byte buffer to read from. - * @return The decoded reply. - */ - virtual DocumentReply::UP doDecode(document::ByteBuffer &buf) const = 0; - - public: - bool encode(const mbus::Routable &obj, vespalib::GrowableByteBuffer &out) const override; - mbus::Routable::UP decode(document::ByteBuffer &in, const LoadTypeSet& loadTypes) const override; - }; - - //////////////////////////////////////////////////////////////////////////////// - // - // Factories - // - //////////////////////////////////////////////////////////////////////////////// - class CreateVisitorMessageFactory : public DocumentMessageFactory { - const document::DocumentTypeRepo &_repo; - protected: - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - - virtual bool encodeBucketSpace(vespalib::stringref bucketSpace, vespalib::GrowableByteBuffer& buf) const; - virtual string decodeBucketSpace(document::ByteBuffer&) const; - public: - CreateVisitorMessageFactory(const document::DocumentTypeRepo &r) : _repo(r) {} - }; - class CreateVisitorReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; - class DestroyVisitorMessageFactory : public DocumentMessageFactory { - protected: - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - }; - class DestroyVisitorReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; - class DocumentIgnoredReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; - class DocumentListMessageFactory : public DocumentMessageFactory { - const document::DocumentTypeRepo &_repo; - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - public: - DocumentListMessageFactory(const document::DocumentTypeRepo &r) - : _repo(r) {} - }; - class DocumentListReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; - class DocumentSummaryMessageFactory : public DocumentMessageFactory { - protected: - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - }; - class DocumentSummaryReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; - class EmptyBucketsMessageFactory : public DocumentMessageFactory { - protected: - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - }; - class EmptyBucketsReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; - class GetBucketListMessageFactory : public DocumentMessageFactory { - virtual bool encodeBucketSpace(vespalib::stringref bucketSpace, vespalib::GrowableByteBuffer& buf) const; - virtual string decodeBucketSpace(document::ByteBuffer&) const; - protected: - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - }; - class GetBucketListReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; - class GetBucketStateMessageFactory : public DocumentMessageFactory { - protected: - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - }; - class GetBucketStateReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; - class GetDocumentMessageFactory : public DocumentMessageFactory { - protected: - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - }; - class GetDocumentReplyFactory : public DocumentReplyFactory { - const document::DocumentTypeRepo &_repo; - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &msg, vespalib::GrowableByteBuffer &buf) const override; - public: - GetDocumentReplyFactory(const document::DocumentTypeRepo &r) : _repo(r) {} - }; - class MapVisitorMessageFactory : public DocumentMessageFactory { - const document::DocumentTypeRepo &_repo; - protected: - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - public: - MapVisitorMessageFactory(const document::DocumentTypeRepo &r) : _repo(r) {} - }; - class MapVisitorReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; - class PutDocumentMessageFactory : public DocumentMessageFactory { - protected: - const document::DocumentTypeRepo &_repo; - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override { - return decodeMessage<PutDocumentMessage>(this, buf); - } - - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - public: - void decodeInto(PutDocumentMessage & msg, document::ByteBuffer & buf) const; - PutDocumentMessageFactory(const document::DocumentTypeRepo &r) : _repo(r) {} - }; - class PutDocumentReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; - class RemoveDocumentMessageFactory : public DocumentMessageFactory { - protected: - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override { - return decodeMessage<RemoveDocumentMessage>(this, buf); - } - - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - public: - void decodeInto(RemoveDocumentMessage & msg, document::ByteBuffer & buf) const; - }; - class RemoveDocumentReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; - class RemoveLocationMessageFactory : public DocumentMessageFactory { - const document::DocumentTypeRepo &_repo; - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - public: - RemoveLocationMessageFactory(const document::DocumentTypeRepo &r) : _repo(r) {} - }; - class RemoveLocationReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; - class SearchResultMessageFactory : public DocumentMessageFactory { - protected: - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - }; - class SearchResultReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; - class StatBucketMessageFactory : public DocumentMessageFactory { - virtual bool encodeBucketSpace(vespalib::stringref bucketSpace, vespalib::GrowableByteBuffer& buf) const; - virtual string decodeBucketSpace(document::ByteBuffer&) const; - protected: - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - }; - class StatBucketReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; - class StatDocumentMessageFactory : public DocumentMessageFactory { - protected: - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - }; - class StatDocumentReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; - class UpdateDocumentMessageFactory : public DocumentMessageFactory { - protected: - const document::DocumentTypeRepo &_repo; - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override { - return decodeMessage<UpdateDocumentMessage>(this, buf); - } - - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - public: - void decodeInto(UpdateDocumentMessage & msg, document::ByteBuffer & buf) const; - UpdateDocumentMessageFactory(const document::DocumentTypeRepo &r) : _repo(r) {} - }; - class UpdateDocumentReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; - class VisitorInfoMessageFactory : public DocumentMessageFactory { - protected: - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - }; - class VisitorInfoReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; - class WrongDistributionReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; - class QueryResultMessageFactory : public DocumentMessageFactory { - protected: - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - }; - class QueryResultReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; - - /////////////////////////////////////////////////////////////////////////// - // - // Utilities - // - /////////////////////////////////////////////////////////////////////////// - - /** - * This is a complement for the vespalib::GrowableByteBuffer.putString() method. - * - * @param in The byte buffer to read from. - * @return The decoded string. - */ - static string decodeString(document::ByteBuffer &in); - - /** - * This is a complement for the vespalib::GrowableByteBuffer.putBoolean() method. - * - * @param in The byte buffer to read from. - * @return The decoded bool. - */ - static bool decodeBoolean(document::ByteBuffer &in); - - /** - * Convenience method to decode a 32-bit int from the given byte buffer. - * - * @param in The byte buffer to read from. - * @return The decoded int. - */ - static int32_t decodeInt(document::ByteBuffer &in); - - /** - * Convenience method to decode a 64-bit int from the given byte buffer. - * - * @param in The byte buffer to read from. - * @return The decoded int. - */ - static int64_t decodeLong(document::ByteBuffer &in); - - /** - * Convenience method to decode a document id from the given byte buffer. - * - * @param in The byte buffer to read from. - * @return The decoded document id. - */ - static document::DocumentId decodeDocumentId(document::ByteBuffer &in); - - /** - * Convenience method to encode a document id to the given byte buffer. - * - * @param id The document id to encode. - * @param out The byte buffer to write to. - */ - static void encodeDocumentId(const document::DocumentId &id, - vespalib::GrowableByteBuffer &out); - - static void decodeTasCondition(DocumentMessage & docMsg, document::ByteBuffer & buf); - static void encodeTasCondition(vespalib::GrowableByteBuffer & buf, const DocumentMessage & docMsg); -}; - -} diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp index 5823cc6720e..c34291c634f 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.cpp @@ -1,10 +1,140 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// @author Vegard Sjonfjell #include "routablefactories60.h" +#include <vespa/document/bucket/bucketidfactory.h> +#include <vespa/document/bucket/fixed_bucket_spaces.h> +#include <vespa/document/select/parser.h> +#include <vespa/document/update/documentupdate.h> +#include <vespa/documentapi/documentapi.h> +#include <vespa/documentapi/loadtypes/loadtypeset.h> +#include <vespa/vespalib/objects/nbostream.h> + +using document::FixedBucketSpaces; +using vespalib::nbostream; +using std::make_shared; +using std::make_unique; namespace documentapi { -// TODO dedupe +bool +RoutableFactories60::DocumentMessageFactory::encode(const mbus::Routable &obj, vespalib::GrowableByteBuffer &out) const +{ + const DocumentMessage &msg = static_cast<const DocumentMessage&>(obj); + out.putByte(msg.getPriority()); + out.putInt(msg.getLoadType().getId()); + return doEncode(msg, out); +} + +mbus::Routable::UP +RoutableFactories60::DocumentMessageFactory::decode(document::ByteBuffer &in, const LoadTypeSet& loadTypes) const +{ + uint8_t pri; + in.getByte(pri); + uint32_t loadClass = decodeInt(in); + + DocumentMessage::UP msg = doDecode(in); + if (msg) { + msg->setPriority((Priority::Value)pri); + msg->setLoadType(loadTypes[loadClass]); + } + + return mbus::Routable::UP(msg.release()); +} + +bool +RoutableFactories60::DocumentReplyFactory::encode(const mbus::Routable &obj, vespalib::GrowableByteBuffer &out) const +{ + const DocumentReply &msg = static_cast<const DocumentReply&>(obj); + out.putByte(msg.getPriority()); + return doEncode(msg, out); +} + +mbus::Routable::UP +RoutableFactories60::DocumentReplyFactory::decode(document::ByteBuffer &in, const LoadTypeSet&) const +{ + uint8_t pri; + in.getByte(pri); + DocumentReply::UP reply = doDecode(in); + if (reply) { + reply->setPriority((Priority::Value)pri); + } + return mbus::Routable::UP(reply.release()); +} + +//////////////////////////////////////////////////////////////////////////////// +// +// Factories +// +//////////////////////////////////////////////////////////////////////////////// + +DocumentMessage::UP +RoutableFactories60::CreateVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + auto msg = std::make_unique<CreateVisitorMessage>(); + + msg->setLibraryName(decodeString(buf)); + msg->setInstanceId(decodeString(buf)); + msg->setControlDestination(decodeString(buf)); + msg->setDataDestination(decodeString(buf)); + msg->setDocumentSelection(decodeString(buf)); + msg->setMaximumPendingReplyCount(decodeInt(buf)); + + int32_t len = decodeInt(buf); + msg->getBuckets().reserve(len); + for (int32_t i = 0; i < len; i++) { + int64_t val; + buf.getLong(val); // NOT using getLongNetwork + msg->getBuckets().push_back(document::BucketId(val)); + } + + msg->setFromTimestamp(decodeLong(buf)); + msg->setToTimestamp(decodeLong(buf)); + msg->setVisitRemoves(decodeBoolean(buf)); + msg->setFieldSet(decodeString(buf)); + msg->setVisitInconsistentBuckets(decodeBoolean(buf)); + msg->getParameters().deserialize(_repo, buf); + msg->setVisitorDispatcherVersion(50); + msg->setVisitorOrdering((document::OrderingSpecification::Order)decodeInt(buf)); + msg->setMaxBucketsPerVisitor(decodeInt(buf)); + msg->setBucketSpace(decodeBucketSpace(buf)); + + return msg; +} + +bool +RoutableFactories60::CreateVisitorMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const CreateVisitorMessage &msg = static_cast<const CreateVisitorMessage&>(obj); + + buf.putString(msg.getLibraryName()); + buf.putString(msg.getInstanceId()); + buf.putString(msg.getControlDestination()); + buf.putString(msg.getDataDestination()); + buf.putString(msg.getDocumentSelection()); + buf.putInt(msg.getMaximumPendingReplyCount()); + buf.putInt(msg.getBuckets().size()); + + for (const auto & bucketId : msg.getBuckets()) { + uint64_t val = bucketId.getRawId(); + buf.putBytes((const char*)&val, 8); + } + + buf.putLong(msg.getFromTimestamp()); + buf.putLong(msg.getToTimestamp()); + buf.putBoolean(msg.visitRemoves()); + buf.putString(msg.getFieldSet()); + buf.putBoolean(msg.visitInconsistentBuckets()); + + int len = msg.getParameters().getSerializedSize(); + char *tmp = buf.allocate(len); + document::ByteBuffer dbuf(tmp, len); + msg.getParameters().serialize(dbuf); + + buf.putInt(msg.getVisitorOrdering()); + buf.putInt(msg.getMaxBucketsPerVisitor()); + return encodeBucketSpace(msg.getBucketSpace(), buf); +} bool RoutableFactories60::CreateVisitorMessageFactory::encodeBucketSpace( vespalib::stringref bucketSpace, @@ -13,32 +143,810 @@ bool RoutableFactories60::CreateVisitorMessageFactory::encodeBucketSpace( return true; } -string RoutableFactories60::CreateVisitorMessageFactory::decodeBucketSpace(document::ByteBuffer& buf) const { +string RoutableFactories60::CreateVisitorMessageFactory::decodeBucketSpace(document::ByteBuffer &buf) const { return doDecodeBucketSpace(buf); } -bool RoutableFactories60::StatBucketMessageFactory::encodeBucketSpace( - vespalib::stringref bucketSpace, - vespalib::GrowableByteBuffer& buf) const { +DocumentMessage::UP +RoutableFactories60::DestroyVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + auto msg = std::make_unique<DestroyVisitorMessage>(); + msg->setInstanceId(decodeString(buf)); + return msg; +} + +bool +RoutableFactories60::DestroyVisitorMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const DestroyVisitorMessage &msg = static_cast<const DestroyVisitorMessage&>(obj); + buf.putString(msg.getInstanceId()); + return true; +} + +DocumentReply::UP +RoutableFactories60::CreateVisitorReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + auto reply = std::make_unique<CreateVisitorReply>(DocumentProtocol::REPLY_CREATEVISITOR); + reply->setLastBucket(document::BucketId((uint64_t)decodeLong(buf))); + vdslib::VisitorStatistics vs; + vs.setBucketsVisited(decodeInt(buf)); + vs.setDocumentsVisited(decodeLong(buf)); + vs.setBytesVisited(decodeLong(buf)); + vs.setDocumentsReturned(decodeLong(buf)); + vs.setBytesReturned(decodeLong(buf)); + vs.setSecondPassDocumentsReturned(decodeLong(buf)); + vs.setSecondPassBytesReturned(decodeLong(buf)); + reply->setVisitorStatistics(vs); + + return reply; +} + +bool +RoutableFactories60::CreateVisitorReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + const CreateVisitorReply &reply = static_cast<const CreateVisitorReply&>(obj); + buf.putLong(reply.getLastBucket().getRawId()); + buf.putInt(reply.getVisitorStatistics().getBucketsVisited()); + buf.putLong(reply.getVisitorStatistics().getDocumentsVisited()); + buf.putLong(reply.getVisitorStatistics().getBytesVisited()); + buf.putLong(reply.getVisitorStatistics().getDocumentsReturned()); + buf.putLong(reply.getVisitorStatistics().getBytesReturned()); + buf.putLong(reply.getVisitorStatistics().getSecondPassDocumentsReturned()); + buf.putLong(reply.getVisitorStatistics().getSecondPassBytesReturned()); + return true; +} + +DocumentReply::UP +RoutableFactories60::DestroyVisitorReplyFactory::doDecode(document::ByteBuffer &) const +{ + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_DESTROYVISITOR); +} + +bool +RoutableFactories60::DestroyVisitorReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const +{ + return true; +} + +DocumentReply::UP +RoutableFactories60::DocumentIgnoredReplyFactory::doDecode(document::ByteBuffer& buf) const +{ + (void) buf; + return DocumentReply::UP(new DocumentIgnoredReply()); +} + +bool +RoutableFactories60::DocumentIgnoredReplyFactory::doEncode( + const DocumentReply& obj, + vespalib::GrowableByteBuffer& buf) const +{ + (void) obj; + (void) buf; + return true; +} + +DocumentMessage::UP +RoutableFactories60::DocumentListMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + auto msg = std::make_unique<DocumentListMessage>(); + msg->setBucketId(document::BucketId(decodeLong(buf))); + + int32_t len = decodeInt(buf); + for (int32_t i = 0; i < len; i++) { + DocumentListMessage::Entry entry(_repo, buf); + msg->getDocuments().push_back(entry); + } + + return msg; +} + +bool +RoutableFactories60::DocumentListMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const DocumentListMessage &msg = static_cast<const DocumentListMessage&>(obj); + + buf.putLong(msg.getBucketId().getRawId()); + buf.putInt(msg.getDocuments().size()); + for (const auto & document : msg.getDocuments()) { + int len = document.getSerializedSize(); + char *tmp = buf.allocate(len); + document::ByteBuffer dbuf(tmp, len); + document.serialize(dbuf); + } + + return true; +} + +DocumentReply::UP +RoutableFactories60::DocumentListReplyFactory::doDecode(document::ByteBuffer &) const +{ + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_DOCUMENTLIST); +} + +bool +RoutableFactories60::DocumentListReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const +{ + return true; +} + +DocumentMessage::UP +RoutableFactories60::DocumentSummaryMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + auto msg = std::make_unique<DocumentSummaryMessage>(); + + msg->deserialize(buf); + + return msg; +} + +bool +RoutableFactories60::DocumentSummaryMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const DocumentSummaryMessage &msg = static_cast<const DocumentSummaryMessage&>(obj); + + int32_t len = msg.getSerializedSize(); + char *tmp = buf.allocate(len); + document::ByteBuffer dbuf(tmp, len); + msg.serialize(dbuf); + + return true; +} + +DocumentReply::UP +RoutableFactories60::DocumentSummaryReplyFactory::doDecode(document::ByteBuffer &) const +{ + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_DOCUMENTSUMMARY); +} + +bool +RoutableFactories60::DocumentSummaryReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const +{ + return true; +} + +DocumentMessage::UP +RoutableFactories60::EmptyBucketsMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + auto msg = std::make_unique<EmptyBucketsMessage>(); + + int32_t len = decodeInt(buf); + std::vector<document::BucketId> buckets(len); + for (int32_t i = 0; i < len; ++i) { + buckets[i] = document::BucketId(decodeLong(buf)); + } + msg->getBucketIds().swap(buckets); + + return msg; +} + +bool +RoutableFactories60::EmptyBucketsMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const EmptyBucketsMessage &msg = static_cast<const EmptyBucketsMessage&>(obj); + + buf.putInt(msg.getBucketIds().size()); + for (const auto & bucketId : msg.getBucketIds()) { + buf.putLong(bucketId.getRawId()); + } + + return true; +} + +DocumentReply::UP +RoutableFactories60::EmptyBucketsReplyFactory::doDecode(document::ByteBuffer &) const +{ + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_EMPTYBUCKETS); +} + +bool +RoutableFactories60::EmptyBucketsReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const +{ + return true; +} + +bool RoutableFactories60::GetBucketListMessageFactory::encodeBucketSpace(vespalib::stringref bucketSpace, + vespalib::GrowableByteBuffer &buf) const +{ doEncodeBucketSpace(bucketSpace, buf); return true; } -string RoutableFactories60::StatBucketMessageFactory::decodeBucketSpace(document::ByteBuffer& buf) const { +string RoutableFactories60::GetBucketListMessageFactory::decodeBucketSpace(document::ByteBuffer &buf) const { return doDecodeBucketSpace(buf); } -bool RoutableFactories60::GetBucketListMessageFactory::encodeBucketSpace( - vespalib::stringref bucketSpace, - vespalib::GrowableByteBuffer& buf) const { +DocumentMessage::UP +RoutableFactories60::GetBucketListMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + document::BucketId bucketId(decodeLong(buf)); + auto msg = std::make_unique<GetBucketListMessage>(bucketId); + msg->setBucketSpace(decodeBucketSpace(buf)); + return msg; +} + +bool +RoutableFactories60::GetBucketListMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const GetBucketListMessage &msg = static_cast<const GetBucketListMessage&>(obj); + buf.putLong(msg.getBucketId().getRawId()); + return encodeBucketSpace(msg.getBucketSpace(), buf); +} + +DocumentReply::UP +RoutableFactories60::GetBucketListReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + auto reply = std::make_unique<GetBucketListReply>(); + + int32_t len = decodeInt(buf); + reply->getBuckets().reserve(len); + for (int32_t i = 0; i < len; i++) { + GetBucketListReply::BucketInfo info; + info._bucket = document::BucketId((uint64_t)decodeLong(buf)); + info._bucketInformation = decodeString(buf); + reply->getBuckets().push_back(info); + } + + return reply; +} + +bool +RoutableFactories60::GetBucketListReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + const GetBucketListReply &reply = static_cast<const GetBucketListReply&>(obj); + + const std::vector<GetBucketListReply::BucketInfo> &buckets = reply.getBuckets(); + buf.putInt(buckets.size()); + for (const auto & bucketInfo : buckets) { + buf.putLong(bucketInfo._bucket.getRawId()); + buf.putString(bucketInfo._bucketInformation); + } + + return true; +} + +DocumentMessage::UP +RoutableFactories60::GetBucketStateMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + auto msg = std::make_unique<GetBucketStateMessage>(); + + msg->setBucketId(document::BucketId((uint64_t)decodeLong(buf))); + + return msg; +} + +bool +RoutableFactories60::GetBucketStateMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const GetBucketStateMessage &msg = static_cast<const GetBucketStateMessage&>(obj); + buf.putLong(msg.getBucketId().getRawId()); + return true; +} + +DocumentReply::UP +RoutableFactories60::GetBucketStateReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + auto reply = std::make_unique<GetBucketStateReply>(); + + int32_t len = decodeInt(buf); + reply->getBucketState().reserve(len); + for (int32_t i = 0; i < len; i++) { + reply->getBucketState().emplace_back(buf); + } + + return reply; +} + +bool +RoutableFactories60::GetBucketStateReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + const GetBucketStateReply &reply = static_cast<const GetBucketStateReply&>(obj); + + buf.putInt(reply.getBucketState().size()); + for (const auto & state : reply.getBucketState()) { + state.serialize(buf); + } + + return true; +} + +DocumentMessage::UP +RoutableFactories60::GetDocumentMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + return DocumentMessage::UP( + new GetDocumentMessage(decodeDocumentId(buf), + decodeString(buf))); +} + +bool +RoutableFactories60::GetDocumentMessageFactory::doEncode(const DocumentMessage &obj, + vespalib::GrowableByteBuffer &buf) const +{ + const GetDocumentMessage &msg = static_cast<const GetDocumentMessage&>(obj); + + encodeDocumentId(msg.getDocumentId(), buf); + buf.putString(msg.getFieldSet()); + return true; +} + +DocumentReply::UP +RoutableFactories60::GetDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + auto reply = std::make_unique<GetDocumentReply>(); + + bool hasDocument = decodeBoolean(buf); + document::Document * document = nullptr; + if (hasDocument) { + auto doc = std::make_shared<document::Document>(_repo, buf); + document = doc.get(); + reply->setDocument(std::move(doc)); + } + int64_t lastModified = decodeLong(buf); + reply->setLastModified(lastModified); + if (hasDocument) { + document->setLastModified(lastModified); + } + + return reply; +} + +bool +RoutableFactories60::GetDocumentReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + const GetDocumentReply &reply = static_cast<const GetDocumentReply&>(obj); + + buf.putByte(reply.hasDocument() ? 1 : 0); + if (reply.hasDocument()) { + nbostream stream; + reply.getDocument().serialize(stream); + buf.putBytes(stream.peek(), stream.size()); + } + buf.putLong(reply.getLastModified()); + + return true; +} + +DocumentMessage::UP +RoutableFactories60::MapVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + auto msg = std::make_unique<MapVisitorMessage>(); + msg->getData().deserialize(_repo, buf); + return msg; +} + +bool +RoutableFactories60::MapVisitorMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const MapVisitorMessage &msg = static_cast<const MapVisitorMessage&>(obj); + + int32_t len = msg.getData().getSerializedSize(); + char *tmp = buf.allocate(len); + document::ByteBuffer dbuf(tmp, len); + msg.getData().serialize(dbuf); + + return true; +} + +DocumentReply::UP +RoutableFactories60::MapVisitorReplyFactory::doDecode(document::ByteBuffer &) const +{ + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_MAPVISITOR); +} + +bool +RoutableFactories60::MapVisitorReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const +{ + return true; +} + +void +RoutableFactories60::PutDocumentMessageFactory::decodeInto(PutDocumentMessage & msg, document::ByteBuffer & buf) const { + msg.setDocument(make_shared<document::Document>(_repo, buf)); + msg.setTimestamp(static_cast<uint64_t>(decodeLong(buf))); + decodeTasCondition(msg, buf); +} + +bool +RoutableFactories60::PutDocumentMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + auto & msg = static_cast<const PutDocumentMessage &>(obj); + nbostream stream; + + msg.getDocument().serialize(stream); + buf.putBytes(stream.peek(), stream.size()); + buf.putLong(static_cast<int64_t>(msg.getTimestamp())); + encodeTasCondition(buf, msg); + + return true; +} + +DocumentReply::UP +RoutableFactories60::PutDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + auto reply = make_unique<WriteDocumentReply>(DocumentProtocol::REPLY_PUTDOCUMENT); + reply->setHighestModificationTimestamp(decodeLong(buf)); + + // Doing an explicit move here to force converting result to an rvalue. + // This is done automatically in GCC >= 5. + return std::move(reply); +} + +bool +RoutableFactories60::PutDocumentReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + const WriteDocumentReply& reply = (const WriteDocumentReply&)obj; + buf.putLong(reply.getHighestModificationTimestamp()); + return true; +} + +void +RoutableFactories60::RemoveDocumentMessageFactory::decodeInto(RemoveDocumentMessage & msg, document::ByteBuffer & buf) const { + msg.setDocumentId(decodeDocumentId(buf)); + decodeTasCondition(msg, buf); +} + +bool +RoutableFactories60::RemoveDocumentMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const RemoveDocumentMessage &msg = static_cast<const RemoveDocumentMessage&>(obj); + encodeDocumentId(msg.getDocumentId(), buf); + encodeTasCondition(buf, msg); + return true; +} + +DocumentReply::UP +RoutableFactories60::RemoveDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + auto reply = std::make_unique<RemoveDocumentReply>(); + reply->setWasFound(decodeBoolean(buf)); + reply->setHighestModificationTimestamp(decodeLong(buf)); + return reply; +} + +bool +RoutableFactories60::RemoveDocumentReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + const RemoveDocumentReply &reply = static_cast<const RemoveDocumentReply&>(obj); + buf.putBoolean(reply.getWasFound()); + buf.putLong(reply.getHighestModificationTimestamp()); + return true; +} + +DocumentMessage::UP +RoutableFactories60::RemoveLocationMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + string selection = decodeString(buf); + + document::BucketIdFactory factory; + document::select::Parser parser(_repo, factory); + + auto msg = std::make_unique<RemoveLocationMessage>(factory, parser, selection); + // FIXME bucket space not part of wire format, implicitly limiting to only default space for now. + msg->setBucketSpace(document::FixedBucketSpaces::default_space_name()); + return msg; +} + +bool +RoutableFactories60::RemoveLocationMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const RemoveLocationMessage &msg = static_cast<const RemoveLocationMessage&>(obj); + buf.putString(msg.getDocumentSelection()); + return true; +} + +DocumentReply::UP +RoutableFactories60::RemoveLocationReplyFactory::doDecode(document::ByteBuffer &) const +{ + return std::make_unique<DocumentReply>(DocumentProtocol::REPLY_REMOVELOCATION); +} + +bool +RoutableFactories60::RemoveLocationReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const +{ + return true; +} + +DocumentMessage::UP +RoutableFactories60::SearchResultMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + auto msg = std::make_unique<SearchResultMessage>(); + msg->deserialize(buf); + return msg; +} + +bool +RoutableFactories60::SearchResultMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const SearchResultMessage &msg = static_cast<const SearchResultMessage&>(obj); + + int len = msg.getSerializedSize(); + char *tmp = buf.allocate(len); + document::ByteBuffer dbuf(tmp, len); + msg.serialize(dbuf); + + return true; +} + +DocumentMessage::UP +RoutableFactories60::QueryResultMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + auto msg = std::make_unique<QueryResultMessage>(); + msg->getSearchResult().deserialize(buf); + msg->getDocumentSummary().deserialize(buf); + + return msg; +} + +bool +RoutableFactories60::QueryResultMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const QueryResultMessage &msg = static_cast<const QueryResultMessage&>(obj); + + int len = msg.getSearchResult().getSerializedSize() + msg.getDocumentSummary().getSerializedSize(); + char *tmp = buf.allocate(len); + document::ByteBuffer dbuf(tmp, len); + msg.getSearchResult().serialize(dbuf); + msg.getDocumentSummary().serialize(dbuf); + + return true; +} + +DocumentReply::UP +RoutableFactories60::SearchResultReplyFactory::doDecode(document::ByteBuffer &) const +{ + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_SEARCHRESULT); +} + +bool +RoutableFactories60::SearchResultReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const +{ + return true; +} + +DocumentReply::UP +RoutableFactories60::QueryResultReplyFactory::doDecode(document::ByteBuffer &) const +{ + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_QUERYRESULT); +} + +bool +RoutableFactories60::QueryResultReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const +{ + return true; +} + +bool RoutableFactories60::StatBucketMessageFactory::encodeBucketSpace(vespalib::stringref bucketSpace, + vespalib::GrowableByteBuffer &buf) const +{ doEncodeBucketSpace(bucketSpace, buf); return true; } -string RoutableFactories60::GetBucketListMessageFactory::decodeBucketSpace(document::ByteBuffer& buf) const { +string RoutableFactories60::StatBucketMessageFactory::decodeBucketSpace(document::ByteBuffer &buf) const { return doDecodeBucketSpace(buf); } +DocumentMessage::UP +RoutableFactories60::StatBucketMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + auto msg = std::make_unique<StatBucketMessage>(); + + msg->setBucketId(document::BucketId(decodeLong(buf))); + msg->setDocumentSelection(decodeString(buf)); + msg->setBucketSpace(decodeBucketSpace(buf)); + + return msg; +} + +bool +RoutableFactories60::StatBucketMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const StatBucketMessage &msg = static_cast<const StatBucketMessage&>(obj); + + buf.putLong(msg.getBucketId().getRawId()); + buf.putString(msg.getDocumentSelection()); + return encodeBucketSpace(msg.getBucketSpace(), buf); +} + +DocumentReply::UP +RoutableFactories60::StatBucketReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + auto reply = std::make_unique<StatBucketReply>(); + reply->setResults(decodeString(buf)); + return reply; +} + +bool +RoutableFactories60::StatBucketReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + const StatBucketReply &reply = static_cast<const StatBucketReply&>(obj); + buf.putString(reply.getResults()); + return true; +} + +DocumentMessage::UP +RoutableFactories60::StatDocumentMessageFactory::doDecode(document::ByteBuffer &) const +{ + return DocumentMessage::UP(); // TODO: remove message type +} + +bool +RoutableFactories60::StatDocumentMessageFactory::doEncode(const DocumentMessage &, vespalib::GrowableByteBuffer &) const +{ + return false; +} + +DocumentReply::UP +RoutableFactories60::StatDocumentReplyFactory::doDecode(document::ByteBuffer &) const +{ + return DocumentReply::UP(); // TODO: remove reply type +} + +bool +RoutableFactories60::StatDocumentReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const +{ + return false; +} + +void +RoutableFactories60::UpdateDocumentMessageFactory::decodeInto(UpdateDocumentMessage & msg, document::ByteBuffer & buf) const { + msg.setDocumentUpdate(document::DocumentUpdate::createHEAD(_repo, buf)); + msg.setOldTimestamp(static_cast<uint64_t>(decodeLong(buf))); + msg.setNewTimestamp(static_cast<uint64_t>(decodeLong(buf))); + decodeTasCondition(msg, buf); +} + +bool +RoutableFactories60::UpdateDocumentMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const UpdateDocumentMessage &msg = static_cast<const UpdateDocumentMessage&>(obj); + + vespalib::nbostream stream; + msg.getDocumentUpdate().serializeHEAD(stream); + buf.putBytes(stream.peek(), stream.size()); + buf.putLong((int64_t)msg.getOldTimestamp()); + buf.putLong((int64_t)msg.getNewTimestamp()); + encodeTasCondition(buf, msg); + + return true; +} + +DocumentReply::UP +RoutableFactories60::UpdateDocumentReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + auto reply = std::make_unique<UpdateDocumentReply>(); + reply->setWasFound(decodeBoolean(buf)); + reply->setHighestModificationTimestamp(decodeLong(buf)); + return reply; +} + +bool +RoutableFactories60::UpdateDocumentReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + const UpdateDocumentReply &reply = static_cast<const UpdateDocumentReply&>(obj); + buf.putBoolean(reply.getWasFound()); + buf.putLong(reply.getHighestModificationTimestamp()); + return true; +} + +DocumentMessage::UP +RoutableFactories60::VisitorInfoMessageFactory::doDecode(document::ByteBuffer &buf) const +{ + auto msg = std::make_unique<VisitorInfoMessage>(); + + int32_t len = decodeInt(buf); + msg->getFinishedBuckets().reserve(len); + for (int32_t i = 0; i < len; i++) { + int64_t val; + buf.getLong(val); // NOT using getLongNetwork + msg->getFinishedBuckets().emplace_back(val); + } + msg->setErrorMessage(decodeString(buf)); + + return msg; +} + +bool +RoutableFactories60::VisitorInfoMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const +{ + const VisitorInfoMessage &msg = static_cast<const VisitorInfoMessage&>(obj); + + buf.putInt(msg.getFinishedBuckets().size()); + for (const auto & bucketId : msg.getFinishedBuckets()) { + uint64_t val = bucketId.getRawId(); + buf.putBytes((const char*)&val, 8); + } + buf.putString(msg.getErrorMessage()); + + return true; +} + +DocumentReply::UP +RoutableFactories60::VisitorInfoReplyFactory::doDecode(document::ByteBuffer &) const +{ + return std::make_unique<VisitorReply>(DocumentProtocol::REPLY_VISITORINFO); +} + +bool +RoutableFactories60::VisitorInfoReplyFactory::doEncode(const DocumentReply &, vespalib::GrowableByteBuffer &) const +{ + return true; +} + +DocumentReply::UP +RoutableFactories60::WrongDistributionReplyFactory::doDecode(document::ByteBuffer &buf) const +{ + auto reply = std::make_unique<WrongDistributionReply>(); + reply->setSystemState(decodeString(buf)); + return reply; +} + +bool +RoutableFactories60::WrongDistributionReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const +{ + const WrongDistributionReply &reply = static_cast<const WrongDistributionReply&>(obj); + buf.putString(reply.getSystemState()); + return true; +} + +string +RoutableFactories60::decodeString(document::ByteBuffer &in) +{ + int32_t len = decodeInt(in); + string ret = string(in.getBufferAtPos(), len); + in.incPos(len); + return ret; +} + +bool +RoutableFactories60::decodeBoolean(document::ByteBuffer &in) +{ + char ret; + in.getBytes(&ret, 1); + return (bool)ret; +} + +int32_t +RoutableFactories60::decodeInt(document::ByteBuffer &in) +{ + int32_t ret; + in.getIntNetwork(ret); + return ret; +} + +int64_t +RoutableFactories60::decodeLong(document::ByteBuffer &in) +{ + int64_t ret; + in.getLongNetwork(ret); + return ret; +} + +document::DocumentId +RoutableFactories60::decodeDocumentId(document::ByteBuffer &in) +{ + nbostream stream(in.getBufferAtPos(), in.getRemaining()); + document::DocumentId ret(stream); + in.incPos(stream.rp()); + return ret; +} + +void +RoutableFactories60::encodeDocumentId(const document::DocumentId &id, vespalib::GrowableByteBuffer &out) +{ + string str = id.toString(); + out.putBytes(str.c_str(), str.size() + 1); +} + +void RoutableFactories60::decodeTasCondition(DocumentMessage & docMsg, document::ByteBuffer & buf) { + auto & msg = static_cast<TestAndSetMessage &>(docMsg); + msg.setCondition(TestAndSetCondition(decodeString(buf))); +} + +void RoutableFactories60::encodeTasCondition(vespalib::GrowableByteBuffer & buf, const DocumentMessage & docMsg) { + auto & msg = static_cast<const TestAndSetMessage &>(docMsg); + buf.putString(msg.getCondition().getSelection()); +} + void RoutableFactories60::doEncodeBucketSpace( vespalib::stringref bucketSpace, vespalib::GrowableByteBuffer& buf) { @@ -48,4 +956,4 @@ string RoutableFactories60::doDecodeBucketSpace(document::ByteBuffer& buf) { return decodeString(buf); } -}
\ No newline at end of file +} diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.h b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.h index e91c9a773f2..75404f1dea1 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories60.h +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories60.h @@ -1,41 +1,421 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// @author Vegard Sjonfjell #pragma once -#include "routablefactories52.h" +#include <vespa/document/util/bytebuffer.h> +#include <vespa/vespalib/util/growablebytebuffer.h> +#include "iroutablefactory.h" +#include <vespa/documentapi/messagebus/messages/putdocumentmessage.h> +#include <vespa/documentapi/messagebus/messages/removedocumentmessage.h> +#include <vespa/documentapi/messagebus/messages/updatedocumentmessage.h> + +namespace document { class DocumentTypeRepo; } + +/** + * Utility class for invoking setApproxSize on a DocumentMessage with the delta + * between the read position of a ByteBuffer at construction and destruction + * time. The assumption being made is that the in-memory footprint of a message + * is reasonably close to its wire-serialized form. + */ +class ScopedApproxSizeSetter { +public: + ScopedApproxSizeSetter(documentapi::DocumentMessage& msg, + const document::ByteBuffer& buf) + : _msg(msg), + _buf(buf), + _posBefore(_buf.getPos()) + { + } + + ~ScopedApproxSizeSetter() { + _msg.setApproxSize(static_cast<uint32_t>(_buf.getPos() - _posBefore)); + } + +private: + documentapi::DocumentMessage& _msg; + const document::ByteBuffer& _buf; + const size_t _posBefore; +}; namespace documentapi { + +template<typename MessageType, typename FactoryType> +DocumentMessage::UP +decodeMessage(const FactoryType * self, document::ByteBuffer & buf) { + auto msg = std::make_unique<MessageType>(); + ScopedApproxSizeSetter sizeSetter(*msg, buf); + self->decodeInto(*msg, buf); + return msg; +} + /** * This class encapsulates all the {@link RoutableFactory} classes needed to implement factories for the document * routable. When adding new factories to this class, please KEEP THE THEM ORDERED alphabetically like they are now. */ -class RoutableFactories60 : public RoutableFactories52 { +class RoutableFactories60 { public: RoutableFactories60() = delete; - // TODO dedupe bucket space codec code + /** + * Implements the shared factory logic required for {@link DocumentMessage} objects, and it offers a more + * convenient interface for implementing {@link RoutableFactory}. + */ + class DocumentMessageFactory : public IRoutableFactory { + protected: + /** + * This method encodes the given message into the given byte buffer. You are guaranteed to only receive messages of + * the type that this factory was registered for. + * + * This method is NOT exception safe. Return false to signal failure. + * + * @param msg The message to encode. + * @param buf The byte buffer to write to. + * @return True if the message was encoded. + */ + virtual bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const = 0; + + /** + * This method decodes a message from the given byte buffer. You are guaranteed to only receive byte buffers + * generated by a previous call to {@link #doEncode(DocumentMessage, GrowableByteBuffer)}. + * + * This method is NOT exception safe. Return null to signal failure. + * + * @param buf The byte buffer to read from. + * @return The decoded message. + */ + virtual DocumentMessage::UP doDecode(document::ByteBuffer &buf) const = 0; - class CreateVisitorMessageFactory : public RoutableFactories52::CreateVisitorMessageFactory { - bool encodeBucketSpace(vespalib::stringref bucketSpace, vespalib::GrowableByteBuffer& buf) const override; - string decodeBucketSpace(document::ByteBuffer&) const override; public: - CreateVisitorMessageFactory(const document::DocumentTypeRepo& r) - : RoutableFactories52::CreateVisitorMessageFactory(r) {} + bool encode(const mbus::Routable &obj, vespalib::GrowableByteBuffer &out) const override; + mbus::Routable::UP decode(document::ByteBuffer &in, const LoadTypeSet& loadTypes) const override; }; - class StatBucketMessageFactory : public RoutableFactories52::StatBucketMessageFactory { - bool encodeBucketSpace(vespalib::stringref bucketSpace, vespalib::GrowableByteBuffer& buf) const override; - string decodeBucketSpace(document::ByteBuffer&) const override; + /** + * Implements the shared factory logic required for {@link DocumentReply} objects, and it offers a more + * convenient interface for implementing {@link RoutableFactory}. + */ + class DocumentReplyFactory : public IRoutableFactory { + protected: + /** + * This method encodes the given reply into the given byte buffer. You are guaranteed to only receive + * replies of the type that this factory was registered for. + * + * This method is NOT exception safe. Return false to signal failure. + * + * @param reply The reply to encode. + * @param buf The byte buffer to write to. + * @return True if the message was encoded. + */ + virtual bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const = 0; + + /** + * This method decodes a reply from the given byte buffer. You are guaranteed to only receive byte buffers + * generated by a previous call to {@link #doEncode(DocumentReply, GrowableByteBuffer)}. + * + * This method is NOT exception safe. Return null to signal failure. + * + * @param buf The byte buffer to read from. + * @return The decoded reply. + */ + virtual DocumentReply::UP doDecode(document::ByteBuffer &buf) const = 0; + public: - StatBucketMessageFactory() = default; + bool encode(const mbus::Routable &obj, vespalib::GrowableByteBuffer &out) const override; + mbus::Routable::UP decode(document::ByteBuffer &in, const LoadTypeSet& loadTypes) const override; }; - class GetBucketListMessageFactory : public RoutableFactories52::GetBucketListMessageFactory { - bool encodeBucketSpace(vespalib::stringref bucketSpace, vespalib::GrowableByteBuffer& buf) const override; - string decodeBucketSpace(document::ByteBuffer&) const override; + //////////////////////////////////////////////////////////////////////////////// + // + // Factories + // + //////////////////////////////////////////////////////////////////////////////// + class CreateVisitorMessageFactory : public DocumentMessageFactory { + const document::DocumentTypeRepo &_repo; + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; + + virtual bool encodeBucketSpace(vespalib::stringref bucketSpace, vespalib::GrowableByteBuffer& buf) const; + virtual string decodeBucketSpace(document::ByteBuffer&) const; + public: + CreateVisitorMessageFactory(const document::DocumentTypeRepo &r) : _repo(r) {} + }; + class CreateVisitorReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; + }; + class DestroyVisitorMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; + }; + class DestroyVisitorReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; + }; + class DocumentIgnoredReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; + }; + class DocumentListMessageFactory : public DocumentMessageFactory { + const document::DocumentTypeRepo &_repo; + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; + public: + DocumentListMessageFactory(const document::DocumentTypeRepo &r) + : _repo(r) {} + }; + class DocumentListReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; + }; + class DocumentSummaryMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; + }; + class DocumentSummaryReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; + }; + class EmptyBucketsMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; + }; + class EmptyBucketsReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; + }; + class GetBucketListMessageFactory : public DocumentMessageFactory { + virtual bool encodeBucketSpace(vespalib::stringref bucketSpace, vespalib::GrowableByteBuffer& buf) const; + virtual string decodeBucketSpace(document::ByteBuffer&) const; + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; + }; + class GetBucketListReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; + }; + class GetBucketStateMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; + }; + class GetBucketStateReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; + }; + class GetDocumentMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; + }; + class GetDocumentReplyFactory : public DocumentReplyFactory { + const document::DocumentTypeRepo &_repo; + DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentReply &msg, vespalib::GrowableByteBuffer &buf) const override; + public: + GetDocumentReplyFactory(const document::DocumentTypeRepo &r) : _repo(r) {} + }; + class MapVisitorMessageFactory : public DocumentMessageFactory { + const document::DocumentTypeRepo &_repo; + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; public: - GetBucketListMessageFactory() = default; + MapVisitorMessageFactory(const document::DocumentTypeRepo &r) : _repo(r) {} }; + class MapVisitorReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; + }; + class PutDocumentMessageFactory : public DocumentMessageFactory { + protected: + const document::DocumentTypeRepo &_repo; + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override { + return decodeMessage<PutDocumentMessage>(this, buf); + } + + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; + public: + void decodeInto(PutDocumentMessage & msg, document::ByteBuffer & buf) const; + PutDocumentMessageFactory(const document::DocumentTypeRepo &r) : _repo(r) {} + }; + class PutDocumentReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; + }; + class RemoveDocumentMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override { + return decodeMessage<RemoveDocumentMessage>(this, buf); + } + + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; + public: + void decodeInto(RemoveDocumentMessage & msg, document::ByteBuffer & buf) const; + }; + class RemoveDocumentReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; + }; + class RemoveLocationMessageFactory : public DocumentMessageFactory { + const document::DocumentTypeRepo &_repo; + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; + public: + RemoveLocationMessageFactory(const document::DocumentTypeRepo &r) : _repo(r) {} + }; + class RemoveLocationReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; + }; + class SearchResultMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; + }; + class SearchResultReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; + }; + class StatBucketMessageFactory : public DocumentMessageFactory { + virtual bool encodeBucketSpace(vespalib::stringref bucketSpace, vespalib::GrowableByteBuffer& buf) const; + virtual string decodeBucketSpace(document::ByteBuffer&) const; + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; + }; + class StatBucketReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; + }; + class StatDocumentMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; + }; + class StatDocumentReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; + }; + class UpdateDocumentMessageFactory : public DocumentMessageFactory { + protected: + const document::DocumentTypeRepo &_repo; + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override { + return decodeMessage<UpdateDocumentMessage>(this, buf); + } + + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; + public: + void decodeInto(UpdateDocumentMessage & msg, document::ByteBuffer & buf) const; + UpdateDocumentMessageFactory(const document::DocumentTypeRepo &r) : _repo(r) {} + }; + class UpdateDocumentReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; + }; + class VisitorInfoMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; + }; + class VisitorInfoReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; + }; + class WrongDistributionReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; + }; + class QueryResultMessageFactory : public DocumentMessageFactory { + protected: + DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; + }; + class QueryResultReplyFactory : public DocumentReplyFactory { + protected: + DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; + bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; + }; + + /////////////////////////////////////////////////////////////////////////// + // + // Utilities + // + /////////////////////////////////////////////////////////////////////////// + + /** + * This is a complement for the vespalib::GrowableByteBuffer.putString() method. + * + * @param in The byte buffer to read from. + * @return The decoded string. + */ + static string decodeString(document::ByteBuffer &in); + + /** + * This is a complement for the vespalib::GrowableByteBuffer.putBoolean() method. + * + * @param in The byte buffer to read from. + * @return The decoded bool. + */ + static bool decodeBoolean(document::ByteBuffer &in); + + /** + * Convenience method to decode a 32-bit int from the given byte buffer. + * + * @param in The byte buffer to read from. + * @return The decoded int. + */ + static int32_t decodeInt(document::ByteBuffer &in); + + /** + * Convenience method to decode a 64-bit int from the given byte buffer. + * + * @param in The byte buffer to read from. + * @return The decoded int. + */ + static int64_t decodeLong(document::ByteBuffer &in); + + /** + * Convenience method to decode a document id from the given byte buffer. + * + * @param in The byte buffer to read from. + * @return The decoded document id. + */ + static document::DocumentId decodeDocumentId(document::ByteBuffer &in); + + /** + * Convenience method to encode a document id to the given byte buffer. + * + * @param id The document id to encode. + * @param out The byte buffer to write to. + */ + static void encodeDocumentId(const document::DocumentId &id, + vespalib::GrowableByteBuffer &out); + static void decodeTasCondition(DocumentMessage & docMsg, document::ByteBuffer & buf); + static void encodeTasCondition(vespalib::GrowableByteBuffer & buf, const DocumentMessage & docMsg); static void doEncodeBucketSpace(vespalib::stringref bucketSpace, vespalib::GrowableByteBuffer& buf); static string doDecodeBucketSpace(document::ByteBuffer&); }; diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-BatchMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-BatchMessage.dat Binary files differdeleted file mode 100644 index c1c43e8081b..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-BatchMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-BatchReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-BatchReply.dat Binary files differdeleted file mode 100644 index be3a9ba1913..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-BatchReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-CreateVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-CreateVisitorMessage.dat Binary files differdeleted file mode 100644 index 4ca8648e702..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-CreateVisitorMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-CreateVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-CreateVisitorReply.dat Binary files differdeleted file mode 100644 index e13917227d1..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-CreateVisitorReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-DestroyVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-DestroyVisitorMessage.dat Binary files differdeleted file mode 100644 index f39b31217e6..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-DestroyVisitorMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-DestroyVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-DestroyVisitorReply.dat Binary files differdeleted file mode 100644 index 1468f027b15..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-DestroyVisitorReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentIgnoredReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentIgnoredReply.dat Binary files differdeleted file mode 100644 index 15a7afe2a59..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentIgnoredReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentListMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentListMessage.dat Binary files differdeleted file mode 100644 index 2d8d12d2704..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentListMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentListReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentListReply.dat Binary files differdeleted file mode 100644 index c8a1cd888f0..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentListReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryMessage-1.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryMessage-1.dat Binary files differdeleted file mode 100644 index 0107dd5f350..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryMessage-1.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryMessage-2.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryMessage-2.dat Binary files differdeleted file mode 100644 index 57187093f28..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryMessage-2.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryMessage-3.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryMessage-3.dat Binary files differdeleted file mode 100644 index 6a516d38d17..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryMessage-3.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryReply.dat Binary files differdeleted file mode 100644 index 16b1e4bc4ef..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-DocumentSummaryReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-EmptyBucketsMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-EmptyBucketsMessage.dat Binary files differdeleted file mode 100644 index b9df278fd7a..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-EmptyBucketsMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-EmptyBucketsReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-EmptyBucketsReply.dat Binary files differdeleted file mode 100644 index 05510dd8c1e..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-EmptyBucketsReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketListMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketListMessage.dat Binary files differdeleted file mode 100644 index fa3de45ac5b..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketListMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketListReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketListReply.dat Binary files differdeleted file mode 100644 index 830994ed785..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketListReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketStateMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketStateMessage.dat Binary files differdeleted file mode 100644 index aa2d206ca3a..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketStateMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketStateReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketStateReply.dat Binary files differdeleted file mode 100644 index 0f91e3759f8..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-GetBucketStateReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-GetDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-GetDocumentMessage.dat Binary files differdeleted file mode 100644 index 3df64ed657f..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-GetDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-GetDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-GetDocumentReply.dat Binary files differdeleted file mode 100644 index c1ad7920a2e..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-GetDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-MapVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-MapVisitorMessage.dat Binary files differdeleted file mode 100644 index 1a8a837ea16..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-MapVisitorMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-MapVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-MapVisitorReply.dat Binary files differdeleted file mode 100644 index 541cd718a66..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-MapVisitorReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-MultiOperationMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-MultiOperationMessage.dat Binary files differdeleted file mode 100644 index 6efc54a6d09..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-MultiOperationMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-MultiOperationReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-MultiOperationReply.dat Binary files differdeleted file mode 100644 index 8ad31a95bd5..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-MultiOperationReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-PutDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-PutDocumentMessage.dat Binary files differdeleted file mode 100644 index 13c25f0b729..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-PutDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-PutDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-PutDocumentReply.dat Binary files differdeleted file mode 100644 index 480544045bb..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-PutDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-1.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-1.dat Binary files differdeleted file mode 100644 index dbf830c9365..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-1.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-2.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-2.dat Binary files differdeleted file mode 100644 index 094143cf78d..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-2.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-3.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-3.dat Binary files differdeleted file mode 100644 index 3341d74052b..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-3.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-4.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-4.dat Binary files differdeleted file mode 100644 index 8aaaefff491..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-4.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-5.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-5.dat Binary files differdeleted file mode 100644 index e66ed1f07d4..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultMessage-5.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultReply.dat Binary files differdeleted file mode 100644 index 003f35d63a7..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-QueryResultReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveDocumentMessage.dat Binary files differdeleted file mode 100644 index 21f2c9b81c4..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveDocumentReply.dat Binary files differdeleted file mode 100644 index bf5db8761e2..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationMessage.dat Binary files differdeleted file mode 100644 index 16850a6aff3..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationMessageGroup.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationMessageGroup.dat Binary files differdeleted file mode 100644 index abd648184d7..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationMessageGroup.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationMessageUser.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationMessageUser.dat Binary files differdeleted file mode 100644 index c2a63cb94c0..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationMessageUser.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationReply.dat Binary files differdeleted file mode 100644 index 752c4dba399..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-RemoveLocationReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-1.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-1.dat Binary files differdeleted file mode 100644 index 988f9fdab1f..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-1.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-2.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-2.dat Binary files differdeleted file mode 100644 index ac277d09643..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-2.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-3.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-3.dat Binary files differdeleted file mode 100644 index 03b49c8a0ac..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-3.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-4.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-4.dat Binary files differdeleted file mode 100644 index d52e574ea44..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-4.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-5.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-5.dat Binary files differdeleted file mode 100644 index e68654e9941..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultMessage-5.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultReply.dat Binary files differdeleted file mode 100644 index cce9c6f8d14..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-SearchResultReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-StatBucketMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-StatBucketMessage.dat Binary files differdeleted file mode 100644 index 1fc2b1cf3c1..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-StatBucketMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-StatBucketReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-StatBucketReply.dat Binary files differdeleted file mode 100644 index 0b98e240018..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-StatBucketReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-UpdateDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-UpdateDocumentMessage.dat Binary files differdeleted file mode 100644 index 93274279f0d..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-UpdateDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-UpdateDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-UpdateDocumentReply.dat Binary files differdeleted file mode 100644 index c7151299366..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-UpdateDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-VisitorInfoMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-VisitorInfoMessage.dat Binary files differdeleted file mode 100644 index 80b44e0c6fd..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-VisitorInfoMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-VisitorInfoReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-VisitorInfoReply.dat Binary files differdeleted file mode 100644 index 57a656c9b2d..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-VisitorInfoReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-WrongDistributionReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-WrongDistributionReply.dat Binary files differdeleted file mode 100644 index 0dbe13225ae..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-WrongDistributionReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-CreateVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-CreateVisitorMessage.dat Binary files differdeleted file mode 100644 index 4ca8648e702..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-CreateVisitorMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-CreateVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-CreateVisitorReply.dat Binary files differdeleted file mode 100644 index e13917227d1..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-CreateVisitorReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-DestroyVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-DestroyVisitorMessage.dat Binary files differdeleted file mode 100644 index f39b31217e6..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-DestroyVisitorMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-DestroyVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-DestroyVisitorReply.dat Binary files differdeleted file mode 100644 index 1468f027b15..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-DestroyVisitorReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-DocumentIgnoredReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-DocumentIgnoredReply.dat Binary files differdeleted file mode 100644 index 15a7afe2a59..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-DocumentIgnoredReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-DocumentListMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-DocumentListMessage.dat Binary files differdeleted file mode 100644 index 2d8d12d2704..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-DocumentListMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-DocumentListReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-DocumentListReply.dat Binary files differdeleted file mode 100644 index c8a1cd888f0..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-DocumentListReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-DocumentSummaryReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-DocumentSummaryReply.dat Binary files differdeleted file mode 100644 index 16b1e4bc4ef..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-DocumentSummaryReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-EmptyBucketsMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-EmptyBucketsMessage.dat Binary files differdeleted file mode 100644 index b9df278fd7a..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-EmptyBucketsMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-EmptyBucketsReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-EmptyBucketsReply.dat Binary files differdeleted file mode 100644 index 05510dd8c1e..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-EmptyBucketsReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-GetBucketListMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-GetBucketListMessage.dat Binary files differdeleted file mode 100644 index fa3de45ac5b..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-GetBucketListMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-GetBucketListReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-GetBucketListReply.dat Binary files differdeleted file mode 100644 index 830994ed785..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-GetBucketListReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-GetBucketStateMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-GetBucketStateMessage.dat Binary files differdeleted file mode 100644 index aa2d206ca3a..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-GetBucketStateMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-GetBucketStateReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-GetBucketStateReply.dat Binary files differdeleted file mode 100644 index 0f91e3759f8..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-GetBucketStateReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-GetDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-GetDocumentMessage.dat Binary files differdeleted file mode 100644 index 3df64ed657f..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-GetDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-GetDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-GetDocumentReply.dat Binary files differdeleted file mode 100644 index c1ad7920a2e..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-GetDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-MapVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-MapVisitorMessage.dat Binary files differdeleted file mode 100644 index 1a8a837ea16..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-MapVisitorMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-MapVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-MapVisitorReply.dat Binary files differdeleted file mode 100644 index 541cd718a66..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-MapVisitorReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-MultiOperationMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-MultiOperationMessage.dat Binary files differdeleted file mode 100644 index 6efc54a6d09..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-MultiOperationMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-MultiOperationReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-MultiOperationReply.dat Binary files differdeleted file mode 100644 index 8ad31a95bd5..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-MultiOperationReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-PutDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-PutDocumentMessage.dat Binary files differdeleted file mode 100644 index 13c25f0b729..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-PutDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-PutDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-PutDocumentReply.dat Binary files differdeleted file mode 100644 index 480544045bb..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-PutDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-QueryResultReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-QueryResultReply.dat Binary files differdeleted file mode 100644 index 003f35d63a7..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-QueryResultReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-RemoveDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-RemoveDocumentMessage.dat Binary files differdeleted file mode 100644 index 21f2c9b81c4..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-RemoveDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-RemoveDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-RemoveDocumentReply.dat Binary files differdeleted file mode 100644 index bf5db8761e2..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-RemoveDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-RemoveLocationMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-RemoveLocationMessage.dat Binary files differdeleted file mode 100644 index 16850a6aff3..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-RemoveLocationMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-RemoveLocationReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-RemoveLocationReply.dat Binary files differdeleted file mode 100644 index 752c4dba399..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-RemoveLocationReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-SearchResultReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-SearchResultReply.dat Binary files differdeleted file mode 100644 index cce9c6f8d14..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-SearchResultReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-StatBucketMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-StatBucketMessage.dat Binary files differdeleted file mode 100644 index 1fc2b1cf3c1..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-StatBucketMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-StatBucketReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-StatBucketReply.dat Binary files differdeleted file mode 100644 index 0b98e240018..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-StatBucketReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-UpdateDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-UpdateDocumentMessage.dat Binary files differdeleted file mode 100644 index 93274279f0d..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-UpdateDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-UpdateDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-UpdateDocumentReply.dat Binary files differdeleted file mode 100644 index c7151299366..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-UpdateDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-VisitorInfoMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-VisitorInfoMessage.dat Binary files differdeleted file mode 100644 index 80b44e0c6fd..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-VisitorInfoMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-VisitorInfoReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-VisitorInfoReply.dat Binary files differdeleted file mode 100644 index 57a656c9b2d..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-VisitorInfoReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-WrongDistributionReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-WrongDistributionReply.dat Binary files differdeleted file mode 100644 index 0dbe13225ae..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-WrongDistributionReply.dat +++ /dev/null diff --git a/documentgen-test/etc/complex/book.sd b/documentgen-test/etc/complex/book.sd index 90dff0982af..872634bf53b 100644 --- a/documentgen-test/etc/complex/book.sd +++ b/documentgen-test/etc/complex/book.sd @@ -44,22 +44,17 @@ search book { } field author type string { bolding: on - # index-to: default, author indexing: summary | index - header } field isbn type string { - # index-to: default, isbn indexing: summary | index } field year type int { indexing: summary | index } field description type string { - # index-to: default, description indexing: summary | index summary: dynamic - body } field myraw type raw { diff --git a/documentgen-test/etc/complex/common.sd b/documentgen-test/etc/complex/common.sd index 017bce4f612..fb49c8b0e52 100644 --- a/documentgen-test/etc/complex/common.sd +++ b/documentgen-test/etc/complex/common.sd @@ -6,10 +6,8 @@ search common { } field title type string { bolding: on - # index-to: default, title indexing: index|summary summary-to: smallsum - body } field mid type int { indexing: attribute|index|summary @@ -33,7 +31,6 @@ search common { indexing: index|summary } field content type raw { - body } annotation date { field exacttime type long { } diff --git a/documentgen-test/pom.xml b/documentgen-test/pom.xml index fdc1b89a438..0d433a3dbd8 100644 --- a/documentgen-test/pom.xml +++ b/documentgen-test/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>documentgen-test</artifactId> <packaging>jar</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>documentgen_test</name> <description>Tests for vespa-documentgen-plugin</description> <dependencies> diff --git a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java index 60d135f6440..deec438a332 100644 --- a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java +++ b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java @@ -789,23 +789,23 @@ public class DocumentGenPluginTest { final Book book = getBook(); assertEquals(book.getMystruct().getD1(), (Double)56.777); assertEquals(book.getMystruct().getCompressionType(), CompressionType.NONE); - assertEquals(book.getBody().getFieldCount(), 3); - assertEquals(book.getHeader().getFieldCount(), 10); + assertEquals(book.getBody().getFieldCount(), 0); + assertEquals(book.getHeader().getFieldCount(), 13); assertEquals(book.getMystruct().getFieldCount(), 4); assertEquals(book.getContent().get(0), 3); assertEquals(book.getContent().get(1), 4); assertEquals(book.getContent().get(2), 5); final Document des = roundtripSerialize(book, typeManagerForBookType()); - assertEquals(des.getBody().getFieldCount(), 3); - assertEquals(des.getHeader().getFieldCount(), 10); + assertEquals(des.getBody().getFieldCount(), 0); + assertEquals(des.getHeader().getFieldCount(), 13); assertEquals(des.getDataType().getName(), "book"); assertEquals(((Raw) des.getFieldValue("content")).getByteBuffer().get(0), 3); assertEquals(((Raw) des.getFieldValue("content")).getByteBuffer().get(1), 4); assertEquals(((Raw) des.getFieldValue("content")).getByteBuffer().get(2), 5); assertEquals(des.getFieldValue("author").toString(), "Herman Melville"); assertEquals(des.getFieldValue("title").toString(), "Moby Dick - Or The Whale"); - assertEquals(des.getBody().getFieldValue("title").toString(), "Moby Dick - Or The Whale"); - assertNull(des.getHeader().getFieldValue("title")); + assertEquals(des.getHeader().getFieldValue("title").toString(), "Moby Dick - Or The Whale"); + assertNull(des.getBody().getFieldValue("title")); assertEquals(des.getHeader().getFieldValue("author").toString(), "Herman Melville"); assertNull(des.getBody().getFieldValue("author")); diff --git a/fat-model-dependencies/pom.xml b/fat-model-dependencies/pom.xml index 3dc6e1eee79..1cefc020d26 100644 --- a/fat-model-dependencies/pom.xml +++ b/fat-model-dependencies/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>fat-model-dependencies</artifactId> <packaging>pom</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/fileacquirer/pom.xml b/fileacquirer/pom.xml index 9980b6bd84b..72529eaab3e 100644 --- a/fileacquirer/pom.xml +++ b/fileacquirer/pom.xml @@ -6,11 +6,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>fileacquirer</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>junit</groupId> diff --git a/filedistribution/pom.xml b/filedistribution/pom.xml index 7f8e63892b2..133268d5872 100644 --- a/filedistribution/pom.xml +++ b/filedistribution/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>filedistribution</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> diff --git a/flags/pom.xml b/flags/pom.xml index fc38676ff20..c1e9eca20ab 100644 --- a/flags/pom.xml +++ b/flags/pom.xml @@ -8,13 +8,13 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>flags</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>${project.artifactId}</name> <description>Feature flags</description> diff --git a/fsa/pom.xml b/fsa/pom.xml index d8a63b6087d..307b871c44a 100644 --- a/fsa/pom.xml +++ b/fsa/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>fsa</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <description> This is the FSA library and tools, an implementation of finite state automata (FSA) and related algorithms. diff --git a/indexinglanguage/pom.xml b/indexinglanguage/pom.xml index 00b3413ae1f..74c2f701a67 100644 --- a/indexinglanguage/pom.xml +++ b/indexinglanguage/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>indexinglanguage</artifactId> <packaging>jar</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>indexinglanguage</name> <description>Interpreter for the Indexing Language</description> <dependencies> diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java index fddbd211e27..50dd7611bb0 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java @@ -179,7 +179,7 @@ public abstract class Expression extends Selectable { /** Creates an expression with simple lingustics for testing */ @SuppressWarnings("deprecation") public static Expression fromString(String expression) throws ParseException { - return fromString(expression, new SimpleLinguistics(false)); + return fromString(expression, new SimpleLinguistics()); } public static Expression fromString(String expression, Linguistics linguistics) throws ParseException { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java index 7addca75d2f..320c47103aa 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java @@ -91,7 +91,7 @@ public final class ScriptExpression extends ExpressionList<StatementExpression> /** Creates an expression with simple lingustics for testing */ @SuppressWarnings("deprecation") public static ScriptExpression fromString(String expression) throws ParseException { - return fromString(expression, new SimpleLinguistics(false)); + return fromString(expression, new SimpleLinguistics()); } public static ScriptExpression fromString(String expression, Linguistics linguistics) throws ParseException { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java index 16d069d84ec..cf1e808946d 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java @@ -92,7 +92,7 @@ public final class StatementExpression extends ExpressionList<Expression> { /** Creates an expression with simple lingustics for testing */ @SuppressWarnings("deprecation") public static StatementExpression fromString(String expression) throws ParseException { - return fromString(expression, new SimpleLinguistics(false)); + return fromString(expression, new SimpleLinguistics()); } public static StatementExpression fromString(String expression, Linguistics linguistics) throws ParseException { diff --git a/jaxrs_client_utils/pom.xml b/jaxrs_client_utils/pom.xml index 43fbc66a9e6..636fbab7bb0 100644 --- a/jaxrs_client_utils/pom.xml +++ b/jaxrs_client_utils/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>jaxrs_client_utils</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jaxrs_utils/pom.xml b/jaxrs_utils/pom.xml index 1753d016ec3..a75182b0966 100644 --- a/jaxrs_utils/pom.xml +++ b/jaxrs_utils/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>jaxrs_utils</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc-security-filters/pom.xml b/jdisc-security-filters/pom.xml index 5e8356e94f1..867a32cc170 100644 --- a/jdisc-security-filters/pom.xml +++ b/jdisc-security-filters/pom.xml @@ -7,12 +7,12 @@ <modelVersion>4.0.0</modelVersion> <artifactId>jdisc-security-filters</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <dependencies> diff --git a/jdisc_core/CMakeLists.txt b/jdisc_core/CMakeLists.txt index b56d6ce1e5b..032eacfadb6 100644 --- a/jdisc_core/CMakeLists.txt +++ b/jdisc_core/CMakeLists.txt @@ -1,4 +1,2 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. install_fat_java_artifact(jdisc_core) - -install(FILES src/main/perl/vespa-jdisc-logfmt.1 DESTINATION man/man1) diff --git a/jdisc_core/pom.xml b/jdisc_core/pom.xml index bbfe750b2a5..d802e0585dc 100644 --- a/jdisc_core/pom.xml +++ b/jdisc_core/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>jdisc_core</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> <dependencies> @@ -47,6 +47,7 @@ <version>1.2</version> </dependency> <dependency> + <!-- TODO: mockito-all is deprecated (includes old hamcrest). Use mockito-core if possible --> <groupId>org.mockito</groupId> <artifactId>mockito-all</artifactId> <scope>test</scope> @@ -112,6 +113,12 @@ <groupId>org.apache.felix</groupId> <artifactId>org.apache.felix.framework</artifactId> <scope>compile</scope> + <exclusions> + <exclusion> + <groupId>org.codehaus.mojo</groupId> + <artifactId>animal-sniffer-annotations</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.felix</groupId> @@ -232,6 +239,7 @@ <configuration> <executable>java</executable> <arguments> + <argument>--add-opens=java.base/java.net=ALL-UNNAMED</argument> <argument>-classpath</argument> <classpath /> <argument>com.yahoo.jdisc.core.ExportPackages</argument> @@ -308,39 +316,6 @@ </plugin> </plugins> </build> - <profiles> - <profile> - <id>test-logfmt</id> - <activation> - <property> - <name>!skipTests</name> - </property> - </activation> - <build> - <plugins> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>exec-maven-plugin</artifactId> - <executions> - <execution> - <id>test-logfmt</id> - <phase>test</phase> - <goals> - <goal>exec</goal> - </goals> - <configuration> - <executable>${project.basedir}/src/test/perl/vespa-jdisc-logfmt-test.sh</executable> - <arguments> - <argument>${project.basedir}/src/main/perl/vespa-jdisc-logfmt</argument> - </arguments> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - </profiles> <properties> <exportPackagesFile>${project.build.directory}/classes/exportPackages.properties</exportPackagesFile> </properties> diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/HeaderFields.java b/jdisc_core/src/main/java/com/yahoo/jdisc/HeaderFields.java index 500b1258735..5c35bed2cec 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/HeaderFields.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/HeaderFields.java @@ -8,7 +8,7 @@ import java.util.*; /** * This is an encapsulation of the header fields that belong to either a {@link Request} or a {@link Response}. It is * a multimap from String to String, with some additional methods for convenience. The keys of this map are compared by - * ignoring their case, so that <tt>get("foo")</tt> returns the same entry as <tt>get("FOO")</tt>. + * ignoring their case, so that <code>get("foo")</code> returns the same entry as <code>get("FOO")</code>. * * @author Simon Thoresen Hult */ @@ -121,7 +121,7 @@ public class HeaderFields implements Map<String, List<String>> { /** * <p>Adds all the entries of the given map to this. This is the same as calling {@link #add(String, List)} for each - * entry in <tt>values</tt>.</p> + * entry in <code>values</code>.</p> * * @param values The values to be added to this. */ @@ -137,8 +137,8 @@ public class HeaderFields implements Map<String, List<String>> { * * @param key The key of the entry to put. * @param value The value to put. - * @return The previous value associated with <tt>key</tt>, or <tt>null</tt> if there was no mapping for - * <tt>key</tt>. + * @return The previous value associated with <code>key</code>, or <code>null</code> if there was no mapping for + * <code>key</code>. */ public List<String> put(String key, String value) { ArrayList<String> list = new ArrayList<String>(1); diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java b/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java index 2a1967abd13..061d803b978 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/Request.java @@ -120,7 +120,6 @@ public class Request extends AbstractResource { * * @return The container instance. */ - // TODO: Vespa 7 remove. public Container container() { return parent != null ? parent.container() : container; } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingMatch.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingMatch.java index 7318b1b38ae..98b0ef870e1 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingMatch.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingMatch.java @@ -22,19 +22,6 @@ public class BindingMatch<T> { * * @param match The match information for this instance. * @param target The target of this match. - * @throws NullPointerException If any argument is null. - * @deprecated use BindingMatch(UriPattern.Match match, T target, UriPattern matched) - */ - @Deprecated - public BindingMatch(UriPattern.Match match, T target) { - this(match, target, null); - } - - /** - * <p>Constructs a new instance of this class.</p> - * - * @param match The match information for this instance. - * @param target The target of this match. * @param matched The matched URI pattern * @throws NullPointerException If any argument is null. */ @@ -62,7 +49,7 @@ public class BindingMatch<T> { * empty string. This method never returns null.</p> * * @param idx The index of the group to return. - * @return The (possibly empty) substring captured by the group during matching, never <tt>null</tt>. + * @return The (possibly empty) substring captured by the group during matching, never <code>null</code>. * @throws IndexOutOfBoundsException If there is no group in the match with the given index. */ public String group(int idx) { diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/MetricConsumer.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/MetricConsumer.java index d9bf67b530f..f060f1840ff 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/MetricConsumer.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/MetricConsumer.java @@ -39,7 +39,7 @@ import java.util.Map; public interface MetricConsumer { /** - * <p>Consume a call to <tt>Metric.set(String, Number, Metric.Context)</tt>.</p> + * <p>Consume a call to <code>Metric.set(String, Number, Metric.Context)</code>.</p> * * @param key The name of the metric to modify. * @param val The value to assign to the named metric. @@ -48,7 +48,7 @@ public interface MetricConsumer { public void set(String key, Number val, Metric.Context ctx); /** - * <p>Consume a call to <tt>Metric.add(String, Number, Metric.Context)</tt>.</p> + * <p>Consume a call to <code>Metric.add(String, Number, Metric.Context)</code>.</p> * * @param key The name of the metric to modify. * @param val The value to add to the named metric. @@ -57,9 +57,9 @@ public interface MetricConsumer { public void add(String key, Number val, Metric.Context ctx); /** - * <p>Creates a <tt>Metric.Context</tt> object that encapsulates the given properties. The returned Context object - * will be passed along every future call to <tt>set(String, Number, Metric.Context)</tt> and - * <tt>add(String, Number, Metric.Context)</tt> where the properties match those given here.</p> + * <p>Creates a <code>Metric.Context</code> object that encapsulates the given properties. The returned Context object + * will be passed along every future call to <code>set(String, Number, Metric.Context)</code> and + * <code>add(String, Number, Metric.Context)</code> where the properties match those given here.</p> * * @param properties The properties to incorporate in the context. * @return The created context. diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/OsgiFramework.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/OsgiFramework.java index 85d745d7aaf..5abc95cc610 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/OsgiFramework.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/OsgiFramework.java @@ -18,9 +18,9 @@ public interface OsgiFramework { /** * <p>Installs a bundle from the specified location. The specified location identifier will be used as the identity - * of the bundle. If a bundle containing the same location identifier is already installed, the <tt>Bundle</tt> + * of the bundle. If a bundle containing the same location identifier is already installed, the <code>Bundle</code> * object for that bundle is returned. All bundles listed in the {@link OsgiHeader#PREINSTALL_BUNDLE} manifest - * header are also installed. The bundle at index 0 of the returned list matches the <tt>bundleLocation</tt> + * header are also installed. The bundle at index 0 of the returned list matches the <code>bundleLocation</code> * argument.</p> * * <p><b>NOTE:</b> When this method installs more than one bundle, <em>AND</em> one of those bundles throw an @@ -29,8 +29,8 @@ public interface OsgiFramework { * BundleInstallationException} that contains the list of successfully installed bundles.</p> * * <p>It would be preferable if this method was exception-safe (that it would roll-back all installed bundles in the - * case of an exception), but that can not be implemented thread-safely since an <tt>Application</tt> may choose to - * install bundles concurrently through any available <tt>BundleContext</tt>.</p> + * case of an exception), but that can not be implemented thread-safely since an <code>Application</code> may choose to + * install bundles concurrently through any available <code>BundleContext</code>.</p> * * @param bundleLocation The location identifier of the bundle to install. * @return The list of Bundle objects installed, the object at index 0 matches the given location. @@ -41,7 +41,7 @@ public interface OsgiFramework { List<Bundle> installBundle(String bundleLocation) throws BundleException; /** - * <p>Starts the given {@link Bundle}s. The parameter <tt>privileged</tt> tells the framework whether or not + * <p>Starts the given {@link Bundle}s. The parameter <code>privileged</code> tells the framework whether or not * privileges are available, and is checked against the {@link OsgiHeader#PRIVILEGED_ACTIVATOR} header of each * Bundle being started. Any bundle that is a fragment is silently ignored.</p> * @@ -63,10 +63,10 @@ public interface OsgiFramework { /** * <p>Returns the BundleContext of this framework's system bundle. The returned BundleContext can be used by the - * caller to act on behalf of this bundle. This method may return <tt>null</tt> if it has no valid + * caller to act on behalf of this bundle. This method may return <code>null</code> if it has no valid * BundleContext.</p> * - * @return A <tt>BundleContext</tt> for the system bundle, or <tt>null</tt>. + * @return A <code>BundleContext</code> for the system bundle, or <code>null</code>. * @throws SecurityException If the caller does not have the appropriate permissions. */ BundleContext bundleContext(); diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/ResourcePool.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/ResourcePool.java index 279d99f63b7..ccbd1693b21 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/ResourcePool.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/ResourcePool.java @@ -87,8 +87,8 @@ public final class ResourcePool extends AbstractResource implements AutoCloseabl * destroyed, it will release the main reference to the resource (by calling {@link SharedResource#release()}).</p> * * @param t The SharedResource to add. - * @param <T> The class of parameter <tt>t</tt>. - * @return The parameter <tt>t</tt>, to allow inlined calls to this function. + * @param <T> The class of parameter <code>t</code>. + * @return The parameter <code>t</code>, to allow inlined calls to this function. */ public <T extends SharedResource> T add(T t) { try { @@ -147,8 +147,8 @@ public final class ResourcePool extends AbstractResource implements AutoCloseabl * {@link SharedResource#refer()} call.</p> * * @param t The SharedResource to retain and add. - * @param <T> The class of parameter <tt>t</tt>. - * @return The parameter <tt>t</tt>, to allow inlined calls to this function. + * @param <T> The class of parameter <code>t</code>. + * @return The parameter <code>t</code>, to allow inlined calls to this function. */ public <T extends SharedResource> T retain(T t) { resources.add(t.refer()); diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/UriPattern.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/UriPattern.java index 89cb6412181..0e6e5d28260 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/UriPattern.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/UriPattern.java @@ -40,7 +40,7 @@ public class UriPattern implements Comparable<UriPattern> { private final int priority; /** - * <p>Creates a new instance of this class that represents the given pattern string, with a priority of <tt>0</tt>. + * <p>Creates a new instance of this class that represents the given pattern string, with a priority of <code>0</code>. * The input string must be on the form <code><scheme>://<host>[:<port>]<path></code>, where * '*' can be used as a wildcard character at any position.</p> * @@ -207,7 +207,7 @@ public class UriPattern implements Comparable<UriPattern> { * the empty string. This method never returns null.</p> * * @param idx The index of the group to return. - * @return The (possibly empty) substring captured by the group during matching, never <tt>null</tt>. + * @return The (possibly empty) substring captured by the group during matching, never <code>null</code>. * @throws IndexOutOfBoundsException If there is no group in the match with the given index. */ public String group(int idx) { diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/package-info.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/package-info.java index 9308bc4d17a..c856712e1af 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/package-info.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/package-info.java @@ -148,5 +148,4 @@ $ sudo jdisc_start target/myapp.jar * @see com.yahoo.jdisc.handler * @see com.yahoo.jdisc.service */ -@com.yahoo.api.annotations.PublicApi package com.yahoo.jdisc.application; diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/client/package-info.java b/jdisc_core/src/main/java/com/yahoo/jdisc/client/package-info.java index e0a17245c1f..c328cc1e382 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/client/package-info.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/client/package-info.java @@ -5,5 +5,4 @@ * * @see com.yahoo.jdisc.client.ClientApplication */ -@com.yahoo.api.annotations.PublicApi package com.yahoo.jdisc.client; diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java index e8e6373f2ed..7a09690b35d 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java @@ -13,7 +13,6 @@ import java.io.FileInputStream; import java.io.FileWriter; import java.io.IOException; import java.nio.file.Files; -import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; import java.util.Properties; @@ -36,45 +35,45 @@ public class ExportPackages { if (!fileName.endsWith(PROPERTIES_FILE)) { throw new IllegalArgumentException("Expected '" + PROPERTIES_FILE + "', got '" + fileName + "'."); } - StringBuilder out = new StringBuilder(); - out.append(getSystemPackages()).append(",") - .append("com.sun.security.auth,") - .append("com.sun.security.auth.module,") - .append("com.sun.management,") - .append("com.yahoo.jdisc,") - .append("com.yahoo.jdisc.application,") - .append("com.yahoo.jdisc.handler,") - .append("com.yahoo.jdisc.service,") - .append("com.yahoo.jdisc.statistics,") - .append("javax.inject;version=1.0.0,") // Included in guice, but not exported. Needed by container-jersey. - .append("org.aopalliance.intercept,") - .append("org.aopalliance.aop,") - .append("sun.misc,") - .append("sun.net.util,") - .append("sun.security.krb5,") - - // TODO: remove for Vespa 7 (xml-apis:xml-apis:1.4.01 is not a bundle, but exposed from system classpath on Java 9) - .append("org.w3c.dom.bootstrap,") - .append("org.w3c.dom.css,") - .append("org.w3c.dom.events,") - .append("org.w3c.dom.html,") - .append("org.w3c.dom.ls,") - .append("org.w3c.dom.ranges,") - .append("org.w3c.dom.stylesheets,") - .append("org.w3c.dom.traversal,") - .append("org.w3c.dom.views"); - - for (int i = 1; i < args.length; ++i) { - out.append(",").append(getExportedPackages(args[i])); - } + String exportPackages = getExportPackages(args); Properties props = new Properties(); - props.setProperty(EXPORT_PACKAGES, out.toString()); + props.setProperty(EXPORT_PACKAGES, exportPackages); try (FileWriter writer = new FileWriter(new File(fileName))) { props.store(writer, "generated by " + ExportPackages.class.getName()); } } + private static String getExportPackages(String[] jars) throws IOException { + StringBuilder out = new StringBuilder(); + out.append(getSystemPackages()).append(", ") + .append("com.yahoo.jdisc, ") + .append("com.yahoo.jdisc.application, ") + .append("com.yahoo.jdisc.handler, ") + .append("com.yahoo.jdisc.service, ") + .append("com.yahoo.jdisc.statistics, ") + .append("javax.inject;version=1.0.0, ") // Included in guice, but not exported. Needed by container-jersey. + .append("org.aopalliance.intercept, ") + .append("org.aopalliance.aop"); + + for (int i = 1; i < jars.length; ++i) { + out.append(", ").append(getExportedPackages(jars[i])); + } + + //TODO: temporary additions for backwards compatibility with Vespa 6. Remove when all apps have been built with 7 + out.append(", ") + .append("javax.annotation, ") + .append("javax.activation, ") + .append("javax.xml.bind.annotation.adapters, ") + .append("javax.xml.bind.annotation, ") + .append("javax.xml.bind.attachment, ") + .append("javax.xml.bind.helpers, ") + .append("javax.xml.bind.util, ") + .append("javax.xml.bind"); + + return out.toString(); + } + public static String readExportProperty() { Properties props = new Properties(); try { diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/FelixParams.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/FelixParams.java index 10dbd09792f..9b877f68efd 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/FelixParams.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/FelixParams.java @@ -12,10 +12,21 @@ import java.util.Map; */ public class FelixParams { - private final StringBuilder exportPackages = new StringBuilder(ExportPackages.readExportProperty()); + private final StringBuilder exportPackages; private String cachePath = null; private boolean loggerEnabled = true; + public FelixParams() { + this(ExportPackages.readExportProperty()); + } + + // For testing only + // Needed because the set of system packages is no longer constant between JVM invocations, + // since Felix 6 and JDK 9. + FelixParams(String exportPackages) { + this.exportPackages = new StringBuilder(exportPackages); + } + public FelixParams exportPackage(String pkg) { exportPackages.append(",").append(pkg); return this; diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogHandler.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogHandler.java index e72df8fc7e1..d774b857b49 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogHandler.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogHandler.java @@ -6,7 +6,9 @@ import org.osgi.framework.Bundle; import org.osgi.framework.ServiceReference; import org.osgi.service.log.LogService; +import java.util.Dictionary; import java.util.HashMap; +import java.util.Hashtable; import java.util.Map; import java.util.logging.Handler; import java.util.logging.Level; @@ -160,5 +162,11 @@ class OsgiLogHandler extends Handler { public int compareTo(Object o) { return 0; } + + @Override + public Dictionary<String, Object> getProperties() { + return new Hashtable<>(); + } + } } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/CallableRequestDispatch.java b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/CallableRequestDispatch.java index cfbd4edcf8f..135a56f2d6b 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/CallableRequestDispatch.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/CallableRequestDispatch.java @@ -8,8 +8,8 @@ import java.util.concurrent.Callable; /** * This is a convenient subclass of {@link RequestDispatch} that implements the {@link Callable} interface. This * should be used in place of {@link RequestDispatch} if you intend to schedule its execution. Because {@link #call()} - * does not return until a {@link Response} becomes available, you can use the <tt>Future</tt> return value of - * <tt>ExecutorService.submit(Callable)</tt> to wait for it. + * does not return until a {@link Response} becomes available, you can use the <code>Future</code> return value of + * <code>ExecutorService.submit(Callable)</code> to wait for it. * * @author Simon Thoresen Hult */ diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/CallableResponseDispatch.java b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/CallableResponseDispatch.java index 0471e05a5f9..e92edf15bba 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/CallableResponseDispatch.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/CallableResponseDispatch.java @@ -9,7 +9,7 @@ import java.util.concurrent.Callable; * This is a convenient subclass of {@link ResponseDispatch} that implements the {@link Callable} interface. This * should be used in place of {@link ResponseDispatch} if you intend to schedule its execution. Because {@link #call()} * does not return until the entirety of the {@link Response} and its content have been consumed, you can use the - * <tt>Future</tt> return value of <tt>ExecutorService.submit(Callable)</tt> to wait for it to complete. + * <code>Future</code> return value of <code>ExecutorService.submit(Callable)</code> to wait for it to complete. * * @author Simon Thoresen Hult */ diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/FutureConjunction.java b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/FutureConjunction.java index 4380ad0cbd2..c48798ca2ab 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/FutureConjunction.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/FutureConjunction.java @@ -11,7 +11,7 @@ import java.util.concurrent.*; /** * <p>This class implements a Future<Boolean> that is conjunction of zero or more other Future<Boolean>s, - * i.e. it evaluates to <tt>true</tt> if, and only if, all its operands evaluate to <tt>true</tt>. To use this class, + * i.e. it evaluates to <code>true</code> if, and only if, all its operands evaluate to <code>true</code>. To use this class, * simply create an instance of it and add operands to it using the {@link #addOperand(ListenableFuture)} method.</p> * TODO: consider rewriting usage of FutureConjunction to use CompletableFuture instead. * diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/RequestDispatch.java b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/RequestDispatch.java index 211c5474fe6..3dc0961cdcb 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/RequestDispatch.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/RequestDispatch.java @@ -73,7 +73,7 @@ public abstract class RequestDispatch implements ListenableFuture<Response>, Res /** * <p>This methods calls {@link #newRequest()} to create a new {@link Request}, and then calls {@link - * Request#connect(ResponseHandler)} on that. This method uses a <tt>finally</tt> block to make sure that the + * Request#connect(ResponseHandler)} on that. This method uses a <code>finally</code> block to make sure that the * Request is always {@link Request#release() released}.</p> * * @return The ContentChannel to write the Request's content to. @@ -98,7 +98,7 @@ public abstract class RequestDispatch implements ListenableFuture<Response>, Res /** * <p>This method calls {@link #connect()} to establish a {@link ContentChannel} for the {@link Request}, and then * iterates through all the ByteBuffers returned by {@link #requestContent()} and writes them to that - * ContentChannel. This method uses a <tt>finally</tt> block to make sure that the ContentChannel is always {@link + * ContentChannel. This method uses a <code>finally</code> block to make sure that the ContentChannel is always {@link * ContentChannel#close(CompletionHandler) closed}.</p> * * <p>The returned Future will wait for all CompletionHandlers associated with the Request have been completed, and diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ResponseDispatch.java b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ResponseDispatch.java index e5e6b752716..22a77968dfd 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ResponseDispatch.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ResponseDispatch.java @@ -82,7 +82,7 @@ public abstract class ResponseDispatch extends ForwardingListenableFuture<Boolea /** * <p>This method calls {@link #connect(ResponseHandler)} to establish a {@link ContentChannel} for the {@link * Response}, and then iterates through all the ByteBuffers returned by {@link #responseContent()} and writes them - * to that ContentChannel. This method uses a <tt>finally</tt> block to make sure that the ContentChannel is always + * to that ContentChannel. This method uses a <code>finally</code> block to make sure that the ContentChannel is always * {@link ContentChannel#close(CompletionHandler) closed}.</p> * <p>The returned Future will wait for all CompletionHandlers associated with the Response have been * completed.</p> diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ThreadedRequestHandler.java b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ThreadedRequestHandler.java index a2bea1566de..6e2895f118b 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ThreadedRequestHandler.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/handler/ThreadedRequestHandler.java @@ -126,7 +126,7 @@ public abstract class ThreadedRequestHandler extends AbstractRequestHandler { /** * <p>Implement this method if you want to access the {@link Request}'s content using a {@link ContentInputStream}. * If you do not override this method, it will dispatch a {@link Response} to the {@link ResponseHandler} with a - * <tt>Response.Status.NOT_IMPLEMENTED</tt> status.</p> + * <code>Response.Status.NOT_IMPLEMENTED</code> status.</p> * * @param request The Request to handle. * @param responseHandler The handler to pass the corresponding {@link Response} to. diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/package-info.java b/jdisc_core/src/main/java/com/yahoo/jdisc/package-info.java index ff8dd242587..d1d842fa3f6 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/package-info.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/package-info.java @@ -50,6 +50,5 @@ * @see com.yahoo.jdisc.service * @see com.yahoo.jdisc.test */ -// TODO: Vespa 7 remove internal classes (at least Container) out of this PublicApi package. @com.yahoo.api.annotations.PublicApi package com.yahoo.jdisc; diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/test/ServerProviderConformanceTest.java b/jdisc_core/src/main/java/com/yahoo/jdisc/test/ServerProviderConformanceTest.java index e7039e85e5e..7e913456074 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/test/ServerProviderConformanceTest.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/test/ServerProviderConformanceTest.java @@ -49,7 +49,7 @@ public abstract class ServerProviderConformanceTest { private static final int NUM_RUNS_EACH_TEST = 10; /** - * <p>This interface declares the adapter between the general conformance test and an actual <tt>ServerProvider</tt> + * <p>This interface declares the adapter between the general conformance test and an actual <code>ServerProvider</code> * implementation. Every test runs as follows:</p> * <ol> * <li>{@link #newConfigModule()} is called to bind server-specific configuration.</li> @@ -59,7 +59,7 @@ public abstract class ServerProviderConformanceTest { * <li>{@link #validateResponse(Object)} is called once per call to {@link #executeRequest(Object, boolean)}.</li> * </ol> * - * @param <T> The <tt>ServerProvider</tt> under test. + * @param <T> The <code>ServerProvider</code> under test. * @param <U> An object that represents a remote client that can connect to the server. * @param <V> An object that holds the response generated by the client when executing a request. */ @@ -80,7 +80,7 @@ public abstract class ServerProviderConformanceTest { /** * <p>An instance of this exception is thrown within the conformance tests that imply that they will throw an - * exception. If your <tt>ServerProvider</tt> is capable of exposing such information, then this class is what you + * exception. If your <code>ServerProvider</code> is capable of exposing such information, then this class is what you * need to look for in the output.</p> */ public static class ConformanceException extends RuntimeException { diff --git a/jdisc_core/src/main/perl/vespa-jdisc-logfmt b/jdisc_core/src/main/perl/vespa-jdisc-logfmt deleted file mode 100755 index 655fd8da24c..00000000000 --- a/jdisc_core/src/main/perl/vespa-jdisc-logfmt +++ /dev/null @@ -1,277 +0,0 @@ -#!/usr/bin/env perl -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -sub findhome { - # Try the VESPA_HOME env variable - return $ENV{'VESPA_HOME'} if defined $ENV{'VESPA_HOME'}; - if ( $0 =~ m{(.*)/bin[^/]*/[^/]*logfmt[^/]*$} ) { - return $1; - } - return "/opt/vespa"; -} - -my $VESPA_HOME = findhome(); - -# TODO: Remove on Vespa 7 - -use 5.006_001; -use strict; -use warnings; - -use File::Basename; -use Getopt::Long qw(:config no_ignore_case); - -my %showflags = ( - time => 1, - fmttime => 1, - msecs => 1, - usecs => 0, - host => 0, - level => 1, - pid => 0, - service => 1, - component => 1, - message => 1 - ); - -my %levelflags = ( - error => 1, - warning => 1, - info => 1, - debug => 0, - unknown => 0 - ); - -# Do not buffer the output -$| = 1; - -my $compore; -my $msgtxre; -my $onlypid; -my $onlysvc; -my $onlyhst; - -my $shortsvc; -my $shortcmp; - -my @optlevels; -my @optaddlevels; -my @optshow; -my $optaddlevels; -my $optlevels; -my $optfollow; -my $optnldequote; -my $opthelp = ''; - -my $bad = 0; - -GetOptions ('level|l=s' => \@optlevels, - 'add-level|L=s' => \@optaddlevels, - 'service|S=s' => \$onlysvc, - 'show|s=s' => \@optshow, - 'pid|p=s' => \$onlypid, - 'component|c=s' => \$compore, - 'message|m=s' => \$msgtxre, - 'help|h' => \$opthelp, - 'follow|f' => \$optfollow, - 'nldequote|N' => \$optnldequote, - 'host|H=s' => \$onlyhst, - 'truncateservice|ts' => \$shortsvc, - 'truncatecomponent|tc|t' => \$shortcmp, - ) or $bad=1; - -if ( @ARGV == 0 and ! -p STDIN) { - push(@ARGV, "$VESPA_HOME/logs/jdisc_core/jdisc_core.log"); -} - -if ( $optfollow ) { - my $filearg = ""; - if ( @ARGV > 1 ) { - print STDERR "ERROR: Cannot follow more than one file\n\n"; - $bad=1; - } else { - $filearg = shift @ARGV if (@ARGV > 0); - open(STDIN, "tail -F $filearg |") - or die "cannot open 'tail -F $filearg' as input pipe\n"; - } -} - -$optaddlevels = join(",", @optaddlevels ); -if ( $optaddlevels ) { - my @l = split(/,/, $optaddlevels); - my $l; - foreach $l ( @l ) { - $levelflags{$l} = 0; - } -} - -if ( $opthelp || $bad ) { - print STDERR "Usage: ", basename($0), " [options] [inputfile ...]\n", - "Options:\n", - " -l LEVELLIST\t--level=LEVELLIST\tselect levels to include\n", - " -L LEVELLIST\t--add-level=LEVELLIST\tdefine extra levels\n", - " -s FIELDLIST\t--show=FIELDLIST\tselect fields to print\n", - " -p PID\t--pid=PID\t\tselect messages from given PID\n", - " -S SERVICE\t--service=SERVICE\tselect messages from given SERVICE\n", - " -H HOST\t--host=HOST\t\tselect messages from given HOST\n", - " -c REGEX\t--component=REGEX\tselect components matching REGEX\n", - " -m REGEX\t--message=REGEX\t\tselect message text matching REGEX\n", - " -f\t\t--follow\t\tinvoke tail -F to follow input file\n", - " -N\t\t--nldequote\t\tdequote newlines in message text field\n", - " -t\t--tc\t--truncatecomponent\tchop component to 15 chars\n", - " \t--ts\t--truncateservice\tchop service to 9 chars\n", - "\n", - "FIELDLIST is comma separated, available fields:\n", - "\t time fmttime msecs usecs host level pid service component message\n", - "Available levels for LEVELLIST:\n", - "\t ", join(" ", sort keys(%levelflags)), "\n", - "for both lists, use 'all' for all possible values, and -xxx to disable xxx.\n"; - exit $bad; -} - -$optlevels = join(",", @optlevels ); -if ( $optlevels ) { - my $k; - unless ( $optlevels =~ s/^\+// or $optlevels =~ m/^-/ ) { - $levelflags{$_} = 0 foreach ( keys %levelflags ); - } - my @l = split(/,|(?=-)/, $optlevels); - my $l; - foreach $l ( @l ) { - my $v = 1; - my $minus = ""; - if ( $l =~ s/^-// ) { $v = 0; $minus = "-"; } - if ( $l eq "all" ) { - foreach $k ( keys %levelflags ) { - $levelflags{$k} = $v; - } - } elsif ( defined $levelflags{$l} ) { - $levelflags{$l} = $v; - } else { - print STDERR "bad level option '$minus$l'\n"; - exit 1; - } - } -} - -my $optshow; -$optshow = join(",", @optshow ); -if ( $optshow ) { - my $k; - unless ( $optshow =~ s/^\+// or $optshow =~ m/^-/ ) { - $showflags{$_} = 0 foreach ( keys %showflags ); - } - my @l = split(/,|(?=-)/, $optshow); - my $l; - foreach $l ( @l ) { - my $v = 1; - my $minus = ""; - if ( $l =~ s/^-// ) { $v = 0; $minus = "-"; } - if ( $l eq "all" ) { - foreach $k ( keys %showflags ) { - $showflags{$k} = $v; - } - } elsif ( defined $showflags{$l} ) { - $showflags{$l} = $v; - } else { - print STDERR "bad show option '$minus$l'\n"; - exit 1; - } - } -} - -while (<>) { - chomp; - if ( /^ - (\d+)\.?(\d*) # seconds, optional fractional seconds - \t - ([^\t]*) # host - \t - (\d+\/?\d*|\-\/\d+) # pid, optional tid - \t - ([^\t]*) # servicename - \t - ([^\t]*) # componentname - \t - (\w+) # level - \t - (.*) # message text - $/x ) - { - my $secs = $1; - my $usec = $2 . "000000"; # make sure we have atleast 6 digits - my $host = $3; - my $pidn = $4; - my $svcn = $5; - my $comp = $6; - my $levl = $7; - my $msgt = $8; - - if ( ! defined $levelflags{$levl} ) { - print STDERR "Warning: unknown level '$levl' in input\n"; - $levelflags{$levl} = 1; - } - next unless ( $levelflags{$levl} ); - - if ($compore && $comp !~ m/$compore/o) { next; } - if ($msgtxre && $msgt !~ m/$msgtxre/o) { next; } - if ($onlypid && $pidn ne $onlypid) { next; } - if ($onlysvc && $svcn ne $onlysvc) { next; } - if ($onlyhst && $host ne $onlyhst) { next; } - - $levl = "\U$levl"; - - my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday); - ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday)=localtime($secs); - my $datestr = sprintf("%04d-%02d-%02d", - 1900+$year, 1+$mon, $mday); - my $timestr = sprintf("%02d:%02d:%02d", - $hour, $min, $sec); - - if ( $showflags{"time"} || $showflags{"fmttime"} ) { - if ($showflags{"fmttime"} ) { - print "[$datestr $timestr"; - if ( $showflags{"usecs"} ) { - printf ".%.6s", $usec; - } elsif ( $showflags{"msecs"} ) { - printf ".%.3s", $usec; - } - print "] "; - } else { - printf "%s.%.6s ", $secs, $usec; - } - } - if ( $showflags{"host"} ) { - printf "%-8s ", $host; - } - if ( $showflags{"level"} ) { - printf "%-7s : ", $levl; - } - if ( $showflags{"pid"} ) { - printf "%5s ", $pidn; - } - if ( $showflags{"service"} ) { - if ( $shortsvc ) { - printf "%-9.9s ", $svcn; - } else { - printf "%-16s ", $svcn; - } - } - if ( $showflags{"component"} ) { - if ( $shortcmp ) { - printf "%-15.15s ", $comp; - } else { - printf "%s\t", $comp; - } - } - if ( $showflags{"message"} ) { - if ( $optnldequote ) { - $msgt = "\n\t${msgt}" if ( $msgt =~ s/\\n/\n\t/g ); - } - print $msgt; - } - print "\n"; - } else { - print STDERR "bad log line: '$_'\n"; - } -} diff --git a/jdisc_core/src/main/perl/vespa-jdisc-logfmt.1 b/jdisc_core/src/main/perl/vespa-jdisc-logfmt.1 deleted file mode 100644 index 2f855d94207..00000000000 --- a/jdisc_core/src/main/perl/vespa-jdisc-logfmt.1 +++ /dev/null @@ -1,215 +0,0 @@ -.\" Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -.\" $Id: logfmt.1,v 1.12 2007-06-19 09:37:25 daljord Exp $ -.\" -.Dd October 29, 2004 -.Dt VESPA-JDISC-LOGFMT \&1 "JDisc documentation" -.Os "Yahoo! JDisc" "2.3" -.Os -.Sh NAME -.Nm vespa-jdisc-logfmt -.Nd select and format messages from JDisc log files -.Sh SYNOPSIS -.Nm -.Op Fl L Ar levellist -.Op Fl l Ar levellist -.Op Fl s Ar fieldlist -.Op Fl p Ar pid -.Op Fl S Ar service -.Op Fl H Ar host -.Op Fl c Ar regex -.Op Fl m Ar regex -.Op Fl t -.Op Fl f -.Op Fl N -.Op Fl ts -.Op Ar -.Sh DESCRIPTION -The -.Nm -utility reads JDisc log files, select messages and writes a formatted -version of selected messages to the standard output. -.Pp -The options are as follows: -.Bl -tag -width ".It Fl l Ar levellist" -.It Fl L Ar levellist -Declares additional log levels that should be treated as known. These -levels are suppressed unless also given as argument to option -l. -.Ar levellist -is a comma separated list of level names. -.It Fl l Ar levellist -Select which log levels to select. -The default is to select "error", "warning" and "info" levels, and -suppress "debug" and "unknown" levels; but when using this option, only -the named levels will be selected. -The -.Ar levellist -is a comma separated list of level names. -The name -.Em all -may be used to add all known levels. -Prepending a minus sign will deselect the level named. -Starting the list with a plus sign will add and remove levels -from the current (or default) list of levels instead -of replacing it. -.It Fl s Ar fieldlist -Select which fields of log messages to show. -The order of the actual output fields is fixed. -When using this option, only the named fields will be shown. The -fieldlist is a comma separated list of field names. The name -.Em all -may be used to add all possible fields. -Prepending a minus sign will turn off display of the named field. -Starting the list with a plus sign will add and remove fields -from the current (or default) list of fields instead -of replacing it. -.Pp -The fields which may be named are: -.Bl -tag -width component -.It time -Print the time in seconds since the epoch. -Ignored if -.Em fmttime -is shown. -.It fmttime -Print the time in human-readable [YYYY-MM-DD HH:mm:ss] format. -Note that the time is printed in the local timezone; to get GMT -output use -.Nm "\*[q]env TZ=GMT vespa-jdisc-logfmt\*[q]" -as your command. -.It msecs -Add milliseconds after the seconds in -.Em time -and -.Em fmttime -output. Ignored if -.Em usecs -is in effect. -.It usecs -Add microseconds after the seconds in -.Em time -and -.Em fmttime -output. -.It host -Print the hostname field. -.It level -Print the level field (uppercased). -.It pid -Print the pid field. -.It service -Print the service field. -.It component -Print the component field. -.It message -Print the message text field. -You probably always want to add this. -.El -.Pp -Using this option several times works as if the given -.Ar fieldlist -arguments had been concatenated into one comma-separated list. -The default fields to show are as if -.Bk -.Op Fl s Ar fmttime,msecs,level,service,component,message -.Ek -had been given. -.It Fl p Ar pid -Select only messages where the pid field matches the -.Ar pid -string exactly. -.It Fl S Ar service -Select only messages where the service field matches the -.Ar service -string exactly. -.It Fl H Ar host -Select only messages where the hostname field matches the -.Ar host -string exactly. -.It Fl c Ar regex -Select only messages where the component field matches the -.Ar regex -given, using -.Xr perlre -regular expression matching. -.It Fl m Ar regex -Select only messages where the message text field matches the -.Ar regex -given, using -.Xr perlre -regular expression matching. -.It Fl f -Invoke tail -F to follow input file -.It Fl N -Dequote quoted newlines in the message text field to an actual newline plus tab. -.It Fl t -Format the component field (if shown) as a fixed-with string, -truncating if necessary. -.It Fl ts -Format the service field (if shown) as a fixed-with string, -truncating if necessary. -.El -.Sh EXAMPLES -The command: -.Pp -.Bd -literal -offset indent -vespa-jdisc-logfmt -l event -s service,message,fmttime,message -.Ed -.Pp -will display only messages with log level "event", -printing a human-readable time (without any fractional seconds), -the service generating the event and the event message, like this: -.Bd -literal -offset indent -[2004-12-07 18:43:01] config-sentinel starting/1 name="logd" -[2004-12-07 18:43:01] logd started/1 name="logdemon" -[2004-12-07 18:45:51] rtc starting/1 name="rtc.index0" -[2004-12-07 18:45:51] rtc.index0 started/1 name="flexindexer.index" -[2004-12-07 18:45:51] rtc.index0 stopping/1 name="flexindexer.index" why="done" -[2004-12-07 18:45:53] rtc stopped/1 name="rtc.index0" pid=50600 exitcode=0 -[2004-12-07 18:46:13] logd stopping/1 name="logdemon" why="done ok." -[2004-12-07 18:46:13] config-sentinel stopped/1 name="logd" pid=49633 exitcode=0 -.Ed -.Pp -Note that the second "message" item in the fieldlist is redundant, -and that order of printed field is fixed no matter what the fieldlist -order is. -.Pp -The command: -.Pp -.Bd -literal -offset indent -vespa-jdisc-logfmt -l all-info,-debug -s level \e - -s time,usecs,component,message -t -l -event -.Ed -.Pp -will display messages with log levels that are -.Em not -any of -.Em info, debug, -or -.Em event, -printing the time in seconds and microseconds, the log level, the -component name, and the message text, possibly somewhat like this: -.Bd -literal -offset indent -1102441382.530423 CONFIG : nc Config handle: 'pandora.0-rtx' -1102441551.471568 CONFIG : flexindexer.doc Adding document type typetest-0 -1102441573.148211 WARNING : logdemon stopping on signal 15 -1102441887.158000 WARNING : com.yahoo.fs4.m read exception -1102441935.569567 WARNING : rtc Dispatch inherited job failed for dir dispatch0 -1102442115.746001 WARNING : fdispatch Search node 172.24.94.75:10124 down -1102442474.205920 WARNING : rtx RTC (tcp/172.24.94.75:10161) : DOWN -1102442474.515877 WARNING : fdispatch Search node localhost:10128 down -1102442983.075669 ERROR : flexindexer.std Unable to find cluster map defaultcluster -.Ed -.Sh FILES -If no file argument is given, -.Nm -will read the last JDisc log file $VESPA_HOME/logs/jdisc_core/jdisc_core.log (this also works with the -.Fl f -option). -Otherwise, reads only the files given as arguments. -To read standard input, supply a single dash '-' as a file argument. -.Sh SEE ALSO -Documentation in the "log" module for input file format. -.Sh HISTORY -Developed as part of Vespa 1.1, later moved to JDisc 2.3. The default output -format reflects the old "fastlib" log formatting, with minor differences -and is intended to be human-readable, not parsed. diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/FelixParamsTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/FelixParamsTestCase.java index 6836ebde496..3c74172b545 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/core/FelixParamsTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/core/FelixParamsTestCase.java @@ -31,7 +31,7 @@ public class FelixParamsTestCase { public void requireThatSystemPackagesAreNotReplaced() { String systemPackages = ExportPackages.getSystemPackages(); - FelixParams params = new FelixParams(); + FelixParams params = new FelixParams(systemPackages); Map<String, String> config = params.toConfig(); assertNotNull(config); String str = config.get(Constants.FRAMEWORK_SYSTEMPACKAGES); diff --git a/jdisc_core/src/test/perl/help.Levent.expected b/jdisc_core/src/test/perl/help.Levent.expected deleted file mode 100644 index 1f7c17cd81c..00000000000 --- a/jdisc_core/src/test/perl/help.Levent.expected +++ /dev/null @@ -1,20 +0,0 @@ -Usage: vespa-jdisc-logfmt [options] [inputfile ...] -Options: - -l LEVELLIST --level=LEVELLIST select levels to include - -L LEVELLIST --add-level=LEVELLIST define extra levels - -s FIELDLIST --show=FIELDLIST select fields to print - -p PID --pid=PID select messages from given PID - -S SERVICE --service=SERVICE select messages from given SERVICE - -H HOST --host=HOST select messages from given HOST - -c REGEX --component=REGEX select components matching REGEX - -m REGEX --message=REGEX select message text matching REGEX - -f --follow invoke tail -F to follow input file - -N --nldequote dequote newlines in message text field - -t --tc --truncatecomponent chop component to 15 chars - --ts --truncateservice chop service to 9 chars - -FIELDLIST is comma separated, available fields: - time fmttime msecs usecs host level pid service component message -Available levels for LEVELLIST: - debug error event info unknown warning -for both lists, use 'all' for all possible values, and -xxx to disable xxx. diff --git a/jdisc_core/src/test/perl/help.expected b/jdisc_core/src/test/perl/help.expected deleted file mode 100644 index cadddb4cf63..00000000000 --- a/jdisc_core/src/test/perl/help.expected +++ /dev/null @@ -1,20 +0,0 @@ -Usage: vespa-jdisc-logfmt [options] [inputfile ...] -Options: - -l LEVELLIST --level=LEVELLIST select levels to include - -L LEVELLIST --add-level=LEVELLIST define extra levels - -s FIELDLIST --show=FIELDLIST select fields to print - -p PID --pid=PID select messages from given PID - -S SERVICE --service=SERVICE select messages from given SERVICE - -H HOST --host=HOST select messages from given HOST - -c REGEX --component=REGEX select components matching REGEX - -m REGEX --message=REGEX select message text matching REGEX - -f --follow invoke tail -F to follow input file - -N --nldequote dequote newlines in message text field - -t --tc --truncatecomponent chop component to 15 chars - --ts --truncateservice chop service to 9 chars - -FIELDLIST is comma separated, available fields: - time fmttime msecs usecs host level pid service component message -Available levels for LEVELLIST: - debug error info unknown warning -for both lists, use 'all' for all possible values, and -xxx to disable xxx. diff --git a/jdisc_core/src/test/perl/jdisc.expected b/jdisc_core/src/test/perl/jdisc.expected deleted file mode 100644 index 6c5d139c0cf..00000000000 --- a/jdisc_core/src/test/perl/jdisc.expected +++ /dev/null @@ -1,17 +0,0 @@ -[2013-01-25 12:46:51.180] INFO : - org.apache.felix.framework ServiceEvent REGISTERED -[2013-01-25 12:46:51.192] INFO : - jdisc_core.app-a BundleEvent INSTALLED -[2013-01-25 12:46:51.249] INFO : - jdisc_core.app-a BundleEvent RESOLVED -[2013-01-25 12:46:51.249] INFO : - jdisc_core.app-a BundleEvent STARTED -[2013-01-25 12:46:51.334] INFO : - jdisc_core.app-a BundleEvent STOPPED -[2013-01-25 12:46:51.335] INFO : - jdisc_core.app-a BundleEvent UNRESOLVED -[2013-01-25 12:46:51.335] INFO : - jdisc_core.app-a BundleEvent UNINSTALLED -[2013-01-25 12:46:51.376] INFO : - org.apache.felix.framework ServiceEvent REGISTERED -[2013-01-25 12:46:51.377] ERROR : - jdisc_core.app-a my_error -[2013-01-25 12:46:51.377] WARNING : - jdisc_core.app-a my_warning -[2013-01-25 12:46:51.377] INFO : - jdisc_core.app-a my_info -[2013-01-25 12:46:51.379] INFO : - jdisc_core.app-a BundleEvent INSTALLED -[2013-01-25 12:46:51.383] INFO : - jdisc_core.app-a BundleEvent RESOLVED -[2013-01-25 12:46:51.383] INFO : - jdisc_core.app-a BundleEvent STARTED -[2013-01-25 12:46:51.389] INFO : - jdisc_core.app-a BundleEvent STOPPED -[2013-01-25 12:46:51.389] INFO : - jdisc_core.app-a BundleEvent UNRESOLVED -[2013-01-25 12:46:51.390] INFO : - jdisc_core.app-a BundleEvent UNINSTALLED diff --git a/jdisc_core/src/test/perl/jdisc.lall.expected b/jdisc_core/src/test/perl/jdisc.lall.expected deleted file mode 100644 index 58875bba8db..00000000000 --- a/jdisc_core/src/test/perl/jdisc.lall.expected +++ /dev/null @@ -1,19 +0,0 @@ -[2013-01-25 12:46:51.180] INFO : - org.apache.felix.framework ServiceEvent REGISTERED -[2013-01-25 12:46:51.192] INFO : - jdisc_core.app-a BundleEvent INSTALLED -[2013-01-25 12:46:51.249] INFO : - jdisc_core.app-a BundleEvent RESOLVED -[2013-01-25 12:46:51.249] INFO : - jdisc_core.app-a BundleEvent STARTED -[2013-01-25 12:46:51.334] INFO : - jdisc_core.app-a BundleEvent STOPPED -[2013-01-25 12:46:51.335] INFO : - jdisc_core.app-a BundleEvent UNRESOLVED -[2013-01-25 12:46:51.335] INFO : - jdisc_core.app-a BundleEvent UNINSTALLED -[2013-01-25 12:46:51.376] INFO : - org.apache.felix.framework ServiceEvent REGISTERED -[2013-01-25 12:46:51.377] ERROR : - jdisc_core.app-a my_error -[2013-01-25 12:46:51.377] WARNING : - jdisc_core.app-a my_warning -[2013-01-25 12:46:51.377] INFO : - jdisc_core.app-a my_info -[2013-01-25 12:46:51.377] DEBUG : - jdisc_core.app-a my_debug -[2013-01-25 12:46:51.377] UNKNOWN : - jdisc_core.app-a my_unknown -[2013-01-25 12:46:51.379] INFO : - jdisc_core.app-a BundleEvent INSTALLED -[2013-01-25 12:46:51.383] INFO : - jdisc_core.app-a BundleEvent RESOLVED -[2013-01-25 12:46:51.383] INFO : - jdisc_core.app-a BundleEvent STARTED -[2013-01-25 12:46:51.389] INFO : - jdisc_core.app-a BundleEvent STOPPED -[2013-01-25 12:46:51.389] INFO : - jdisc_core.app-a BundleEvent UNRESOLVED -[2013-01-25 12:46:51.390] INFO : - jdisc_core.app-a BundleEvent UNINSTALLED diff --git a/jdisc_core/src/test/perl/jdisc.lall_info.expected b/jdisc_core/src/test/perl/jdisc.lall_info.expected deleted file mode 100644 index da834dddc9b..00000000000 --- a/jdisc_core/src/test/perl/jdisc.lall_info.expected +++ /dev/null @@ -1,4 +0,0 @@ -[2013-01-25 12:46:51.377] ERROR : - jdisc_core.app-a my_error -[2013-01-25 12:46:51.377] WARNING : - jdisc_core.app-a my_warning -[2013-01-25 12:46:51.377] DEBUG : - jdisc_core.app-a my_debug -[2013-01-25 12:46:51.377] UNKNOWN : - jdisc_core.app-a my_unknown diff --git a/jdisc_core/src/test/perl/jdisc.log b/jdisc_core/src/test/perl/jdisc.log deleted file mode 100644 index 37a74a595ca..00000000000 --- a/jdisc_core/src/test/perl/jdisc.log +++ /dev/null @@ -1,19 +0,0 @@ -1359114411.180 gentleadd-lm 35172 - org.apache.felix.framework info ServiceEvent REGISTERED -1359114411.192 gentleadd-lm 35172 - jdisc_core.app-a info BundleEvent INSTALLED -1359114411.249 gentleadd-lm 35172 - jdisc_core.app-a info BundleEvent RESOLVED -1359114411.249 gentleadd-lm 35172 - jdisc_core.app-a info BundleEvent STARTED -1359114411.334 gentleadd-lm 35172 - jdisc_core.app-a info BundleEvent STOPPED -1359114411.335 gentleadd-lm 35172 - jdisc_core.app-a info BundleEvent UNRESOLVED -1359114411.335 gentleadd-lm 35172 - jdisc_core.app-a info BundleEvent UNINSTALLED -1359114411.376 gentleadd-lm 35172 - org.apache.felix.framework info ServiceEvent REGISTERED -1359114411.377 gentleadd-lm 35172 - jdisc_core.app-a error my_error -1359114411.377 gentleadd-lm 35172 - jdisc_core.app-a warning my_warning -1359114411.377 gentleadd-lm 35172 - jdisc_core.app-a info my_info -1359114411.377 gentleadd-lm 35172 - jdisc_core.app-a debug my_debug -1359114411.377 gentleadd-lm 35172 - jdisc_core.app-a unknown my_unknown -1359114411.379 gentleadd-lm 35172 - jdisc_core.app-a info BundleEvent INSTALLED -1359114411.383 gentleadd-lm 35172 - jdisc_core.app-a info BundleEvent RESOLVED -1359114411.383 gentleadd-lm 35172 - jdisc_core.app-a info BundleEvent STARTED -1359114411.389 gentleadd-lm 35172 - jdisc_core.app-a info BundleEvent STOPPED -1359114411.389 gentleadd-lm 35172 - jdisc_core.app-a info BundleEvent UNRESOLVED -1359114411.390 gentleadd-lm 35172 - jdisc_core.app-a info BundleEvent UNINSTALLED diff --git a/jdisc_core/src/test/perl/jdisc.spid.expected b/jdisc_core/src/test/perl/jdisc.spid.expected deleted file mode 100644 index 44963161dcf..00000000000 --- a/jdisc_core/src/test/perl/jdisc.spid.expected +++ /dev/null @@ -1,17 +0,0 @@ -[2013-01-25 12:46:51.180] INFO : 35172 - org.apache.felix.framework ServiceEvent REGISTERED -[2013-01-25 12:46:51.192] INFO : 35172 - jdisc_core.app-a BundleEvent INSTALLED -[2013-01-25 12:46:51.249] INFO : 35172 - jdisc_core.app-a BundleEvent RESOLVED -[2013-01-25 12:46:51.249] INFO : 35172 - jdisc_core.app-a BundleEvent STARTED -[2013-01-25 12:46:51.334] INFO : 35172 - jdisc_core.app-a BundleEvent STOPPED -[2013-01-25 12:46:51.335] INFO : 35172 - jdisc_core.app-a BundleEvent UNRESOLVED -[2013-01-25 12:46:51.335] INFO : 35172 - jdisc_core.app-a BundleEvent UNINSTALLED -[2013-01-25 12:46:51.376] INFO : 35172 - org.apache.felix.framework ServiceEvent REGISTERED -[2013-01-25 12:46:51.377] ERROR : 35172 - jdisc_core.app-a my_error -[2013-01-25 12:46:51.377] WARNING : 35172 - jdisc_core.app-a my_warning -[2013-01-25 12:46:51.377] INFO : 35172 - jdisc_core.app-a my_info -[2013-01-25 12:46:51.379] INFO : 35172 - jdisc_core.app-a BundleEvent INSTALLED -[2013-01-25 12:46:51.383] INFO : 35172 - jdisc_core.app-a BundleEvent RESOLVED -[2013-01-25 12:46:51.383] INFO : 35172 - jdisc_core.app-a BundleEvent STARTED -[2013-01-25 12:46:51.389] INFO : 35172 - jdisc_core.app-a BundleEvent STOPPED -[2013-01-25 12:46:51.389] INFO : 35172 - jdisc_core.app-a BundleEvent UNRESOLVED -[2013-01-25 12:46:51.390] INFO : 35172 - jdisc_core.app-a BundleEvent UNINSTALLED diff --git a/jdisc_core/src/test/perl/vespa-jdisc-logfmt-test.sh b/jdisc_core/src/test/perl/vespa-jdisc-logfmt-test.sh deleted file mode 100755 index e1c3d2a9dd4..00000000000 --- a/jdisc_core/src/test/perl/vespa-jdisc-logfmt-test.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -MYPATH=`dirname ${0}` -DIFF=/usr/bin/diff -LOGFMT=${1} - -if which perl &> /dev/null; then - echo "Running vespa-jdisc-logfmt test suite." -else - echo "Ignoring vespa-jdisc-logfmt test suite as there is no perl executable." - exit 0 -fi - -set -e -export TZ=CET -export VESPA_HOME=$(mktemp -d /tmp/mockup-vespahome-XXXXXX)/ -mkdir -p $VESPA_HOME/libexec/vespa -touch $VESPA_HOME/libexec/vespa/common-env.sh - -echo - -${LOGFMT} -h 2>&1 | ${DIFF} - ${MYPATH}/help.expected -${LOGFMT} -h -L event 2>&1 | ${DIFF} - ${MYPATH}/help.Levent.expected - -${LOGFMT} ${MYPATH}/jdisc.log 2>&1 | ${DIFF} - ${MYPATH}/jdisc.expected -${LOGFMT} -l all ${MYPATH}/jdisc.log 2>&1 | ${DIFF} - ${MYPATH}/jdisc.lall.expected -${LOGFMT} -l all,-info ${MYPATH}/jdisc.log 2>&1 | ${DIFF} - ${MYPATH}/jdisc.lall_info.expected -${LOGFMT} -s +pid ${MYPATH}/jdisc.log 2>&1 | ${DIFF} - ${MYPATH}/jdisc.spid.expected - -${LOGFMT} ${MYPATH}/vespa.log 2>&1 | ${DIFF} - ${MYPATH}/vespa.expected -${LOGFMT} -L event ${MYPATH}/vespa.log 2>&1 | ${DIFF} - ${MYPATH}/vespa.Levent.expected -${LOGFMT} -L event -l all ${MYPATH}/vespa.log 2>&1 | ${DIFF} - ${MYPATH}/vespa.Levent.lall.expected - -rm -r ${VESPA_HOME} -echo All tests passed. diff --git a/jdisc_core/src/test/perl/vespa.Levent.expected b/jdisc_core/src/test/perl/vespa.Levent.expected deleted file mode 100644 index 334ba5f5b28..00000000000 --- a/jdisc_core/src/test/perl/vespa.Levent.expected +++ /dev/null @@ -1,9 +0,0 @@ -[2012-11-27 14:22:48.120] INFO : configserver stdout ROOT = /home/vespa -[2012-11-27 14:22:48.232] INFO : configserver stdout Running without a pid file. -[2012-11-27 14:22:48.336] INFO : configserver stdout LANG = en_US.UTF-8 -[2012-11-27 14:22:48.393] INFO : configserver stdout env LD_PRELOAD=/home/vespa/libexec64/yjava_daemon_preload.so: /home/vespa/bin64/yjava_daemon -sentinel -rlimit_files 16384 -logdest /home/vespa/logs/vespa/yjava_daemon.out -loglevel error -jvm server -maxrestarts 0 -procs 1 -home /home/vespa/share/yjava_jdk/java -Djava.library.path=/home/vespa/lib64:/home/vespa/lib64 -Dvespa.log.control.dir=/home/vespa/var/db/vespa/logcontrol -XX:ThreadStackSize=512 -XX:+UseConcMarkSweepGC -XX:MaxTenuringThreshold=128 -XX:MaxPermSize=512m -Dconfig.id=dir:/home/vespa/conf/configserver -Dyjava_remote_ip_servlet_filter.logLevel=DEBUG -Dorg.apache.commons.logging.Log=org.apache.commons.logging.impl.Jdk14Logger -Dcom.yahoo.protect.Process.forcedExitActive=true -Dzookeeperlogfile=/home/vespa/logs/vespa/zookeeper.log -Xms1536m -Xmx1536m -XX:MaxDirectMemorySize=267m -Djava.awt.headless=true -Dsun.rmi.dgc.client.gcInterval=3600000 -Dsun.rmi.dgc.server.gcInterval=3600000 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/home/vespa/var/crash -Dsun.net.client.defaultConnectTimeout=5000 -Dsun.net.client.defaultReadTimeout=60000 -Djavax.net.ssl.keyStoreType=JKS -javaagent:/home/vespa/lib/jars/yjava_ysecure_agent.jar -Djdisc_core.config.file=/home/vespa/var/jdisc_core/config.properties -Djdisc.export.packages=yjava.security.ysecure,yjava.security.yca,yjava.security.yck,org.apache.velocity,org.apache.velocity.app,org.apache.velocity.context,org.apache.velocity.runtime,org.apache.velocity.runtime.log, -Djdisc.cache.path=/home/vespa/var/vespa/bundlecache/dir:_home_y_conf_configserver -Djdisc.debug.resources=false -Djdisc.bundle.path=/home/vespa/lib/jars -Djdisc.logger.enabled=false -Djdisc.logger.level=ALL -Djdisc.logger.tag=dir:/home/vespa/conf/configserver -user yahoo -cp /home/vespa/lib/jars/jdisc_core-with-dependencies.jar:lib/jars/yjava_bcookie.jar:lib/jars/yjava_bcookie_jni.jar:lib/jars/yjava_byauth.jar:lib/jars/yjava_cookie_data_servlet_filter.jar:lib/jars/yjava_daemon.jar:lib/jars/yjava_jmx_singleton_server.jar:lib/jars/yjava_remote_ip_servlet_filter.jar:lib/jars/yjava_resource_handler.jar:lib/jars/yjava_servlet.jar:lib/jars/yjava_servlet_filters.jar:lib/jars/yjava_yca.jar:lib/jars/yjava_yck.jar:lib/jars/yjava_yhdrs:lib/jars/yjava_yiv.jar:lib/jars/yjava_yiv_servlet.jar:lib/jars/yjava_ynet.jar:lib/jars/yjava_ysecure.jar:lib/jars/yjava_ysecure_agent.jar:lib/jars/yjava_ysecure_native.jar:share/jports/org_apache_velocity__velocity.jar:share/jports/commons_collections__commons_collections.jar:share/jports/commons_lang__commons_lang.jar: -ynet FILTER_YAHOO_ANY --wait-for-jvm-init 10 -Dzookeeper.jmx.log4j.disable=true -Dconfigsources=tcp/localhost:19070 -XX:OnOutOfMemoryError="kill -9 %p" com.yahoo.jdisc.core.BootstrapDaemon file:/home/vespa/lib/jars/container-disc-with-dependencies.jar -[2012-11-27 14:22:50.876] INFO : configserver Container.com.yahoo.container.handler.config.HandlersConfigurerDi Installing bundles from the latest application -[2012-11-27 14:22:50.884] INFO : configserver Container.com.yahoo.container.handler.BundleLoader Installing bundle from disk with reference 'file:/home/vespa/lib/jars/config-models/sherpa-config-model-plugin.jar' -[2012-11-27 14:22:50.889] INFO : configserver Container.com.yahoo.container.handler.BundleLoader Installing bundle from disk with reference 'file:/home/vespa/lib/jars/configserver-container-plugin.jar' -[2012-11-27 14:22:54.285] INFO : configserver Container.com.yahoo.container.handler.config.HandlersConfigurerDi Installing bundles from the latest application -[2012-11-27 14:22:57.489] INFO : configserver Container.com.yahoo.container.handler.config.HandlersConfigurerDi Switched to the latest deployed set of handlers, and dependent components, e.g. search chains, searchers and document processors. Application switch number: 0 diff --git a/jdisc_core/src/test/perl/vespa.Levent.lall.expected b/jdisc_core/src/test/perl/vespa.Levent.lall.expected deleted file mode 100644 index ef58eee7bfc..00000000000 --- a/jdisc_core/src/test/perl/vespa.Levent.lall.expected +++ /dev/null @@ -1,19 +0,0 @@ -[2012-11-27 14:22:48.091] EVENT : configserver runserver starting/1 name="/home/vespa/bin/vespa-start-container-daemon -Dzookeeper.jmx.log4j.disable=true -Dconfigsources=tcp/localhost:19070 (pid 18151)" -[2012-11-27 14:22:48.100] DEBUG : configserver qrs-start exporting: YELL_MA_EURO=INXIGHT -[2012-11-27 14:22:48.100] DEBUG : configserver qrs-start Not setting ulimit -v, no limit set. -[2012-11-27 14:22:48.120] INFO : configserver stdout ROOT = /home/vespa -[2012-11-27 14:22:48.232] INFO : configserver stdout Running without a pid file. -[2012-11-27 14:22:48.336] INFO : configserver stdout LANG = en_US.UTF-8 -[2012-11-27 14:22:48.393] INFO : configserver stdout env LD_PRELOAD=/home/vespa/libexec64/yjava_daemon_preload.so: /home/vespa/bin64/yjava_daemon -sentinel -rlimit_files 16384 -logdest /home/vespa/logs/vespa/yjava_daemon.out -loglevel error -jvm server -maxrestarts 0 -procs 1 -home /home/vespa/share/yjava_jdk/java -Djava.library.path=/home/vespa/lib64:/home/vespa/lib64 -Dvespa.log.control.dir=/home/vespa/var/db/vespa/logcontrol -XX:ThreadStackSize=512 -XX:+UseConcMarkSweepGC -XX:MaxTenuringThreshold=128 -XX:MaxPermSize=512m -Dconfig.id=dir:/home/vespa/conf/configserver -Dyjava_remote_ip_servlet_filter.logLevel=DEBUG -Dorg.apache.commons.logging.Log=org.apache.commons.logging.impl.Jdk14Logger -Dcom.yahoo.protect.Process.forcedExitActive=true -Dzookeeperlogfile=/home/vespa/logs/vespa/zookeeper.log -Xms1536m -Xmx1536m -XX:MaxDirectMemorySize=267m -Djava.awt.headless=true -Dsun.rmi.dgc.client.gcInterval=3600000 -Dsun.rmi.dgc.server.gcInterval=3600000 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/home/vespa/var/crash -Dsun.net.client.defaultConnectTimeout=5000 -Dsun.net.client.defaultReadTimeout=60000 -Djavax.net.ssl.keyStoreType=JKS -javaagent:/home/vespa/lib/jars/yjava_ysecure_agent.jar -Djdisc_core.config.file=/home/vespa/var/jdisc_core/config.properties -Djdisc.export.packages=yjava.security.ysecure,yjava.security.yca,yjava.security.yck,org.apache.velocity,org.apache.velocity.app,org.apache.velocity.context,org.apache.velocity.runtime,org.apache.velocity.runtime.log, -Djdisc.cache.path=/home/vespa/var/vespa/bundlecache/dir:_home_y_conf_configserver -Djdisc.debug.resources=false -Djdisc.bundle.path=/home/vespa/lib/jars -Djdisc.logger.enabled=false -Djdisc.logger.level=ALL -Djdisc.logger.tag=dir:/home/vespa/conf/configserver -user yahoo -cp /home/vespa/lib/jars/jdisc_core-with-dependencies.jar:lib/jars/yjava_bcookie.jar:lib/jars/yjava_bcookie_jni.jar:lib/jars/yjava_byauth.jar:lib/jars/yjava_cookie_data_servlet_filter.jar:lib/jars/yjava_daemon.jar:lib/jars/yjava_jmx_singleton_server.jar:lib/jars/yjava_remote_ip_servlet_filter.jar:lib/jars/yjava_resource_handler.jar:lib/jars/yjava_servlet.jar:lib/jars/yjava_servlet_filters.jar:lib/jars/yjava_yca.jar:lib/jars/yjava_yck.jar:lib/jars/yjava_yhdrs:lib/jars/yjava_yiv.jar:lib/jars/yjava_yiv_servlet.jar:lib/jars/yjava_ynet.jar:lib/jars/yjava_ysecure.jar:lib/jars/yjava_ysecure_agent.jar:lib/jars/yjava_ysecure_native.jar:share/jports/org_apache_velocity__velocity.jar:share/jports/commons_collections__commons_collections.jar:share/jports/commons_lang__commons_lang.jar: -ynet FILTER_YAHOO_ANY --wait-for-jvm-init 10 -Dzookeeper.jmx.log4j.disable=true -Dconfigsources=tcp/localhost:19070 -XX:OnOutOfMemoryError="kill -9 %p" com.yahoo.jdisc.core.BootstrapDaemon file:/home/vespa/lib/jars/container-disc-with-dependencies.jar -[2012-11-27 14:22:50.876] INFO : configserver Container.com.yahoo.container.handler.config.HandlersConfigurerDi Installing bundles from the latest application -[2012-11-27 14:22:50.884] INFO : configserver Container.com.yahoo.container.handler.BundleLoader Installing bundle from disk with reference 'file:/home/vespa/lib/jars/config-models/sherpa-config-model-plugin.jar' -[2012-11-27 14:22:50.889] INFO : configserver Container.com.yahoo.container.handler.BundleLoader Installing bundle from disk with reference 'file:/home/vespa/lib/jars/configserver-container-plugin.jar' -[2012-11-27 14:22:54.285] INFO : configserver Container.com.yahoo.container.handler.config.HandlersConfigurerDi Installing bundles from the latest application -[2012-11-27 14:22:57.430] EVENT : configserver Container.com.yahoo.statistics.Counter count/1 name=requests value=0 -[2012-11-27 14:22:57.431] EVENT : configserver Container.com.yahoo.statistics.Counter count/1 name=requestsCached value=0 -[2012-11-27 14:22:57.432] EVENT : configserver Container.com.yahoo.statistics.Counter count/1 name=requestsNotCached value=0 -[2012-11-27 14:22:57.432] EVENT : configserver Container.com.yahoo.statistics.Counter count/1 name=failedRequests value=0 -[2012-11-27 14:22:57.432] EVENT : configserver Container.com.yahoo.statistics.Counter count/1 name=procTime value=0 -[2012-11-27 14:22:57.432] EVENT : configserver Container.com.yahoo.statistics.Counter count/1 name=procTimeCached value=0 -[2012-11-27 14:22:57.432] EVENT : configserver Container.com.yahoo.statistics.Counter count/1 name=procTimeNotCached value=0 -[2012-11-27 14:22:57.489] INFO : configserver Container.com.yahoo.container.handler.config.HandlersConfigurerDi Switched to the latest deployed set of handlers, and dependent components, e.g. search chains, searchers and document processors. Application switch number: 0 diff --git a/jdisc_core/src/test/perl/vespa.expected b/jdisc_core/src/test/perl/vespa.expected deleted file mode 100644 index 897a7084dae..00000000000 --- a/jdisc_core/src/test/perl/vespa.expected +++ /dev/null @@ -1,18 +0,0 @@ -Warning: unknown level 'event' in input -[2012-11-27 14:22:48.091] EVENT : configserver runserver starting/1 name="/home/vespa/bin/vespa-start-container-daemon -Dzookeeper.jmx.log4j.disable=true -Dconfigsources=tcp/localhost:19070 (pid 18151)" -[2012-11-27 14:22:48.120] INFO : configserver stdout ROOT = /home/vespa -[2012-11-27 14:22:48.232] INFO : configserver stdout Running without a pid file. -[2012-11-27 14:22:48.336] INFO : configserver stdout LANG = en_US.UTF-8 -[2012-11-27 14:22:48.393] INFO : configserver stdout env LD_PRELOAD=/home/vespa/libexec64/yjava_daemon_preload.so: /home/vespa/bin64/yjava_daemon -sentinel -rlimit_files 16384 -logdest /home/vespa/logs/vespa/yjava_daemon.out -loglevel error -jvm server -maxrestarts 0 -procs 1 -home /home/vespa/share/yjava_jdk/java -Djava.library.path=/home/vespa/lib64:/home/vespa/lib64 -Dvespa.log.control.dir=/home/vespa/var/db/vespa/logcontrol -XX:ThreadStackSize=512 -XX:+UseConcMarkSweepGC -XX:MaxTenuringThreshold=128 -XX:MaxPermSize=512m -Dconfig.id=dir:/home/vespa/conf/configserver -Dyjava_remote_ip_servlet_filter.logLevel=DEBUG -Dorg.apache.commons.logging.Log=org.apache.commons.logging.impl.Jdk14Logger -Dcom.yahoo.protect.Process.forcedExitActive=true -Dzookeeperlogfile=/home/vespa/logs/vespa/zookeeper.log -Xms1536m -Xmx1536m -XX:MaxDirectMemorySize=267m -Djava.awt.headless=true -Dsun.rmi.dgc.client.gcInterval=3600000 -Dsun.rmi.dgc.server.gcInterval=3600000 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/home/vespa/var/crash -Dsun.net.client.defaultConnectTimeout=5000 -Dsun.net.client.defaultReadTimeout=60000 -Djavax.net.ssl.keyStoreType=JKS -javaagent:/home/vespa/lib/jars/yjava_ysecure_agent.jar -Djdisc_core.config.file=/home/vespa/var/jdisc_core/config.properties -Djdisc.export.packages=yjava.security.ysecure,yjava.security.yca,yjava.security.yck,org.apache.velocity,org.apache.velocity.app,org.apache.velocity.context,org.apache.velocity.runtime,org.apache.velocity.runtime.log, -Djdisc.cache.path=/home/vespa/var/vespa/bundlecache/dir:_home_y_conf_configserver -Djdisc.debug.resources=false -Djdisc.bundle.path=/home/vespa/lib/jars -Djdisc.logger.enabled=false -Djdisc.logger.level=ALL -Djdisc.logger.tag=dir:/home/vespa/conf/configserver -user yahoo -cp /home/vespa/lib/jars/jdisc_core-with-dependencies.jar:lib/jars/yjava_bcookie.jar:lib/jars/yjava_bcookie_jni.jar:lib/jars/yjava_byauth.jar:lib/jars/yjava_cookie_data_servlet_filter.jar:lib/jars/yjava_daemon.jar:lib/jars/yjava_jmx_singleton_server.jar:lib/jars/yjava_remote_ip_servlet_filter.jar:lib/jars/yjava_resource_handler.jar:lib/jars/yjava_servlet.jar:lib/jars/yjava_servlet_filters.jar:lib/jars/yjava_yca.jar:lib/jars/yjava_yck.jar:lib/jars/yjava_yhdrs:lib/jars/yjava_yiv.jar:lib/jars/yjava_yiv_servlet.jar:lib/jars/yjava_ynet.jar:lib/jars/yjava_ysecure.jar:lib/jars/yjava_ysecure_agent.jar:lib/jars/yjava_ysecure_native.jar:share/jports/org_apache_velocity__velocity.jar:share/jports/commons_collections__commons_collections.jar:share/jports/commons_lang__commons_lang.jar: -ynet FILTER_YAHOO_ANY --wait-for-jvm-init 10 -Dzookeeper.jmx.log4j.disable=true -Dconfigsources=tcp/localhost:19070 -XX:OnOutOfMemoryError="kill -9 %p" com.yahoo.jdisc.core.BootstrapDaemon file:/home/vespa/lib/jars/container-disc-with-dependencies.jar -[2012-11-27 14:22:50.876] INFO : configserver Container.com.yahoo.container.handler.config.HandlersConfigurerDi Installing bundles from the latest application -[2012-11-27 14:22:50.884] INFO : configserver Container.com.yahoo.container.handler.BundleLoader Installing bundle from disk with reference 'file:/home/vespa/lib/jars/config-models/sherpa-config-model-plugin.jar' -[2012-11-27 14:22:50.889] INFO : configserver Container.com.yahoo.container.handler.BundleLoader Installing bundle from disk with reference 'file:/home/vespa/lib/jars/configserver-container-plugin.jar' -[2012-11-27 14:22:54.285] INFO : configserver Container.com.yahoo.container.handler.config.HandlersConfigurerDi Installing bundles from the latest application -[2012-11-27 14:22:57.430] EVENT : configserver Container.com.yahoo.statistics.Counter count/1 name=requests value=0 -[2012-11-27 14:22:57.431] EVENT : configserver Container.com.yahoo.statistics.Counter count/1 name=requestsCached value=0 -[2012-11-27 14:22:57.432] EVENT : configserver Container.com.yahoo.statistics.Counter count/1 name=requestsNotCached value=0 -[2012-11-27 14:22:57.432] EVENT : configserver Container.com.yahoo.statistics.Counter count/1 name=failedRequests value=0 -[2012-11-27 14:22:57.432] EVENT : configserver Container.com.yahoo.statistics.Counter count/1 name=procTime value=0 -[2012-11-27 14:22:57.432] EVENT : configserver Container.com.yahoo.statistics.Counter count/1 name=procTimeCached value=0 -[2012-11-27 14:22:57.432] EVENT : configserver Container.com.yahoo.statistics.Counter count/1 name=procTimeNotCached value=0 -[2012-11-27 14:22:57.489] INFO : configserver Container.com.yahoo.container.handler.config.HandlersConfigurerDi Switched to the latest deployed set of handlers, and dependent components, e.g. search chains, searchers and document processors. Application switch number: 0 diff --git a/jdisc_core/src/test/perl/vespa.log b/jdisc_core/src/test/perl/vespa.log deleted file mode 100644 index 36210bdb798..00000000000 --- a/jdisc_core/src/test/perl/vespa.log +++ /dev/null @@ -1,19 +0,0 @@ -1354022568.091108 example.yahoo.com 18150/38735 configserver runserver event starting/1 name="/home/vespa/bin/vespa-start-container-daemon -Dzookeeper.jmx.log4j.disable=true -Dconfigsources=tcp/localhost:19070 (pid 18151)" -1354022568.100151 example.yahoo.com 18151 configserver qrs-start debug exporting: YELL_MA_EURO=INXIGHT -1354022568.100217 example.yahoo.com 18151 configserver qrs-start debug Not setting ulimit -v, no limit set. -1354022568.120716 example.yahoo.com 18151 configserver stdout info ROOT = /home/vespa -1354022568.232852 example.yahoo.com 18151 configserver stdout info Running without a pid file. -1354022568.336341 example.yahoo.com 18151 configserver stdout info LANG = en_US.UTF-8 -1354022568.393706 example.yahoo.com 18151 configserver stdout info env LD_PRELOAD=/home/vespa/libexec64/yjava_daemon_preload.so: /home/vespa/bin64/yjava_daemon -sentinel -rlimit_files 16384 -logdest /home/vespa/logs/vespa/yjava_daemon.out -loglevel error -jvm server -maxrestarts 0 -procs 1 -home /home/vespa/share/yjava_jdk/java -Djava.library.path=/home/vespa/lib64:/home/vespa/lib64 -Dvespa.log.control.dir=/home/vespa/var/db/vespa/logcontrol -XX:ThreadStackSize=512 -XX:+UseConcMarkSweepGC -XX:MaxTenuringThreshold=128 -XX:MaxPermSize=512m -Dconfig.id=dir:/home/vespa/conf/configserver -Dyjava_remote_ip_servlet_filter.logLevel=DEBUG -Dorg.apache.commons.logging.Log=org.apache.commons.logging.impl.Jdk14Logger -Dcom.yahoo.protect.Process.forcedExitActive=true -Dzookeeperlogfile=/home/vespa/logs/vespa/zookeeper.log -Xms1536m -Xmx1536m -XX:MaxDirectMemorySize=267m -Djava.awt.headless=true -Dsun.rmi.dgc.client.gcInterval=3600000 -Dsun.rmi.dgc.server.gcInterval=3600000 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/home/vespa/var/crash -Dsun.net.client.defaultConnectTimeout=5000 -Dsun.net.client.defaultReadTimeout=60000 -Djavax.net.ssl.keyStoreType=JKS -javaagent:/home/vespa/lib/jars/yjava_ysecure_agent.jar -Djdisc_core.config.file=/home/vespa/var/jdisc_core/config.properties -Djdisc.export.packages=yjava.security.ysecure,yjava.security.yca,yjava.security.yck,org.apache.velocity,org.apache.velocity.app,org.apache.velocity.context,org.apache.velocity.runtime,org.apache.velocity.runtime.log, -Djdisc.cache.path=/home/vespa/var/vespa/bundlecache/dir:_home_y_conf_configserver -Djdisc.debug.resources=false -Djdisc.bundle.path=/home/vespa/lib/jars -Djdisc.logger.enabled=false -Djdisc.logger.level=ALL -Djdisc.logger.tag=dir:/home/vespa/conf/configserver -user yahoo -cp /home/vespa/lib/jars/jdisc_core-with-dependencies.jar:lib/jars/yjava_bcookie.jar:lib/jars/yjava_bcookie_jni.jar:lib/jars/yjava_byauth.jar:lib/jars/yjava_cookie_data_servlet_filter.jar:lib/jars/yjava_daemon.jar:lib/jars/yjava_jmx_singleton_server.jar:lib/jars/yjava_remote_ip_servlet_filter.jar:lib/jars/yjava_resource_handler.jar:lib/jars/yjava_servlet.jar:lib/jars/yjava_servlet_filters.jar:lib/jars/yjava_yca.jar:lib/jars/yjava_yck.jar:lib/jars/yjava_yhdrs:lib/jars/yjava_yiv.jar:lib/jars/yjava_yiv_servlet.jar:lib/jars/yjava_ynet.jar:lib/jars/yjava_ysecure.jar:lib/jars/yjava_ysecure_agent.jar:lib/jars/yjava_ysecure_native.jar:share/jports/org_apache_velocity__velocity.jar:share/jports/commons_collections__commons_collections.jar:share/jports/commons_lang__commons_lang.jar: -ynet FILTER_YAHOO_ANY --wait-for-jvm-init 10 -Dzookeeper.jmx.log4j.disable=true -Dconfigsources=tcp/localhost:19070 -XX:OnOutOfMemoryError="kill -9 %p" com.yahoo.jdisc.core.BootstrapDaemon file:/home/vespa/lib/jars/container-disc-with-dependencies.jar -1354022570.876 example.yahoo.com 18151/1 configserver Container.com.yahoo.container.handler.config.HandlersConfigurerDi info Installing bundles from the latest application -1354022570.884 example.yahoo.com 18151/1 configserver Container.com.yahoo.container.handler.BundleLoader info Installing bundle from disk with reference 'file:/home/vespa/lib/jars/config-models/sherpa-config-model-plugin.jar' -1354022570.889 example.yahoo.com 18151/1 configserver Container.com.yahoo.container.handler.BundleLoader info Installing bundle from disk with reference 'file:/home/vespa/lib/jars/configserver-container-plugin.jar' -1354022574.285 example.yahoo.com 18151/1 configserver Container.com.yahoo.container.handler.config.HandlersConfigurerDi info Installing bundles from the latest application -1354022577.430 example.yahoo.com 18151/1 configserver Container.com.yahoo.statistics.Counter event count/1 name=requests value=0 -1354022577.431 example.yahoo.com 18151/1 configserver Container.com.yahoo.statistics.Counter event count/1 name=requestsCached value=0 -1354022577.432 example.yahoo.com 18151/1 configserver Container.com.yahoo.statistics.Counter event count/1 name=requestsNotCached value=0 -1354022577.432 example.yahoo.com 18151/1 configserver Container.com.yahoo.statistics.Counter event count/1 name=failedRequests value=0 -1354022577.432 example.yahoo.com 18151/1 configserver Container.com.yahoo.statistics.Counter event count/1 name=procTime value=0 -1354022577.432 example.yahoo.com 18151/1 configserver Container.com.yahoo.statistics.Counter event count/1 name=procTimeCached value=0 -1354022577.432 example.yahoo.com 18151/1 configserver Container.com.yahoo.statistics.Counter event count/1 name=procTimeNotCached value=0 -1354022577.489 example.yahoo.com 18151/1 configserver Container.com.yahoo.container.handler.config.HandlersConfigurerDi info Switched to the latest deployed set of handlers, and dependent components, e.g. search chains, searchers and document processors. Application switch number: 0 diff --git a/jdisc_core_test/integration_test/pom.xml b/jdisc_core_test/integration_test/pom.xml index e31fa122232..b1cebcdfcc6 100644 --- a/jdisc_core_test/integration_test/pom.xml +++ b/jdisc_core_test/integration_test/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>integration-test-parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>integration_test</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> <dependencies> @@ -243,8 +243,11 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> <configuration> - <!-- Allow installing fragment bundles, see felix.framework:ExtensionManager.addExtensionBundle --> + <!-- java.lang + java.net are opened to avoid "WARNING: Illegal reflective access ... "--> + <!-- jdk.internal.loader is opened to allow installing extension bundles, see felix.framework:ExtensionManager.addExtensionBundle --> <argLine> + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/jdk.internal.loader=ALL-UNNAMED </argLine> </configuration> diff --git a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/application/BundleActivatorIntegrationTest.java b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/application/BundleActivatorIntegrationTest.java index 03e56d7c26d..9450d2f26c1 100644 --- a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/application/BundleActivatorIntegrationTest.java +++ b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/application/BundleActivatorIntegrationTest.java @@ -7,6 +7,7 @@ import org.osgi.framework.Bundle; import org.osgi.framework.BundleContext; import org.osgi.framework.ServiceReference; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -25,8 +26,12 @@ public class BundleActivatorIntegrationTest { Class<?> serviceClass = bundle.loadClass("com.yahoo.jdisc.bundle.my_act.MyService"); assertNotNull(serviceClass); BundleContext ctx = osgi.bundleContext(); - ServiceReference<?> serviceRef = ctx.getServiceReference(serviceClass.getName()); + + ServiceReference<?>[] serviceRefs = bundle.getRegisteredServices(); + assertEquals(1, serviceRefs.length); + ServiceReference<?> serviceRef = serviceRefs[0]; assertNotNull(serviceRef); + Object service = ctx.getService(serviceRef); assertNotNull(service); assertTrue(serviceClass.isInstance(service)); @@ -37,11 +42,16 @@ public class BundleActivatorIntegrationTest { public void requireThatApplicationBundleActivatorHasAccessToCurrentContainer() throws Exception { TestDriver driver = TestDriver.newApplicationBundleInstance("app-g-act.jar", false); OsgiFramework osgi = driver.osgiFramework(); - Class<?> serviceClass = osgi.bundles().get(1).loadClass("com.yahoo.jdisc.bundle.g_act.MyService"); + Bundle bundle = osgi.bundles().get(1); + Class<?> serviceClass = bundle.loadClass("com.yahoo.jdisc.bundle.g_act.MyService"); assertNotNull(serviceClass); BundleContext ctx = osgi.bundleContext(); - ServiceReference<?> serviceRef = ctx.getServiceReference(serviceClass.getName()); + + ServiceReference<?>[] serviceRefs = bundle.getRegisteredServices(); + assertEquals(1, serviceRefs.length); + ServiceReference<?> serviceRef = serviceRefs[0]; assertNotNull(serviceRef); + Object service = ctx.getService(serviceRef); assertNotNull(service); assertTrue(serviceClass.isInstance(service)); diff --git a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/ExportPackagesIntegrationTest.java b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/ExportPackagesIntegrationTest.java index c965bec8544..d2c1d3b5140 100644 --- a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/ExportPackagesIntegrationTest.java +++ b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/ExportPackagesIntegrationTest.java @@ -2,6 +2,7 @@ package com.yahoo.jdisc.core; import com.yahoo.jdisc.test.TestDriver; +import org.junit.Ignore; import org.junit.Test; import org.osgi.framework.Bundle; import org.osgi.framework.BundleException; @@ -17,6 +18,7 @@ import static org.junit.Assert.assertTrue; */ public class ExportPackagesIntegrationTest { + @Ignore // jdisc_core.jar cannot be installed as a bundle since Felix 6.0, due to exporting java.* packages. @Test public void requireThatManifestContainsExportPackage() throws BundleException { FelixFramework felix = TestDriver.newOsgiFramework(); diff --git a/jdisc_core_test/pom.xml b/jdisc_core_test/pom.xml index a0c2780570b..6fa2dd27b83 100644 --- a/jdisc_core_test/pom.xml +++ b/jdisc_core_test/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>integration-test-parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>pom</packaging> <name>${project.artifactId}</name> <modules> diff --git a/jdisc_core_test/test_bundles/app-a/pom.xml b/jdisc_core_test/test_bundles/app-a/pom.xml index 767ffb32784..a6f43e8a8a2 100644 --- a/jdisc_core_test/test_bundles/app-a/pom.xml +++ b/jdisc_core_test/test_bundles/app-a/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>app-a</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/app-b-priv/pom.xml b/jdisc_core_test/test_bundles/app-b-priv/pom.xml index b54a1fd56d8..94ead0ea2ec 100644 --- a/jdisc_core_test/test_bundles/app-b-priv/pom.xml +++ b/jdisc_core_test/test_bundles/app-b-priv/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>app-b-priv</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/app-ca/pom.xml b/jdisc_core_test/test_bundles/app-ca/pom.xml index 5b655dd003e..c77b75ae5cf 100644 --- a/jdisc_core_test/test_bundles/app-ca/pom.xml +++ b/jdisc_core_test/test_bundles/app-ca/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>app-ca</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/app-dj/pom.xml b/jdisc_core_test/test_bundles/app-dj/pom.xml index 64ed25e2ba2..6ff8165cdb6 100644 --- a/jdisc_core_test/test_bundles/app-dj/pom.xml +++ b/jdisc_core_test/test_bundles/app-dj/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>app-dj</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/app-ej-priv/pom.xml b/jdisc_core_test/test_bundles/app-ej-priv/pom.xml index 5e6fe9cd451..db7c05542b3 100644 --- a/jdisc_core_test/test_bundles/app-ej-priv/pom.xml +++ b/jdisc_core_test/test_bundles/app-ej-priv/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>app-ej-priv</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/app-f-more/pom.xml b/jdisc_core_test/test_bundles/app-f-more/pom.xml index b8995baf46d..886e0ccb758 100644 --- a/jdisc_core_test/test_bundles/app-f-more/pom.xml +++ b/jdisc_core_test/test_bundles/app-f-more/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>app-f-more</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/app-g-act/pom.xml b/jdisc_core_test/test_bundles/app-g-act/pom.xml index aaa740efade..929b337f48d 100644 --- a/jdisc_core_test/test_bundles/app-g-act/pom.xml +++ b/jdisc_core_test/test_bundles/app-g-act/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>app-g-act</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/app-h-log/pom.xml b/jdisc_core_test/test_bundles/app-h-log/pom.xml index 0092dd1a11f..2a9603151b3 100644 --- a/jdisc_core_test/test_bundles/app-h-log/pom.xml +++ b/jdisc_core_test/test_bundles/app-h-log/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>app-h-log</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-a/pom.xml b/jdisc_core_test/test_bundles/cert-a/pom.xml index 05ce9d4b072..ec27fc0b005 100644 --- a/jdisc_core_test/test_bundles/cert-a/pom.xml +++ b/jdisc_core_test/test_bundles/cert-a/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>cert-a</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-b/pom.xml b/jdisc_core_test/test_bundles/cert-b/pom.xml index c634dda581a..dc9f06a33f0 100644 --- a/jdisc_core_test/test_bundles/cert-b/pom.xml +++ b/jdisc_core_test/test_bundles/cert-b/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>cert-b</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> <build> @@ -21,7 +21,6 @@ <artifactId>bundle-plugin</artifactId> <extensions>true</extensions> <configuration> - <!-- The Vespa bundle-plugin doesn't include groupId. TODO Vespa 7: remove if that is fixed. --> <Bundle-SymbolicName>${project.groupId}.${project.artifactId}</Bundle-SymbolicName> </configuration> </plugin> diff --git a/jdisc_core_test/test_bundles/cert-ca/pom.xml b/jdisc_core_test/test_bundles/cert-ca/pom.xml index 313e26e9b98..d458105121e 100644 --- a/jdisc_core_test/test_bundles/cert-ca/pom.xml +++ b/jdisc_core_test/test_bundles/cert-ca/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>cert-ca</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/cert-dc/pom.xml b/jdisc_core_test/test_bundles/cert-dc/pom.xml index a25356f368a..6fbd918eaef 100644 --- a/jdisc_core_test/test_bundles/cert-dc/pom.xml +++ b/jdisc_core_test/test_bundles/cert-dc/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>cert-dc</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/cert-eab/pom.xml b/jdisc_core_test/test_bundles/cert-eab/pom.xml index d5747517c5f..6b0f5ba6028 100644 --- a/jdisc_core_test/test_bundles/cert-eab/pom.xml +++ b/jdisc_core_test/test_bundles/cert-eab/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>cert-eab</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/cert-fac/pom.xml b/jdisc_core_test/test_bundles/cert-fac/pom.xml index e27a87aa04b..f6b14f1b6a1 100644 --- a/jdisc_core_test/test_bundles/cert-fac/pom.xml +++ b/jdisc_core_test/test_bundles/cert-fac/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>cert-fac</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/cert-gg/pom.xml b/jdisc_core_test/test_bundles/cert-gg/pom.xml index 7295ef352d3..43ae2dc73c3 100644 --- a/jdisc_core_test/test_bundles/cert-gg/pom.xml +++ b/jdisc_core_test/test_bundles/cert-gg/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>cert-gg</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-hi/pom.xml b/jdisc_core_test/test_bundles/cert-hi/pom.xml index cafcc6d83d7..42b75cf96b6 100644 --- a/jdisc_core_test/test_bundles/cert-hi/pom.xml +++ b/jdisc_core_test/test_bundles/cert-hi/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>cert-hi</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-ih/pom.xml b/jdisc_core_test/test_bundles/cert-ih/pom.xml index 2fbebe9fb00..3df15c66fb5 100644 --- a/jdisc_core_test/test_bundles/cert-ih/pom.xml +++ b/jdisc_core_test/test_bundles/cert-ih/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>cert-ih</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-j-priv/pom.xml b/jdisc_core_test/test_bundles/cert-j-priv/pom.xml index c7770861e9f..e9ea8a8dac8 100644 --- a/jdisc_core_test/test_bundles/cert-j-priv/pom.xml +++ b/jdisc_core_test/test_bundles/cert-j-priv/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>cert-j-priv</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-k-pkgs/pom.xml b/jdisc_core_test/test_bundles/cert-k-pkgs/pom.xml index 37e8469afdc..c3875cf48eb 100644 --- a/jdisc_core_test/test_bundles/cert-k-pkgs/pom.xml +++ b/jdisc_core_test/test_bundles/cert-k-pkgs/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>cert-k-pkgs</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-l1/pom.xml b/jdisc_core_test/test_bundles/cert-l1/pom.xml index 156635687ae..a7c0367adcb 100644 --- a/jdisc_core_test/test_bundles/cert-l1/pom.xml +++ b/jdisc_core_test/test_bundles/cert-l1/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>cert-l1</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-l2/pom.xml b/jdisc_core_test/test_bundles/cert-l2/pom.xml index ba879d861f2..4749ad7a2fe 100644 --- a/jdisc_core_test/test_bundles/cert-l2/pom.xml +++ b/jdisc_core_test/test_bundles/cert-l2/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>cert-l2</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-ml/pom.xml b/jdisc_core_test/test_bundles/cert-ml/pom.xml index 133ce9807b8..1c6e3bcbc41 100644 --- a/jdisc_core_test/test_bundles/cert-ml/pom.xml +++ b/jdisc_core_test/test_bundles/cert-ml/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>cert-ml</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-nac/pom.xml b/jdisc_core_test/test_bundles/cert-nac/pom.xml index e2bfeee9494..6408a62d2eb 100644 --- a/jdisc_core_test/test_bundles/cert-nac/pom.xml +++ b/jdisc_core_test/test_bundles/cert-nac/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>cert-nac</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/cert-oa-path/pom.xml b/jdisc_core_test/test_bundles/cert-oa-path/pom.xml index c019cd48889..1b5ba98fee5 100644 --- a/jdisc_core_test/test_bundles/cert-oa-path/pom.xml +++ b/jdisc_core_test/test_bundles/cert-oa-path/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>cert-oa-path</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/cert-p-jar/pom.xml b/jdisc_core_test/test_bundles/cert-p-jar/pom.xml index 1cb585d2937..94a3f5c2322 100644 --- a/jdisc_core_test/test_bundles/cert-p-jar/pom.xml +++ b/jdisc_core_test/test_bundles/cert-p-jar/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>cert-p-jar</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> </project> diff --git a/jdisc_core_test/test_bundles/cert-q-frag/pom.xml b/jdisc_core_test/test_bundles/cert-q-frag/pom.xml index 4170438150c..a94b9a0c70a 100644 --- a/jdisc_core_test/test_bundles/cert-q-frag/pom.xml +++ b/jdisc_core_test/test_bundles/cert-q-frag/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>cert-q-frag</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-rq/pom.xml b/jdisc_core_test/test_bundles/cert-rq/pom.xml index 4f6cf36c87c..1d45e48ca46 100644 --- a/jdisc_core_test/test_bundles/cert-rq/pom.xml +++ b/jdisc_core_test/test_bundles/cert-rq/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>cert-rq</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/cert-s-act/pom.xml b/jdisc_core_test/test_bundles/cert-s-act/pom.xml index c1a11c45055..47c7718be39 100644 --- a/jdisc_core_test/test_bundles/cert-s-act/pom.xml +++ b/jdisc_core_test/test_bundles/cert-s-act/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>cert-s-act</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/cert-tp/pom.xml b/jdisc_core_test/test_bundles/cert-tp/pom.xml index 45278efeff8..4d4f84e245e 100644 --- a/jdisc_core_test/test_bundles/cert-tp/pom.xml +++ b/jdisc_core_test/test_bundles/cert-tp/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>cert-tp</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/cert-us/pom.xml b/jdisc_core_test/test_bundles/cert-us/pom.xml index 059341d6505..33aad6fad95 100644 --- a/jdisc_core_test/test_bundles/cert-us/pom.xml +++ b/jdisc_core_test/test_bundles/cert-us/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>cert-us</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_core_test/test_bundles/my-bundle-activator/pom.xml b/jdisc_core_test/test_bundles/my-bundle-activator/pom.xml index 8931c26a73e..9b0693e4a30 100644 --- a/jdisc_core_test/test_bundles/my-bundle-activator/pom.xml +++ b/jdisc_core_test/test_bundles/my-bundle-activator/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>my-bundle-activator</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/my-guice-module/pom.xml b/jdisc_core_test/test_bundles/my-guice-module/pom.xml index ed64d9d0525..a0321d6a3f9 100644 --- a/jdisc_core_test/test_bundles/my-guice-module/pom.xml +++ b/jdisc_core_test/test_bundles/my-guice-module/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>my-guice-module</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/my-server-provider/pom.xml b/jdisc_core_test/test_bundles/my-server-provider/pom.xml index d2a96b9953c..f83bbc69bec 100644 --- a/jdisc_core_test/test_bundles/my-server-provider/pom.xml +++ b/jdisc_core_test/test_bundles/my-server-provider/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>my-server-provider</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>bundle</packaging> <name>${project.artifactId}</name> <build> diff --git a/jdisc_core_test/test_bundles/pom.xml b/jdisc_core_test/test_bundles/pom.xml index a5887cacf3c..a8713fb4e41 100644 --- a/jdisc_core_test/test_bundles/pom.xml +++ b/jdisc_core_test/test_bundles/pom.xml @@ -8,10 +8,10 @@ <parent> <groupId>com.yahoo.vespa.jdisc_core</groupId> <artifactId>integration-test-parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> </parent> <artifactId>test_bundles</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>pom</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/jdisc_http_service/abi-spec.json b/jdisc_http_service/abi-spec.json index 8fe945267ec..099b009fba5 100644 --- a/jdisc_http_service/abi-spec.json +++ b/jdisc_http_service/abi-spec.json @@ -33,7 +33,6 @@ "public com.yahoo.jdisc.http.ConnectorConfig$Builder responseHeaderSize(int)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder acceptQueueSize(int)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder reuseAddress(boolean)", - "public com.yahoo.jdisc.http.ConnectorConfig$Builder soLingerTime(double)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder idleTimeout(double)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder stopTimeout(double)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder tcpKeepAliveEnabled(boolean)", @@ -82,33 +81,9 @@ "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder certificateFile(java.lang.String)", "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder caCertificateFile(java.lang.String)", "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder clientAuth(com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder keyDbKey(java.lang.String)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder excludeProtocol(com.yahoo.jdisc.http.ConnectorConfig$Ssl$ExcludeProtocol$Builder)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder excludeProtocol(java.util.List)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder includeProtocol(com.yahoo.jdisc.http.ConnectorConfig$Ssl$IncludeProtocol$Builder)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder includeProtocol(java.util.List)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder excludeCipherSuite(com.yahoo.jdisc.http.ConnectorConfig$Ssl$ExcludeCipherSuite$Builder)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder excludeCipherSuite(java.util.List)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder includeCipherSuite(com.yahoo.jdisc.http.ConnectorConfig$Ssl$IncludeCipherSuite$Builder)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder includeCipherSuite(java.util.List)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder keyStoreType(com.yahoo.jdisc.http.ConnectorConfig$Ssl$KeyStoreType$Enum)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder keyStorePath(java.lang.String)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder pemKeyStore(com.yahoo.jdisc.http.ConnectorConfig$Ssl$PemKeyStore$Builder)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder trustStoreType(com.yahoo.jdisc.http.ConnectorConfig$Ssl$TrustStoreType$Enum)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder trustStorePath(java.lang.String)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder useTrustStorePassword(boolean)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder sslKeyManagerFactoryAlgorithm(java.lang.String)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder protocol(java.lang.String)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder prng(java.lang.String)", "public com.yahoo.jdisc.http.ConnectorConfig$Ssl build()" ], - "fields": [ - "public java.util.List excludeProtocol", - "public java.util.List includeProtocol", - "public java.util.List excludeCipherSuite", - "public java.util.List includeCipherSuite", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$PemKeyStore$Builder pemKeyStore" - ] + "fields": [] }, "com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum": { "superClass": "java.lang.Enum", @@ -145,217 +120,6 @@ "public static final com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum NEED_AUTH" ] }, - "com.yahoo.jdisc.http.ConnectorConfig$Ssl$ExcludeCipherSuite$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigBuilder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$Ssl$ExcludeCipherSuite)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$ExcludeCipherSuite$Builder name(java.lang.String)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$ExcludeCipherSuite build()" - ], - "fields": [] - }, - "com.yahoo.jdisc.http.ConnectorConfig$Ssl$ExcludeCipherSuite": { - "superClass": "com.yahoo.config.InnerNode", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$Ssl$ExcludeCipherSuite$Builder)", - "public java.lang.String name()" - ], - "fields": [] - }, - "com.yahoo.jdisc.http.ConnectorConfig$Ssl$ExcludeProtocol$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigBuilder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$Ssl$ExcludeProtocol)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$ExcludeProtocol$Builder name(java.lang.String)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$ExcludeProtocol build()" - ], - "fields": [] - }, - "com.yahoo.jdisc.http.ConnectorConfig$Ssl$ExcludeProtocol": { - "superClass": "com.yahoo.config.InnerNode", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$Ssl$ExcludeProtocol$Builder)", - "public java.lang.String name()" - ], - "fields": [] - }, - "com.yahoo.jdisc.http.ConnectorConfig$Ssl$IncludeCipherSuite$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigBuilder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$Ssl$IncludeCipherSuite)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$IncludeCipherSuite$Builder name(java.lang.String)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$IncludeCipherSuite build()" - ], - "fields": [] - }, - "com.yahoo.jdisc.http.ConnectorConfig$Ssl$IncludeCipherSuite": { - "superClass": "com.yahoo.config.InnerNode", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$Ssl$IncludeCipherSuite$Builder)", - "public java.lang.String name()" - ], - "fields": [] - }, - "com.yahoo.jdisc.http.ConnectorConfig$Ssl$IncludeProtocol$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigBuilder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$Ssl$IncludeProtocol)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$IncludeProtocol$Builder name(java.lang.String)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$IncludeProtocol build()" - ], - "fields": [] - }, - "com.yahoo.jdisc.http.ConnectorConfig$Ssl$IncludeProtocol": { - "superClass": "com.yahoo.config.InnerNode", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$Ssl$IncludeProtocol$Builder)", - "public java.lang.String name()" - ], - "fields": [] - }, - "com.yahoo.jdisc.http.ConnectorConfig$Ssl$KeyStoreType$Enum": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.jdisc.http.ConnectorConfig$Ssl$KeyStoreType$Enum[] values()", - "public static com.yahoo.jdisc.http.ConnectorConfig$Ssl$KeyStoreType$Enum valueOf(java.lang.String)" - ], - "fields": [ - "public static final enum com.yahoo.jdisc.http.ConnectorConfig$Ssl$KeyStoreType$Enum JKS", - "public static final enum com.yahoo.jdisc.http.ConnectorConfig$Ssl$KeyStoreType$Enum PEM" - ] - }, - "com.yahoo.jdisc.http.ConnectorConfig$Ssl$KeyStoreType": { - "superClass": "com.yahoo.config.EnumNode", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$Ssl$KeyStoreType$Enum)" - ], - "fields": [ - "public static final com.yahoo.jdisc.http.ConnectorConfig$Ssl$KeyStoreType$Enum JKS", - "public static final com.yahoo.jdisc.http.ConnectorConfig$Ssl$KeyStoreType$Enum PEM" - ] - }, - "com.yahoo.jdisc.http.ConnectorConfig$Ssl$PemKeyStore$Builder": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.config.ConfigBuilder" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$Ssl$PemKeyStore)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$PemKeyStore$Builder keyPath(java.lang.String)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$PemKeyStore$Builder certificatePath(java.lang.String)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$PemKeyStore build()" - ], - "fields": [] - }, - "com.yahoo.jdisc.http.ConnectorConfig$Ssl$PemKeyStore": { - "superClass": "com.yahoo.config.InnerNode", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$Ssl$PemKeyStore$Builder)", - "public java.lang.String keyPath()", - "public java.lang.String certificatePath()" - ], - "fields": [] - }, - "com.yahoo.jdisc.http.ConnectorConfig$Ssl$TrustStoreType$Enum": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.jdisc.http.ConnectorConfig$Ssl$TrustStoreType$Enum[] values()", - "public static com.yahoo.jdisc.http.ConnectorConfig$Ssl$TrustStoreType$Enum valueOf(java.lang.String)" - ], - "fields": [ - "public static final enum com.yahoo.jdisc.http.ConnectorConfig$Ssl$TrustStoreType$Enum JKS" - ] - }, - "com.yahoo.jdisc.http.ConnectorConfig$Ssl$TrustStoreType": { - "superClass": "com.yahoo.config.EnumNode", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$Ssl$TrustStoreType$Enum)" - ], - "fields": [ - "public static final com.yahoo.jdisc.http.ConnectorConfig$Ssl$TrustStoreType$Enum JKS" - ] - }, "com.yahoo.jdisc.http.ConnectorConfig$Ssl": { "superClass": "com.yahoo.config.InnerNode", "interfaces": [], @@ -369,25 +133,7 @@ "public java.lang.String privateKeyFile()", "public java.lang.String certificateFile()", "public java.lang.String caCertificateFile()", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum clientAuth()", - "public java.lang.String keyDbKey()", - "public java.util.List excludeProtocol()", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$ExcludeProtocol excludeProtocol(int)", - "public java.util.List includeProtocol()", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$IncludeProtocol includeProtocol(int)", - "public java.util.List excludeCipherSuite()", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$ExcludeCipherSuite excludeCipherSuite(int)", - "public java.util.List includeCipherSuite()", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$IncludeCipherSuite includeCipherSuite(int)", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$KeyStoreType$Enum keyStoreType()", - "public java.lang.String keyStorePath()", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$PemKeyStore pemKeyStore()", - "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$TrustStoreType$Enum trustStoreType()", - "public java.lang.String trustStorePath()", - "public boolean useTrustStorePassword()", - "public java.lang.String sslKeyManagerFactoryAlgorithm()", - "public java.lang.String protocol()", - "public java.lang.String prng()" + "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum clientAuth()" ], "fields": [] }, @@ -449,7 +195,6 @@ "public int responseHeaderSize()", "public int acceptQueueSize()", "public boolean reuseAddress()", - "public double soLingerTime()", "public double idleTimeout()", "public double stopTimeout()", "public boolean tcpKeepAliveEnabled()", @@ -483,31 +228,20 @@ "public com.yahoo.jdisc.http.Cookie setDomain(java.lang.String)", "public java.lang.String getPath()", "public com.yahoo.jdisc.http.Cookie setPath(java.lang.String)", - "public java.lang.String getComment()", - "public com.yahoo.jdisc.http.Cookie setComment(java.lang.String)", - "public java.lang.String getCommentURL()", - "public com.yahoo.jdisc.http.Cookie setCommentURL(java.lang.String)", - "public java.lang.String getCommentUrl()", - "public com.yahoo.jdisc.http.Cookie setCommentUrl(java.lang.String)", "public int getMaxAge(java.util.concurrent.TimeUnit)", "public com.yahoo.jdisc.http.Cookie setMaxAge(int, java.util.concurrent.TimeUnit)", - "public int getVersion()", - "public com.yahoo.jdisc.http.Cookie setVersion(int)", "public boolean isSecure()", "public com.yahoo.jdisc.http.Cookie setSecure(boolean)", "public boolean isHttpOnly()", "public com.yahoo.jdisc.http.Cookie setHttpOnly(boolean)", - "public boolean isDiscard()", - "public com.yahoo.jdisc.http.Cookie setDiscard(boolean)", - "public java.util.Set ports()", "public boolean equals(java.lang.Object)", "public int hashCode()", "public java.lang.String toString()", "public static java.lang.String toCookieHeader(java.lang.Iterable)", "public static java.util.List fromCookieHeader(java.lang.String)", - "public static java.lang.String toSetCookieHeader(java.lang.Iterable)", + "public static java.util.List toSetCookieHeaders(java.lang.Iterable)", "public static java.util.List toSetCookieHeaderAll(java.lang.Iterable)", - "public static java.util.List fromSetCookieHeader(java.lang.String)" + "public static com.yahoo.jdisc.http.Cookie fromSetCookieHeader(java.lang.String)" ], "fields": [] }, @@ -1168,21 +902,17 @@ "methods": [ "protected void <init>(com.yahoo.jdisc.http.Cookie)", "public static com.yahoo.jdisc.http.filter.JDiscCookieWrapper wrap(com.yahoo.jdisc.http.Cookie)", - "public java.lang.String getComment()", "public java.lang.String getDomain()", "public int getMaxAge()", "public java.lang.String getName()", "public java.lang.String getPath()", "public boolean getSecure()", "public java.lang.String getValue()", - "public int getVersion()", - "public void setComment(java.lang.String)", "public void setDomain(java.lang.String)", "public void setMaxAge(int)", "public void setPath(java.lang.String)", "public void setSecure(boolean)", "public void setValue(java.lang.String)", - "public void setVersion(int)", "public com.yahoo.jdisc.http.Cookie getCookie()" ], "fields": [] @@ -1398,5 +1128,19 @@ "public abstract org.eclipse.jetty.util.ssl.SslContextFactory getInstance(java.lang.String, int)" ], "fields": [] + }, + "com.yahoo.jdisc.http.ssl.ThrowingSslContextFactoryProvider": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.jdisc.http.ssl.SslContextFactoryProvider" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public org.eclipse.jetty.util.ssl.SslContextFactory getInstance(java.lang.String, int)" + ], + "fields": [] } }
\ No newline at end of file diff --git a/jdisc_http_service/pom.xml b/jdisc_http_service/pom.xml index 3e1b81d62c6..6f73d581c0e 100644 --- a/jdisc_http_service/pom.xml +++ b/jdisc_http_service/pom.xml @@ -8,26 +8,16 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>jdisc_http_service</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> <dependencies> <!-- PROVIDED SCOPE --> <dependency> - <groupId>org.bouncycastle</groupId> - <artifactId>bcpkix-jdk15on</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.bouncycastle</groupId> - <artifactId>bcprov-jdk15on</artifactId> - <scope>provided</scope> - </dependency> - <dependency> <groupId>com.google.inject</groupId> <artifactId>guice</artifactId> <scope>provided</scope> @@ -129,6 +119,11 @@ <artifactId>spring-test</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.bouncycastle</groupId> + <artifactId>bcpkix-jdk15on</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> @@ -172,8 +167,6 @@ <extensions>true</extensions> <configuration> <discPreInstallBundle> - bcpkix-jdk15on-${bouncycastle.version}.jar, - bcprov-jdk15on-${bouncycastle.version}.jar, javax.servlet-api-3.1.0.jar, jetty-continuation-${jetty.version}.jar, jetty-http-${jetty.version}.jar, diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/Cookie.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/Cookie.java index a43310aff51..06c93db7ede 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/Cookie.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/Cookie.java @@ -7,15 +7,11 @@ import org.eclipse.jetty.server.Response; import java.net.HttpCookie; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; -import java.util.StringTokenizer; import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -24,22 +20,17 @@ import java.util.stream.StreamSupport; * * Note: RFC 2109 and RFC 2965 is no longer supported. All fields that are not part of RFC 6265 are deprecated. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge * @author bjorncs */ public class Cookie { - private final static Logger log = Logger.getLogger(Cookie.class.getName()); - private final Set<Integer> ports = new HashSet<>(); private String name; private String value; private String domain; private String path; - private String comment; - private String commentUrl; private long maxAgeSeconds = Integer.MIN_VALUE; - private int version; private boolean secure; private boolean httpOnly; private boolean discard; @@ -53,10 +44,7 @@ public class Cookie { value = cookie.value; domain = cookie.domain; path = cookie.path; - comment = cookie.comment; - commentUrl = cookie.commentUrl; maxAgeSeconds = cookie.maxAgeSeconds; - version = cookie.version; secure = cookie.secure; httpOnly = cookie.httpOnly; discard = cookie.discard; @@ -103,38 +91,6 @@ public class Cookie { return this; } - @Deprecated - public String getComment() { - return comment; - } - - @Deprecated - public Cookie setComment(String comment) { - this.comment = comment; - return this; - } - - @Deprecated - public String getCommentURL() { - return getCommentUrl(); - } - - @Deprecated - public Cookie setCommentURL(String commentUrl) { - return setCommentUrl(commentUrl); - } - - @Deprecated - public String getCommentUrl() { - return commentUrl; - } - - @Deprecated - public Cookie setCommentUrl(String commentUrl) { - this.commentUrl = commentUrl; - return this; - } - public int getMaxAge(TimeUnit unit) { return (int)unit.convert(maxAgeSeconds, TimeUnit.SECONDS); } @@ -144,17 +100,6 @@ public class Cookie { return this; } - @Deprecated - public int getVersion() { - return version; - } - - @Deprecated - public Cookie setVersion(int version) { - this.version = version; - return this; - } - public boolean isSecure() { return secure; } @@ -173,29 +118,12 @@ public class Cookie { return this; } - @Deprecated - public boolean isDiscard() { - return discard; - } - - @Deprecated - public Cookie setDiscard(boolean discard) { - this.discard = discard; - return this; - } - - @Deprecated - public Set<Integer> ports() { - return ports; - } - @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Cookie cookie = (Cookie) o; return maxAgeSeconds == cookie.maxAgeSeconds && - version == cookie.version && secure == cookie.secure && httpOnly == cookie.httpOnly && discard == cookie.discard && @@ -203,14 +131,12 @@ public class Cookie { Objects.equals(name, cookie.name) && Objects.equals(value, cookie.value) && Objects.equals(domain, cookie.domain) && - Objects.equals(path, cookie.path) && - Objects.equals(comment, cookie.comment) && - Objects.equals(commentUrl, cookie.commentUrl); + Objects.equals(path, cookie.path); } @Override public int hashCode() { - return Objects.hash(ports, name, value, domain, path, comment, commentUrl, maxAgeSeconds, version, secure, httpOnly, discard); + return Objects.hash(ports, name, value, domain, path, maxAgeSeconds, secure, httpOnly, discard); } @Override @@ -229,19 +155,12 @@ public class Cookie { return StreamSupport.stream(cookies.spliterator(), false) .map(cookie -> { HttpCookie httpCookie = new HttpCookie(cookie.getName(), cookie.getValue()); - httpCookie.setComment(cookie.getComment()); - httpCookie.setCommentURL(cookie.getCommentURL()); - httpCookie.setDiscard(cookie.isDiscard()); httpCookie.setDomain(cookie.getDomain()); httpCookie.setHttpOnly(cookie.isHttpOnly()); httpCookie.setMaxAge(cookie.getMaxAge(TimeUnit.SECONDS)); httpCookie.setPath(cookie.getPath()); httpCookie.setSecure(cookie.isSecure()); - httpCookie.setVersion(cookie.getVersion()); - String portList = cookie.ports().stream() - .map(Number::toString) - .collect(Collectors.joining(",")); - httpCookie.setPortlist(portList); + httpCookie.setVersion(0); return httpCookie.toString(); }) .collect(Collectors.joining(";")); @@ -255,29 +174,17 @@ public class Cookie { Cookie cookie = new Cookie(); cookie.setName(servletCookie.getName()); cookie.setValue(servletCookie.getValue()); - cookie.setComment(servletCookie.getComment()); cookie.setPath(servletCookie.getPath()); cookie.setDomain(servletCookie.getDomain()); cookie.setMaxAge(servletCookie.getMaxAge(), TimeUnit.SECONDS); cookie.setSecure(servletCookie.getSecure()); - cookie.setVersion(servletCookie.getVersion()); cookie.setHttpOnly(servletCookie.isHttpOnly()); return cookie; }) .collect(Collectors.toList()); } - /** - * @deprecated Use {@link #toSetCookieHeaderAll(Iterable)} instead. - */ - @Deprecated - public static String toSetCookieHeader(Iterable<? extends Cookie> cookies) { - List<String> encodedCookies = toSetCookieHeaderAll(cookies); - return encodedCookies.isEmpty() ? null : encodedCookies.get(0); - } - - // TODO Rename to toSetCookieHeader for Vespa 7 - public static List<String> toSetCookieHeaderAll(Iterable<? extends Cookie> cookies) { + public static List<String> toSetCookieHeaders(Iterable<? extends Cookie> cookies) { // Ugly, bot Jetty does not provide a dedicated cookie parser (will be included in Jetty 10) Response response = new Response(null, null); for (Cookie cookie : cookies) { @@ -293,45 +200,25 @@ public class Cookie { return new ArrayList<>(response.getHeaders("Set-Cookie")); } - // TODO Change return type to Cookie for Vespa 7 - public static List<Cookie> fromSetCookieHeader(String headerVal) { + @Deprecated // TODO Vespa 8 Remove + public static List<String> toSetCookieHeaderAll(Iterable<? extends Cookie> cookies) { + return toSetCookieHeaders(cookies); + } + + public static Cookie fromSetCookieHeader(String headerVal) { return HttpCookie.parse(headerVal).stream() .map(httpCookie -> { Cookie cookie = new Cookie(); cookie.setName(httpCookie.getName()); cookie.setValue(httpCookie.getValue()); - cookie.setComment(httpCookie.getComment()); - cookie.setCommentUrl(httpCookie.getCommentURL()); - cookie.setDiscard(httpCookie.getDiscard()); cookie.setDomain(httpCookie.getDomain()); cookie.setHttpOnly(httpCookie.isHttpOnly()); - cookie.setMaxAge((int)httpCookie.getMaxAge(), TimeUnit.SECONDS); + cookie.setMaxAge((int) httpCookie.getMaxAge(), TimeUnit.SECONDS); cookie.setPath(httpCookie.getPath()); cookie.setSecure(httpCookie.getSecure()); - cookie.setVersion(httpCookie.getVersion()); - cookie.ports().addAll(parsePortList(httpCookie.getPortlist())); return cookie; }) - .collect(Collectors.toList()); - } - - - private static List<Integer> parsePortList(String rawPortList) { - if (rawPortList == null) return Collections.emptyList(); - - List<Integer> ports = new ArrayList<>(); - StringTokenizer tokenizer = new StringTokenizer(rawPortList, ","); - while (tokenizer.hasMoreTokens()) { - String rawPort = tokenizer.nextToken().trim(); - if (!rawPort.isEmpty()) { - try { - ports.add(Integer.parseInt(rawPort)); - } catch (NumberFormatException e) { - log.log(Level.FINE, "Unable to parse port: " + rawPort, e); - } - } - } - return ports; + .findFirst().get(); } } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/HttpResponse.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/HttpResponse.java index 8905549a002..f7138ba0e2b 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/HttpResponse.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/HttpResponse.java @@ -69,7 +69,7 @@ public class HttpResponse extends Response implements ServletOrJdiscHttpResponse } List<Cookie> ret = new LinkedList<>(); for (String cookie : cookies) { - ret.addAll(Cookie.fromSetCookieHeader(cookie)); + ret.add(Cookie.fromSetCookieHeader(cookie)); } return ret; } @@ -77,7 +77,7 @@ public class HttpResponse extends Response implements ServletOrJdiscHttpResponse public void encodeSetCookieHeader(List<Cookie> cookies) { headers().remove(HttpHeaders.Names.SET_COOKIE); for (Cookie cookie : cookies) { - headers().add(HttpHeaders.Names.SET_COOKIE, Cookie.toSetCookieHeaderAll(Arrays.asList(cookie))); + headers().add(HttpHeaders.Names.SET_COOKIE, Cookie.toSetCookieHeaders(Arrays.asList(cookie))); } } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/SecretStore.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/SecretStore.java index b70fc5fbd96..4f739c5bd78 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/SecretStore.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/SecretStore.java @@ -9,7 +9,7 @@ package com.yahoo.jdisc.http; * @author bjorncs * @deprecated Use com.yahoo.container.jdisc.secretstore.SecretStore */ -@Deprecated +@Deprecated // Vespa 8 public interface SecretStore { /** Returns the secret for this key */ diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/JDiscCookieWrapper.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/JDiscCookieWrapper.java index 8845f233fab..2b9c650d545 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/JDiscCookieWrapper.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/JDiscCookieWrapper.java @@ -8,7 +8,7 @@ import java.util.concurrent.TimeUnit; /** * Wrapper of Cookie. * - * @author tejalk + * @author Tejal Knot * */ public class JDiscCookieWrapper { @@ -23,11 +23,6 @@ public class JDiscCookieWrapper { return new JDiscCookieWrapper(cookie); } - @Deprecated - public String getComment() { - return cookie.getComment(); - } - public String getDomain() { return cookie.getDomain(); } @@ -52,16 +47,6 @@ public class JDiscCookieWrapper { return cookie.getValue(); } - @Deprecated - public int getVersion() { - return cookie.getVersion(); - } - - @Deprecated - public void setComment(String purpose) { - cookie.setComment(purpose); - } - public void setDomain(String pattern) { cookie.setDomain(pattern); } @@ -82,11 +67,6 @@ public class JDiscCookieWrapper { cookie.setValue(newValue); } - @Deprecated - public void setVersion(int version) { - cookie.setVersion(version); - } - /** * Return com.yahoo.jdisc.http.Cookie * diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/ServletFilterResponse.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/ServletFilterResponse.java index 1c8383ae427..b603e7776f1 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/ServletFilterResponse.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/filter/ServletFilterResponse.java @@ -12,8 +12,6 @@ import java.util.List; /** * Servlet implementation for JDisc filter responses. - * - * @since 5.27 */ class ServletFilterResponse extends DiscFilterResponse { @@ -77,7 +75,7 @@ class ServletFilterResponse extends DiscFilterResponse { @Override public void setCookies(List<Cookie> cookies) { removeHeaders(HttpHeaders.Names.SET_COOKIE); - List<String> setCookieHeaders = Cookie.toSetCookieHeaderAll(cookies); + List<String> setCookieHeaders = Cookie.toSetCookieHeaders(cookies); setCookieHeaders.forEach(cookie -> addHeader(HttpHeaders.Names.SET_COOKIE, cookie)); } } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java index 26db07f9ed7..a445230769b 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java @@ -28,13 +28,14 @@ import java.util.logging.Logger; * This class is a bridge between Jetty's {@link org.eclipse.jetty.server.handler.RequestLogHandler} * and our own configurable access logging in different formats provided by {@link AccessLog}. * - * @author bakksjo + * @author Oyvind Bakksjo * @author bjorncs */ public class AccessLogRequestLog extends AbstractLifeCycle implements RequestLog { private static final Logger logger = Logger.getLogger(AccessLogRequestLog.class.getName()); + // TODO These hardcoded headers should be provided by config instead private static final String HEADER_NAME_X_FORWARDED_FOR = "x-forwarded-for"; private static final String HEADER_NAME_Y_RA = "y-ra"; private static final String HEADER_NAME_Y_RP = "y-rp"; @@ -83,7 +84,6 @@ public class AccessLogRequestLog extends AbstractLifeCycle implements RequestLog public static void populateAccessLogEntryFromHttpServletRequest( final HttpServletRequest request, final AccessLogEntry accessLogEntry) { - setUriFromRequest(request, accessLogEntry); accessLogEntry.setRawPath(request.getRequestURI()); String queryString = request.getQueryString(); @@ -135,53 +135,4 @@ public class AccessLogRequestLog extends AbstractLifeCycle implements RequestLog .map(Integer::valueOf) .orElseGet(request::getRemotePort); } - - @SuppressWarnings("deprecation") - private static void setUriFromRequest(HttpServletRequest request, AccessLogEntry accessLogEntry) { - tryCreateUriFromRequest(request) - .ifPresent(accessLogEntry::setURI); // setURI is deprecated - } - - // This is a mess and does not work correctly - private static Optional<URI> tryCreateUriFromRequest(HttpServletRequest request) { - final String quotedQuery = request.getQueryString(); - final String quotedPath = request.getRequestURI(); - try { - final StringBuilder uriBuffer = new StringBuilder(); - uriBuffer.append(quotedPath); - if (quotedQuery != null) { - uriBuffer.append('?').append(quotedQuery); - } - return Optional.of(new URI(uriBuffer.toString())); - } catch (URISyntaxException e) { - return setUriFromMalformedInput(quotedPath, quotedQuery); - } - } - - private static Optional<URI> setUriFromMalformedInput(final String quotedPath, final String quotedQuery) { - try { - final String scheme = null; - final String authority = null; - final String fragment = null; - return Optional.of(new URI(scheme, authority, unquote(quotedPath), unquote(quotedQuery), fragment)); - } catch (URISyntaxException e) { - // I have no idea how this can happen here now... - logger.log(Level.WARNING, "Could not convert String URI to URI object", e); - return Optional.empty(); - } - } - - private static String unquote(final String quotedQuery) { - if (quotedQuery == null) { - return null; - } - try { - // inconsistent handling of semi-colon added here... - return URLDecoder.decode(quotedQuery, StandardCharsets.UTF_8.name()); - } catch (IllegalArgumentException e) { - return quotedQuery; - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); // should not happen - } - } } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java index 95f26e8bc1b..617e081bd24 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java @@ -38,7 +38,7 @@ class HttpRequestFactory { } public static URI getUri(HttpServletRequest servletRequest) { - String query = extraQuote(servletRequest.getQueryString()); + String query = servletRequest.getQueryString(); try { return URI.create(servletRequest.getRequestURL() + (query != null ? '?' + query : "")); } catch (IllegalArgumentException e) { @@ -59,54 +59,6 @@ class HttpRequestFactory { } } - // TODO Remove this ugly, non-complete escaping in Vespa 7 - private static String extraQuote(String queryString) { - // TODO: Use an URI builder - if (queryString == null) return null; - - int toAndIncluding = -1; - for (int i = 0; i < queryString.length(); ++i) { - if (quote(queryString.charAt(i)) != null) { - break; - } - toAndIncluding = i; - } - - String washed; - if (toAndIncluding != (queryString.length() - 1)) { - StringBuilder w = new StringBuilder(queryString.substring(0, toAndIncluding + 1)); - for (int i = toAndIncluding + 1; i < queryString.length(); ++i) { - String s = quote(queryString.charAt(i)); - if (s == null) { - w.append(queryString.charAt(i)); - } else { - w.append(s); - } - } - washed = w.toString(); - } else { - washed = queryString; - } - return washed; - } - - private static String quote(char c) { - switch(c) { - case '\\': - return "%5C"; - case '^': - return "%5E"; - case '{': - return "%7B"; - case '|': - return "%7C"; - case '}': - return "%7D"; - default: - return null; - } - } - private static X509Certificate[] getCertChain(HttpServletRequest servletRequest) { return (X509Certificate[]) servletRequest.getAttribute("javax.servlet.request.X509Certificate"); } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java index 2f5fe7612c8..20c8f945b82 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java @@ -84,7 +84,6 @@ class JDiscHttpServlet extends HttpServlet { * Override to set connector attribute before the request becomes an upgrade request in the web socket case. * (After the upgrade, the HttpConnection is no longer available.) */ - @SuppressWarnings("deprecation") @Override protected void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { @@ -93,7 +92,6 @@ class JDiscHttpServlet extends HttpServlet { Metric.Context metricContext = getMetricContext(request); context.metric.add(JettyHttpServer.Metrics.NUM_REQUESTS, 1, metricContext); context.metric.add(JettyHttpServer.Metrics.JDISC_HTTP_REQUESTS, 1, metricContext); - context.metric.add(JettyHttpServer.Metrics.MANHATTAN_NUM_REQUESTS, 1, metricContext); if (JETTY_UNSUPPORTED_METHODS.contains(request.getMethod().toUpperCase())) { dispatchHttpRequest(request, response); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java index 40be93f2111..07d3d77dff2 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java @@ -66,7 +66,6 @@ import java.util.stream.Collectors; * @author bjorncs */ @Beta -// TODO Vespa 7: Remove unused Manhattan metrics public class JettyHttpServer extends AbstractServerProvider { public interface Metrics { @@ -83,15 +82,12 @@ public class JettyHttpServer extends AbstractServerProvider { String NUM_BYTES_RECEIVED = "serverBytesReceived"; String NUM_BYTES_SENT = "serverBytesSent"; - @Deprecated String MANHATTAN_NUM_BYTES_RECEIVED = "http.in.bytes"; - @Deprecated String MANHATTAN_NUM_BYTES_SENT = "http.out.bytes"; String NUM_CONNECTIONS = "serverNumConnections"; /* For historical reasons, these are all aliases for the same metric. 'jdisc.http' should ideally be the only one. */ String JDISC_HTTP_REQUESTS = "jdisc.http.requests"; String NUM_REQUESTS = "serverNumRequests"; - @Deprecated String MANHATTAN_NUM_REQUESTS = "http.requests"; String NUM_SUCCESSFUL_RESPONSES = "serverNumSuccessfulResponses"; String NUM_FAILED_RESPONSES = "serverNumFailedResponses"; @@ -99,10 +95,8 @@ public class JettyHttpServer extends AbstractServerProvider { String NUM_FAILED_WRITES = "serverNumFailedResponseWrites"; String TOTAL_SUCCESSFUL_LATENCY = "serverTotalSuccessfulResponseLatency"; - @Deprecated String MANHATTAN_TOTAL_SUCCESSFUL_LATENCY = "http.latency"; String TOTAL_FAILED_LATENCY = "serverTotalFailedResponseLatency"; String TIME_TO_FIRST_BYTE = "serverTimeToFirstByte"; - @Deprecated String MANHATTAN_TIME_TO_FIRST_BYTE = "http.out.firstbytetime"; String RESPONSES_1XX = "http.status.1xx"; String RESPONSES_2XX = "http.status.2xx"; @@ -113,7 +107,6 @@ public class JettyHttpServer extends AbstractServerProvider { String RESPONSES_403 = "http.status.403"; String STARTED_MILLIS = "serverStartedMillis"; - @Deprecated String MANHATTAN_STARTED_MILLIS = "proc.uptime"; String URI_LENGTH = "jdisc.http.request.uri_length"; String CONTENT_SIZE = "jdisc.http.request.content_size"; @@ -350,11 +343,9 @@ public class JettyHttpServer extends AbstractServerProvider { } - @SuppressWarnings("deprecation") private void setServerMetrics(HttpResponseStatisticsCollector statisticsCollector) { long timeSinceStarted = System.currentTimeMillis() - timeStarted; metric.set(Metrics.STARTED_MILLIS, timeSinceStarted, null); - metric.set(Metrics.MANHATTAN_STARTED_MILLIS, timeSinceStarted, null); addResponseMetrics(statisticsCollector); } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricReporter.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricReporter.java index 4b01a475842..21a64792731 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricReporter.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricReporter.java @@ -29,22 +29,18 @@ public class MetricReporter { this.requestStartTime = requestStartTime; } - @SuppressWarnings("deprecation") public void successfulWrite(int numBytes) { setTimeToFirstByteFirstTime(); metric.add(Metrics.NUM_SUCCESSFUL_WRITES, 1, context); metric.set(Metrics.NUM_BYTES_SENT, numBytes, context); - metric.set(Metrics.MANHATTAN_NUM_BYTES_SENT, numBytes, context); } - @SuppressWarnings("deprecation") private void setTimeToFirstByteFirstTime() { boolean isFirstWrite = firstSetOfTimeToFirstByte.getAndSet(false); if (isFirstWrite) { long timeToFirstByte = getRequestLatency(); metric.set(Metrics.TIME_TO_FIRST_BYTE, timeToFirstByte, context); - metric.set(Metrics.MANHATTAN_TIME_TO_FIRST_BYTE, timeToFirstByte, context); } } @@ -52,14 +48,12 @@ public class MetricReporter { metric.add(Metrics.NUM_FAILED_WRITES, 1, context); } - @SuppressWarnings("deprecation") public void successfulResponse() { setTimeToFirstByteFirstTime(); long requestLatency = getRequestLatency(); metric.set(Metrics.TOTAL_SUCCESSFUL_LATENCY, requestLatency, context); - metric.set(Metrics.MANHATTAN_TOTAL_SUCCESSFUL_LATENCY, requestLatency, context); metric.add(Metrics.NUM_SUCCESSFUL_RESPONSES, 1, context); } @@ -75,10 +69,8 @@ public class MetricReporter { metric.add(Metrics.NUM_PREMATURELY_CLOSED_CONNECTIONS, 1, context); } - @SuppressWarnings("deprecation") public void successfulRead(int bytes_received) { metric.set(JettyHttpServer.Metrics.NUM_BYTES_RECEIVED, bytes_received, context); - metric.set(JettyHttpServer.Metrics.MANHATTAN_NUM_BYTES_RECEIVED, bytes_received, context); } private long getRequestLatency() { diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/servlet/ServletResponse.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/servlet/ServletResponse.java index 5825d3f5194..48c8f577de9 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/servlet/ServletResponse.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/servlet/ServletResponse.java @@ -18,8 +18,6 @@ import java.util.Map; /** * JDisc wrapper to use a {@link javax.servlet.http.HttpServletResponse} * with JDisc security filters. - * - * @since 5.26 */ public class ServletResponse extends HttpServletResponseWrapper implements ServletOrJdiscHttpResponse { @@ -60,7 +58,7 @@ public class ServletResponse extends HttpServletResponseWrapper implements Servl } List<Cookie> ret = new LinkedList<>(); for (String cookie : cookies) { - ret.addAll(Cookie.fromSetCookieHeader(cookie)); + ret.add(Cookie.fromSetCookieHeader(cookie)); } return ret; } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/ThrowingSslContextFactoryProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/ThrowingSslContextFactoryProvider.java new file mode 100644 index 00000000000..b28ef54e822 --- /dev/null +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/ThrowingSslContextFactoryProvider.java @@ -0,0 +1,16 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.http.ssl; + +import org.eclipse.jetty.util.ssl.SslContextFactory; + +/** + * A dummy implementation of {@link SslContextFactoryProvider} to be injected into non-ssl connectors + * + * @author bjorncs + */ +public class ThrowingSslContextFactoryProvider implements SslContextFactoryProvider { + @Override + public SslContextFactory getInstance(String containerId, int port) { + throw new UnsupportedOperationException(); + } +}
\ No newline at end of file diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java index fa31f58dfc0..c381ba738a3 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/DefaultSslContextFactoryProvider.java @@ -48,14 +48,6 @@ public class DefaultSslContextFactoryProvider implements SslContextFactoryProvid break; } - // NOTE: All ciphers matching ^TLS_RSA_.*$ are disabled by default in Jetty 9.4.12+ (https://github.com/eclipse/jetty.project/issues/2807) - // JDisc will allow these ciphers by default to support older clients (e.g. Java 8u60 and curl 7.29.0) - // Removing the exclusion will allow for the TLS_RSA variants that are not covered by other exclusions - String[] excludedCiphersWithoutTlsRsaExclusion = Arrays.stream(factory.getExcludeCipherSuites()) - .filter(cipher -> !cipher.equals("^TLS_RSA_.*$")) - .toArray(String[]::new); - factory.setExcludeCipherSuites(excludedCiphersWithoutTlsRsaExclusion); - // Check if using new ssl syntax from services.xml factory.setKeyStore(createKeystore(sslConfig)); factory.setKeyStorePassword(""); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/LegacySslContextFactoryProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/LegacySslContextFactoryProvider.java deleted file mode 100644 index 281f80c3aeb..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/LegacySslContextFactoryProvider.java +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.ssl.impl; - -import com.yahoo.config.InnerNode; -import com.yahoo.jdisc.http.ConnectorConfig; -import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; -import com.yahoo.jdisc.http.ssl.pem.PemSslKeyStore; -import org.eclipse.jetty.util.ssl.SslContextFactory; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.security.KeyStore; -import java.util.Arrays; -import java.util.List; -import java.util.function.BiConsumer; -import java.util.function.Function; -import java.util.logging.Logger; - -/** - * A implementation of {@link SslContextFactoryProvider} to be injected into non-ssl connectors or connectors using legacy ssl config - * - * @author bjorncs - */ -// TODO Vespa 7: Remove legacy ssl config -public class LegacySslContextFactoryProvider implements SslContextFactoryProvider { - private static final Logger log = Logger.getLogger(LegacySslContextFactoryProvider.class.getName()); - - private final ConnectorConfig connectorConfig; - @SuppressWarnings("deprecation") - private final com.yahoo.jdisc.http.SecretStore secretStore; - - public LegacySslContextFactoryProvider(ConnectorConfig connectorConfig, - @SuppressWarnings("deprecation") com.yahoo.jdisc.http.SecretStore secretStore) { - validateConfig(connectorConfig.ssl()); - this.connectorConfig = connectorConfig; - this.secretStore = secretStore; - } - - @Override - public SslContextFactory getInstance(String containerId, int port) { - ConnectorConfig.Ssl sslConfig = connectorConfig.ssl(); - if (!sslConfig.enabled()) throw new IllegalStateException(); - SslContextFactory factory = new JDiscSslContextFactory(); - - switch (sslConfig.clientAuth()) { - case NEED_AUTH: - factory.setNeedClientAuth(true); - break; - case WANT_AUTH: - factory.setWantClientAuth(true); - break; - } - - // NOTE: All ciphers matching ^TLS_RSA_.*$ are disabled by default in Jetty 9.4.12+ (https://github.com/eclipse/jetty.project/issues/2807) - // JDisc will allow these ciphers by default to support older clients (e.g. Java 8u60 and curl 7.29.0) - // Removing the exclusion will allow for the TLS_RSA variants that are not covered by other exclusions - String[] excludedCiphersWithoutTlsRsaExclusion = Arrays.stream(factory.getExcludeCipherSuites()) - .filter(cipher -> !cipher.equals("^TLS_RSA_.*$")) - .toArray(String[]::new); - factory.setExcludeCipherSuites(excludedCiphersWithoutTlsRsaExclusion); - - switch (sslConfig.keyStoreType()) { - case JKS: - factory.setKeyStorePath(sslConfig.keyStorePath()); - factory.setKeyStoreType("JKS"); - factory.setKeyStorePassword(secretStore.getSecret(sslConfig.keyDbKey())); - break; - case PEM: - factory.setKeyStorePath(sslConfig.keyStorePath()); - factory.setKeyStore(createPemKeyStore(sslConfig.pemKeyStore())); - break; - } - - if (!sslConfig.trustStorePath().isEmpty()) { - factory.setTrustStorePath(sslConfig.trustStorePath()); - factory.setTrustStoreType(sslConfig.trustStoreType().toString()); - if (sslConfig.useTrustStorePassword()) { - factory.setTrustStorePassword(secretStore.getSecret(sslConfig.keyDbKey())); - } - } - - if (!sslConfig.prng().isEmpty()) { - factory.setSecureRandomAlgorithm(sslConfig.prng()); - } - - setStringArrayParameter( - factory, sslConfig.excludeProtocol(), ConnectorConfig.Ssl.ExcludeProtocol::name, SslContextFactory::setExcludeProtocols); - setStringArrayParameter( - factory, sslConfig.includeProtocol(), ConnectorConfig.Ssl.IncludeProtocol::name, SslContextFactory::setIncludeProtocols); - setStringArrayParameter( - factory, sslConfig.excludeCipherSuite(), ConnectorConfig.Ssl.ExcludeCipherSuite::name, SslContextFactory::setExcludeCipherSuites); - setStringArrayParameter( - factory, sslConfig.includeCipherSuite(), ConnectorConfig.Ssl.IncludeCipherSuite::name, SslContextFactory::setIncludeCipherSuites); - - factory.setKeyManagerFactoryAlgorithm(sslConfig.sslKeyManagerFactoryAlgorithm()); - factory.setProtocol(sslConfig.protocol()); - - return factory; - } - - private static void validateConfig(ConnectorConfig.Ssl config) { - if (!config.enabled()) return; - switch (config.keyStoreType()) { - case JKS: - validateJksConfig(config); - break; - case PEM: - validatePemConfig(config); - break; - } - if (!config.trustStorePath().isEmpty() && config.useTrustStorePassword() && config.keyDbKey().isEmpty()) { - throw new IllegalArgumentException("Missing password for JKS truststore"); - } - } - - private static void validateJksConfig(ConnectorConfig.Ssl ssl) { - if (!ssl.pemKeyStore().keyPath().isEmpty() || ! ssl.pemKeyStore().certificatePath().isEmpty()) { - throw new IllegalArgumentException("pemKeyStore attributes can not be set when keyStoreType is JKS."); - } - if (ssl.keyDbKey().isEmpty()) { - throw new IllegalArgumentException("Missing password for JKS keystore"); - } - } - - private static void validatePemConfig(ConnectorConfig.Ssl ssl) { - if (! ssl.keyStorePath().isEmpty()) { - throw new IllegalArgumentException("keyStorePath can not be set when keyStoreType is PEM"); - } - if (!ssl.keyDbKey().isEmpty()) { - log.warning("Encrypted PEM key stores are not supported. Password is only applied to truststore"); - } - if (ssl.pemKeyStore().certificatePath().isEmpty()) { - throw new IllegalArgumentException("Missing certificate path."); - } - if (ssl.pemKeyStore().keyPath().isEmpty()) { - throw new IllegalArgumentException("Missing key path."); - } - } - - private static KeyStore createPemKeyStore(ConnectorConfig.Ssl.PemKeyStore pemKeyStore) { - try { - Path certificatePath = Paths.get(pemKeyStore.certificatePath()); - Path keyPath = Paths.get(pemKeyStore.keyPath()); - return new PemSslKeyStore(certificatePath, keyPath).loadJavaKeyStore(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } catch (Exception e) { - throw new RuntimeException("Failed setting up key store for " + pemKeyStore.keyPath() + ", " + pemKeyStore.certificatePath(), e); - } - } - - private static <T extends InnerNode> void setStringArrayParameter(SslContextFactory sslContextFactory, - List<T> configValues, - Function<T, String> nameProperty, - BiConsumer<SslContextFactory, String[]> setter) { - if (!configValues.isEmpty()) { - String[] nameArray = configValues.stream().map(nameProperty).toArray(String[]::new); - setter.accept(sslContextFactory, nameArray); - } - } - -} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/pem/PemKeyStore.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/pem/PemKeyStore.java deleted file mode 100644 index b52e923662f..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/pem/PemKeyStore.java +++ /dev/null @@ -1,336 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.ssl.pem; - -import com.google.common.base.Preconditions; -import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; -import org.bouncycastle.cert.X509CertificateHolder; -import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.bouncycastle.openssl.PEMException; -import org.bouncycastle.openssl.PEMKeyPair; -import org.bouncycastle.openssl.PEMParser; -import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter; - -import javax.annotation.concurrent.GuardedBy; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.Reader; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.Key; -import java.security.KeyStore; -import java.security.KeyStore.LoadStoreParameter; -import java.security.KeyStoreException; -import java.security.KeyStoreSpi; -import java.security.NoSuchAlgorithmException; -import java.security.UnrecoverableKeyException; -import java.security.cert.Certificate; -import java.security.cert.CertificateException; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.Enumeration; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.function.Consumer; - -import static com.yahoo.jdisc.http.server.jetty.Exceptions.throwUnchecked; - -/** - * Exposes keys and certificates from unencrypted PEM keystore. - * - * @author Tony Vaagenes - * @author bjorncs - */ -public class PemKeyStore extends KeyStoreSpi { - - private static String KEY_ALIAS = "KEY"; - - static List<String> aliases = Collections.emptyList(); - static Map<String, String> attributes = Collections.emptyMap(); - private static final BouncyCastleProvider bouncyCastleProvider = new BouncyCastleProvider(); - - @GuardedBy("this") - private StoreRole storeRole; - @GuardedBy("this") - private Key privateKey; - @GuardedBy("this") - private final Map<String, Certificate> aliasToCertificate = new LinkedHashMap<>(); - - /** - * The user is responsible for closing any readers given in the parameter. - */ - @Override - public synchronized void engineLoad(LoadStoreParameter parameter) throws IOException { - if (storeRole != null) - throw new IllegalStateException("Already initialized."); - - if (parameter instanceof KeyStoreLoadParameter) { - storeRole = new KeyStoreRole(); - loadKeyStore((KeyStoreLoadParameter) parameter); - } else if (parameter instanceof TrustStoreLoadParameter) { - storeRole = new TrustStoreRole(); - loadTrustStore((TrustStoreLoadParameter) parameter); - } else { - throw new IllegalArgumentException("Expected key store or trust store load parameter, got " + parameter.getClass()); - } - } - - private void loadTrustStore(TrustStoreLoadParameter parameter) throws IOException { - withPemParser(parameter.certificateReader, this::loadCertificates); - } - - private void loadKeyStore(KeyStoreLoadParameter parameter) throws IOException{ - withPemParser(parameter.keyReader, this::loadPrivateKey); - withPemParser(parameter.certificateReader, this::loadCertificates); - } - - private static void withPemParser(ReaderForPath reader, Consumer<PEMParser> f) throws IOException { - try { - //parser.close() will close the underlying reader, - //which we want to avoid. - //See engineLoad comment. - PEMParser parser = new PEMParser(reader.reader); - f.accept(parser); - } catch (Exception e) { - throw new RuntimeException("Failed loading pem key store " + reader.path, e); - } - } - - private void loadPrivateKey(PEMParser parser) { - try { - Object object = parser.readObject(); - PrivateKeyInfo privateKeyInfo; - if (object instanceof PEMKeyPair) { // Legacy PKCS1 - privateKeyInfo = ((PEMKeyPair) object).getPrivateKeyInfo(); - } else if (object instanceof PrivateKeyInfo) { // PKCS8 - privateKeyInfo = (PrivateKeyInfo) object; - } else { - throw new UnsupportedOperationException( - "Expected " + PrivateKeyInfo.class + " or " + PEMKeyPair.class + ", got " + object.getClass()); - } - - Object nextObject = parser.readObject(); - if (nextObject != null) { - throw new UnsupportedOperationException( - "Expected a single private key, but found a second element " + nextObject.getClass()); - } - - setPrivateKey(privateKeyInfo); - } catch (Exception e) { - throw throwUnchecked(e); - } - } - - private synchronized void setPrivateKey(PrivateKeyInfo privateKey) throws PEMException { - JcaPEMKeyConverter converter = new JcaPEMKeyConverter().setProvider(bouncyCastleProvider); - this.privateKey = converter.getPrivateKey(privateKey); - } - - private void loadCertificates(PEMParser parser) { - try { - Object pemObject; - while ((pemObject = parser.readObject()) != null) { - addCertificate(pemObject); - } - - if (aliasToCertificate.isEmpty()) - throw new RuntimeException("No certificates available"); - } catch (Exception e) { - throw throwUnchecked(e); - } - } - - private synchronized void addCertificate(Object pemObject) throws CertificateException { - if (pemObject instanceof X509CertificateHolder) { - JcaX509CertificateConverter converter = new JcaX509CertificateConverter().setProvider(bouncyCastleProvider); - String alias = "cert-" + aliasToCertificate.size(); - aliasToCertificate.put(alias, converter.getCertificate((X509CertificateHolder) pemObject)); - } else { - throw new UnsupportedOperationException("Expected X509 certificate, got " + pemObject.getClass()); - } - } - - @Override - public synchronized Enumeration<String> engineAliases() { - return Collections.enumeration(storeRole.engineAliases()); - - } - - @Override - public synchronized boolean engineIsKeyEntry(String alias) { - return KEY_ALIAS.equals(alias); - } - - @Override - public synchronized Key engineGetKey(String alias, char[] password) throws NoSuchAlgorithmException, UnrecoverableKeyException { - Preconditions.checkArgument(KEY_ALIAS.equals(alias)); - return privateKey; - } - - @Override - public synchronized boolean engineIsCertificateEntry(String alias) { - return aliasToCertificate.containsKey(alias); - } - - - @Override - public synchronized Certificate engineGetCertificate(String alias) { - return aliasToCertificate.get(alias); - } - - @Override - public synchronized Certificate[] engineGetCertificateChain(String alias) { - Preconditions.checkArgument(KEY_ALIAS.equals(alias)); - return aliasToCertificate.values().toArray(new Certificate[aliasToCertificate.size()]); - } - - - @Override - public synchronized boolean engineContainsAlias(String alias) { - return storeRole.engineContainsAlias(alias); - } - - @Override - public synchronized int engineSize() { - return storeRole.engineSize(); - } - - @Override - public synchronized String engineGetCertificateAlias(final Certificate certificate) { - for (Entry<String, Certificate> entry : aliasToCertificate.entrySet()) { - if (entry.getValue() == certificate) - return entry.getKey(); - } - - return null; - } - - @Override - public synchronized Date engineGetCreationDate(String alias) { - throw new UnsupportedOperationException(); - } - - @Override - public synchronized void engineSetKeyEntry(String alias, Key key, char[] password, Certificate[] chain) throws KeyStoreException { - throw new UnsupportedOperationException(); - } - - @Override - public synchronized void engineSetKeyEntry(String alias, byte[] key, Certificate[] chain) throws KeyStoreException { - throw new UnsupportedOperationException(); - } - - @Override - public synchronized void engineSetCertificateEntry(String alias, Certificate cert) throws KeyStoreException { - throw new UnsupportedOperationException(); - } - - @Override - public synchronized void engineDeleteEntry(String alias) throws KeyStoreException { - throw new UnsupportedOperationException(); - } - - - @Override - public synchronized void engineStore(OutputStream stream, char[] password) throws IOException, NoSuchAlgorithmException, CertificateException { - throw new UnsupportedOperationException(); - } - - @Override - public synchronized void engineLoad(InputStream stream, char[] password) throws IOException, NoSuchAlgorithmException, CertificateException { - throw new UnsupportedOperationException(); - } - - private interface StoreRole { - Collection<String> engineAliases(); - boolean engineContainsAlias(String alias); - int engineSize(); - } - - private class KeyStoreRole implements StoreRole { - @Override - public Collection<String> engineAliases() { - return Collections.singletonList(KEY_ALIAS); - } - - @Override - public boolean engineContainsAlias(String alias) { - return KEY_ALIAS.equals(alias); - } - - @Override - public int engineSize() { - return 1; - } - } - - private class TrustStoreRole implements StoreRole{ - @Override - public Collection<String> engineAliases() { - return aliasToCertificate.keySet(); - } - - @Override - public boolean engineContainsAlias(String alias) { - return aliasToCertificate.containsKey(alias); - } - - @Override - public int engineSize() { - return aliasToCertificate.size(); - } - } - - // A reader along with the path used to construct it. - private static class ReaderForPath { - final Reader reader; - final Path path; - - private ReaderForPath(Reader reader, Path path) { - this.reader = reader; - this.path = path; - } - - static ReaderForPath of(Path path) { - try { - return new ReaderForPath(Files.newBufferedReader(path), path); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - } - - static class TrustStoreLoadParameter implements KeyStore.LoadStoreParameter { - final ReaderForPath certificateReader; - - TrustStoreLoadParameter(Path certificateReader) { - this.certificateReader = ReaderForPath.of(certificateReader); - } - - @Override - public KeyStore.ProtectionParameter getProtectionParameter() { - return null; - } - } - - static class KeyStoreLoadParameter implements KeyStore.LoadStoreParameter { - final ReaderForPath certificateReader; - final ReaderForPath keyReader; - - KeyStoreLoadParameter(Path certificateReader, Path keyReader) { - this.certificateReader = ReaderForPath.of(certificateReader); - this.keyReader = ReaderForPath.of(keyReader); - } - - @Override - public KeyStore.ProtectionParameter getProtectionParameter() { - return null; - } - } - -} diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/pem/PemSslKeyStore.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/pem/PemSslKeyStore.java deleted file mode 100644 index fe41d74f692..00000000000 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/pem/PemSslKeyStore.java +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.http.ssl.pem; - -import com.yahoo.jdisc.http.ssl.pem.PemKeyStore.KeyStoreLoadParameter; -import com.yahoo.jdisc.http.ssl.pem.PemKeyStore.TrustStoreLoadParameter; - -import java.io.IOException; -import java.nio.file.Path; -import java.security.KeyStore; -import java.security.KeyStore.LoadStoreParameter; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.Provider; -import java.security.Security; -import java.security.cert.CertificateException; - -/** - * Responsible for creating pem key stores. - * - * @author Tony Vaagenes - * @author bjorncs - */ -public class PemSslKeyStore { - - static { - Security.addProvider(new PemKeyStoreProvider()); - } - - private static final String KEY_STORE_TYPE = "PEM"; - - private final LoadStoreParameter loadParameter; - private KeyStore keyStore; - - public PemSslKeyStore(Path certificatePath, Path keyPath) { - this.loadParameter = new KeyStoreLoadParameter(certificatePath, keyPath); - } - - public PemSslKeyStore(Path certificatePath) { - this.loadParameter = new TrustStoreLoadParameter(certificatePath); - } - - public KeyStore loadJavaKeyStore() - throws KeyStoreException, CertificateException, NoSuchAlgorithmException, IOException { - if (keyStore == null) { - keyStore = KeyStore.getInstance(KEY_STORE_TYPE); - keyStore.load(loadParameter); - } - return keyStore; - } - - private static class PemKeyStoreProvider extends Provider { - - static final String NAME = "PEMKeyStoreProvider"; - static final double VERSION = 1; - static final String DESCRIPTION = "Provides PEM keystore support"; - - @SuppressWarnings("deprecation") // TODO: Remove annotation and use new super ctor when we don't need Java 8 support anymore. - PemKeyStoreProvider() { - super(NAME, VERSION, DESCRIPTION); - putService(new Service(this, "KeyStore", "PEM", PemKeyStore. class.getName(), PemKeyStore.aliases, PemKeyStore.attributes)); - } - } - -} diff --git a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def index 676fdbc8157..30c74ed2c35 100644 --- a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def +++ b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def @@ -25,10 +25,6 @@ acceptQueueSize int default=0 # Whether the server socket reuses addresses. reuseAddress bool default=true -# TODO Vespa 7: Remove soLingerTime - Jetty no longer support it. -# DEPRECATED No longer in use -soLingerTime double default=-1.0 - # The maximum idle time for a connection, which roughly translates to the Socket.setSoTimeout(int). idleTimeout double default=180.0 @@ -70,60 +66,3 @@ ssl.caCertificateFile string default="" # Client authentication mode. See SSLEngine.getNeedClientAuth()/getWantClientAuth() for details. ssl.clientAuth enum { DISABLED, WANT_AUTH, NEED_AUTH } default=DISABLED - - -######################################################################################### -# Config below is deprecated. Do not use -######################################################################################### - -# The name of the key to the password to the key store if in the secret store, if JKS is used. -# Must be empty with PEM -# By default this is also used to look up the password to the trust store. -ssl.keyDbKey string default="" -# TODO Rename keyDbKey to keyStorePassword after introducing custom services.xml syntax - -# Names of protocols to exclude. -ssl.excludeProtocol[].name string - -# Names of protocols to include. -ssl.includeProtocol[].name string - -# Names of cipher suites to exclude. -ssl.excludeCipherSuite[].name string - -# Names of cipher suites to include. -ssl.includeCipherSuite[].name string - -# The type of the keystore. -ssl.keyStoreType enum { JKS, PEM } default=JKS - -# JKS only - the path to the keystore. -ssl.keyStorePath string default="" - -ssl.pemKeyStore.keyPath string default="" -ssl.pemKeyStore.certificatePath string default="" - -ssl.trustStoreType enum { JKS } default=JKS - -# JKS only - the path to the truststore. -ssl.trustStorePath string default="" - -# TODO Add separate config for truststore password - -# Whether we should use keyDbKey as password to the trust store (true, default), -# or use no password with the trust store (false) -ssl.useTrustStorePassword bool default=true -# TODO Fix broken semantics with truststore and keystore password in Vespa 7 / Vespa 8 - -# The algorithm name used by the KeyManagerFactory. -ssl.sslKeyManagerFactoryAlgorithm string default="SunX509" - -# The SSL protocol passed to SSLContext.getInstance() -ssl.protocol string default="TLS" - -# The SecureRandom implementation passed to SSLEngine.init() -# Java have a default pseudo-random number generator (PRNG) for crypto operations. This default may have performance -# issues on some platform (e.g. NativePRNG in Linux utilizes a global lock). Changing the generator to SHA1PRNG may -# improve performance. Set value to empty string to use the default generator. -ssl.prng string default="" - diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/CookieTestCase.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/CookieTestCase.java index dab4f91f631..4c651f79666 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/CookieTestCase.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/CookieTestCase.java @@ -131,7 +131,6 @@ public class CookieTestCase { } @Test - @SuppressWarnings("deprecation") public void requireThatSetCookieCanBeDecoded() { final Cookie foo = new Cookie(); foo.setName("foo.name"); @@ -141,7 +140,6 @@ public class CookieTestCase { foo.setMaxAge(0, TimeUnit.SECONDS); foo.setSecure(true); foo.setHttpOnly(true); - foo.setVersion(1); assertDecodeSetCookie(foo, "foo.name=foo.value;Max-Age=0;Path=path;Domain=domain;Secure;HTTPOnly;"); final Cookie bar = new Cookie(); @@ -150,7 +148,6 @@ public class CookieTestCase { bar.setPath("path"); bar.setDomain("domain"); bar.setMaxAge(0, TimeUnit.SECONDS); - bar.setVersion(1); assertDecodeSetCookie(bar, "bar.name=bar.value;Max-Age=0;Path=path;Domain=domain;"); } @@ -193,11 +190,13 @@ public class CookieTestCase { } private static void assertEncodeCookie(String expectedResult, List<Cookie> cookies) { - assertThat(Cookie.toCookieHeader(cookies), equalTo(expectedResult)); + String actual = Cookie.toCookieHeader(cookies); + String expectedResult1 = expectedResult; + assertThat(actual, equalTo(expectedResult1)); } private static void assertEncodeSetCookie(List<String> expectedResult, List<Cookie> cookies) { - assertThat(Cookie.toSetCookieHeaderAll(cookies), containsInAnyOrder(expectedResult.toArray())); + assertThat(Cookie.toSetCookieHeaders(cookies), containsInAnyOrder(expectedResult.toArray())); } private static void assertDecodeCookie(List<Cookie> expected, String toDecode) { @@ -205,7 +204,7 @@ public class CookieTestCase { } private static void assertDecodeSetCookie(final Cookie expected, String toDecode) { - assertThat(Cookie.fromSetCookieHeader(toDecode), containsInAnyOrder(expected)); + assertThat(Cookie.fromSetCookieHeader(toDecode), equalTo(expected)); } private static Cookie newCookie(final String name) { diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/HttpResponseTestCase.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/HttpResponseTestCase.java index 027d50317b2..d727be020cb 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/HttpResponseTestCase.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/HttpResponseTestCase.java @@ -92,7 +92,7 @@ public class HttpResponseTestCase { response.encodeSetCookieHeader(cookies); final List<String> headers = response.headers().get(HttpHeaders.Names.SET_COOKIE); assertEquals(1, headers.size()); - assertEquals(Cookie.toSetCookieHeaderAll(cookies), headers); + assertEquals(Cookie.toSetCookieHeaders(cookies), headers); } @Test @@ -102,7 +102,7 @@ public class HttpResponseTestCase { response.encodeSetCookieHeader(cookies); final List<String> headers = response.headers().get(HttpHeaders.Names.SET_COOKIE); assertEquals(2, headers.size()); - assertEquals(Cookie.toSetCookieHeaderAll(Arrays.asList(new Cookie("foo", "bar"), new Cookie("baz", "cox"))), + assertEquals(Cookie.toSetCookieHeaders(Arrays.asList(new Cookie("foo", "bar"), new Cookie("baz", "cox"))), headers); } diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/filter/JDiscCookieWrapperTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/filter/JDiscCookieWrapperTest.java index 3439dc172ee..86ca424c90e 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/filter/JDiscCookieWrapperTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/filter/JDiscCookieWrapperTest.java @@ -7,7 +7,6 @@ import org.testng.annotations.Test; import java.util.concurrent.TimeUnit; -@SuppressWarnings("deprecation") public class JDiscCookieWrapperTest { @Test @@ -15,19 +14,15 @@ public class JDiscCookieWrapperTest { Cookie cookie = new Cookie("name", "value"); JDiscCookieWrapper wrapper = JDiscCookieWrapper.wrap(cookie); - wrapper.setComment("comment"); wrapper.setDomain("yahoo.com"); wrapper.setMaxAge(10); wrapper.setPath("/path"); - wrapper.setVersion(1); Assert.assertEquals(wrapper.getName(), cookie.getName()); Assert.assertEquals(wrapper.getValue(), cookie.getValue()); Assert.assertEquals(wrapper.getDomain(), cookie.getDomain()); - Assert.assertEquals(wrapper.getComment(), cookie.getComment()); Assert.assertEquals(wrapper.getMaxAge(), cookie.getMaxAge(TimeUnit.SECONDS)); Assert.assertEquals(wrapper.getPath(), cookie.getPath()); - Assert.assertEquals(wrapper.getVersion(), cookie.getVersion()); Assert.assertEquals(wrapper.getSecure(), cookie.isSecure()); } diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLogTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLogTest.java index 1048d7b6422..d5043f7b989 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLogTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLogTest.java @@ -15,11 +15,12 @@ import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.testng.Assert.assertTrue; /** - * @author bakksjo + * @author Oyvind Bakksjo + * @author bjorncs */ -@SuppressWarnings("deprecation") // AccessLogEntry.setURI/getURI are deprecated public class AccessLogRequestLogTest { @Test public void requireThatQueryWithUnquotedSpecialCharactersIsHandled() { @@ -30,7 +31,8 @@ public class AccessLogRequestLogTest { AccessLogRequestLog.populateAccessLogEntryFromHttpServletRequest(httpServletRequest, accessLogEntry); - assertThat(accessLogEntry.getURI(), is(not(nullValue()))); + assertThat(accessLogEntry.getRawPath(), is(not(nullValue()))); + assertTrue(accessLogEntry.getRawQuery().isPresent()); } @Test @@ -44,37 +46,12 @@ public class AccessLogRequestLogTest { AccessLogRequestLog.populateAccessLogEntryFromHttpServletRequest(httpServletRequest, accessLogEntry); - assertThat(accessLogEntry.getURI().toString(), is(path + '?' + query)); + assertThat(accessLogEntry.getRawPath(), is(path)); + assertThat(accessLogEntry.getRawQuery().get(), is(query)); } @Test - public void requireThatNoQueryPartIsHandledWhenRequestIsMalformed() { - final HttpServletRequest httpServletRequest = mock(HttpServletRequest.class); - final String path = "/s>earch/"; - when(httpServletRequest.getRequestURI()).thenReturn(path); - final String query = null; - when(httpServletRequest.getQueryString()).thenReturn(query); - final AccessLogEntry accessLogEntry = new AccessLogEntry(); - - AccessLogRequestLog.populateAccessLogEntryFromHttpServletRequest(httpServletRequest, accessLogEntry); - - assertThat(accessLogEntry.getURI().toString(), is("/s%3Eearch/")); - - } - - @Test - public void invalid_percent_escape_patterns_in_query_string_are_escaped() { - HttpServletRequest httpServletRequest = mock(HttpServletRequest.class); - when(httpServletRequest.getRequestURI()).thenReturn("/search/"); - when(httpServletRequest.getQueryString()).thenReturn("q=%%2"); - - AccessLogEntry accessLogEntry = new AccessLogEntry(); - AccessLogRequestLog.populateAccessLogEntryFromHttpServletRequest(httpServletRequest, accessLogEntry); - assertThat(accessLogEntry.getURI().toString(), is("/search/?q=%25%252")); - } - - @Test public void raw_path_and_query_are_set_from_request() { HttpServletRequest httpServletRequest = mock(HttpServletRequest.class); String rawPath = "//search/"; diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java index eb18a3ee341..cf32801ce88 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java @@ -17,9 +17,6 @@ import java.net.InetSocketAddress; import java.nio.channels.ServerSocketChannel; import java.util.Map; -import static com.yahoo.jdisc.http.ConnectorConfig.Ssl; -import static com.yahoo.jdisc.http.ConnectorConfig.Ssl.KeyStoreType.Enum.JKS; -import static com.yahoo.jdisc.http.ConnectorConfig.Ssl.KeyStoreType.Enum.PEM; import static org.hamcrest.CoreMatchers.equalTo; /** @@ -27,30 +24,6 @@ import static org.hamcrest.CoreMatchers.equalTo; */ public class ConnectorFactoryTest { - @Test(expectedExceptions = IllegalArgumentException.class) - public void ssl_jks_config_is_validated() { - ConnectorConfig config = new ConnectorConfig( - new ConnectorConfig.Builder() - .ssl(new Ssl.Builder() - .enabled(true) - .keyStoreType(JKS) - .pemKeyStore( - new Ssl.PemKeyStore.Builder() - .keyPath("nonEmpty")))); - ConnectorFactory willThrowException = createConnectorFactory(config); - } - - @Test(expectedExceptions = IllegalArgumentException.class) - public void ssl_pem_config_is_validated() { - ConnectorConfig config = new ConnectorConfig( - new ConnectorConfig.Builder() - .ssl(new Ssl.Builder() - .enabled(true) - .keyStoreType(PEM) - .keyStorePath("nonEmpty"))); - ConnectorFactory willThrowException = createConnectorFactory(config); - } - @Test public void requireThatNoPreBoundChannelWorks() throws Exception { Server server = new Server(); diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactoryTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactoryTest.java index a15bf4c117a..6a2e35b617c 100644 --- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactoryTest.java +++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactoryTest.java @@ -492,16 +492,6 @@ public class HttpRequestFactoryTest { } @Test - public final void test() { - String noise = "query=a" + "\\" + "^{|}&other=madeit"; - HttpServletRequest servletRequest = new MockRequest( - "http://yahoo.com/search?" + noise); - HttpRequest request = HttpRequestFactory.newJDiscRequest( - new MockContainer(), servletRequest); - assertThat(request.getUri().getQuery(), equalTo(noise)); - } - - @Test public final void testIllegalQuery() { try { HttpRequestFactory.newJDiscRequest( diff --git a/jdisc_jetty/pom.xml b/jdisc_jetty/pom.xml index 404476f7bf2..89e573651e9 100644 --- a/jdisc_jetty/pom.xml +++ b/jdisc_jetty/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>jdisc_jetty</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>jar</packaging> <dependencies> <dependency> diff --git a/jdisc_messagebus_service/pom.xml b/jdisc_messagebus_service/pom.xml index 6201f413b9a..eb41609a500 100644 --- a/jdisc_messagebus_service/pom.xml +++ b/jdisc_messagebus_service/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>jdisc_messagebus_service</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> diff --git a/jrt/pom.xml b/jrt/pom.xml index 641ed7e69ac..5208c0417cc 100644 --- a/jrt/pom.xml +++ b/jrt/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>jrt</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>jrt</name> <dependencies> <dependency> diff --git a/linguistics/abi-spec.json b/linguistics/abi-spec.json index df8d008ec42..d56e56e23b5 100644 --- a/linguistics/abi-spec.json +++ b/linguistics/abi-spec.json @@ -210,12 +210,9 @@ "public abstract com.yahoo.language.process.Segmenter getSegmenter()", "public abstract com.yahoo.language.detect.Detector getDetector()", "public abstract com.yahoo.language.process.GramSplitter getGramSplitter()", - "public abstract com.yahoo.language.process.CharacterClasses getCharacterClasses()", - "public abstract com.yahoo.collections.Tuple2 getVersion(com.yahoo.language.Linguistics$Component)" + "public abstract com.yahoo.language.process.CharacterClasses getCharacterClasses()" ], - "fields": [ - "public static final com.yahoo.language.Linguistics SIMPLE" - ] + "fields": [] }, "com.yahoo.language.LinguisticsCase": { "superClass": "java.lang.Object", @@ -460,9 +457,7 @@ ], "methods": [ "public static com.yahoo.language.process.StemMode[] values()", - "public static com.yahoo.language.process.StemMode valueOf(java.lang.String)", - "public int getValue()", - "public static com.yahoo.language.process.StemMode valueOf(int)" + "public static com.yahoo.language.process.StemMode valueOf(java.lang.String)" ], "fields": [ "public static final enum com.yahoo.language.process.StemMode NONE", diff --git a/linguistics/pom.xml b/linguistics/pom.xml index 093b731f180..e341258c75f 100644 --- a/linguistics/pom.xml +++ b/linguistics/pom.xml @@ -7,12 +7,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>linguistics</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>org.hamcrest</groupId> diff --git a/linguistics/src/main/java/com/yahoo/language/Language.java b/linguistics/src/main/java/com/yahoo/language/Language.java index ab8bcd4459f..655a9003fb1 100644 --- a/linguistics/src/main/java/com/yahoo/language/Language.java +++ b/linguistics/src/main/java/com/yahoo/language/Language.java @@ -529,10 +529,10 @@ public enum Language { } /** - * <p>Convenience method for calling <tt>fromLocale(LocaleFactory.fromLanguageTag(languageTag))</tt>.</p> + * <p>Convenience method for calling <code>fromLocale(LocaleFactory.fromLanguageTag(languageTag))</code>.</p> * - * @param languageTag The language tag for which the <tt>Language</tt> to return. - * @return the corresponding <tt>Language</tt>, or {@link #UNKNOWN} if not known. + * @param languageTag The language tag for which the <code>Language</code> to return. + * @return the corresponding <code>Language</code>, or {@link #UNKNOWN} if not known. */ public static Language fromLanguageTag(String languageTag) { if (languageTag == null) return UNKNOWN; @@ -540,7 +540,7 @@ public enum Language { } /** - * <p>Returns the <tt>Language</tt> whose {@link #languageCode()} is equal to <tt>locale.getLanguage()</tt>, with + * <p>Returns the <code>Language</code> whose {@link #languageCode()} is equal to <code>locale.getLanguage()</code>, with * the following additions:</p> * <ul> * <li>Language code "in" translates to {@link #INDONESIAN}</li> @@ -551,8 +551,8 @@ public enum Language { * is "hans", in which case it translates to {@link #CHINESE_SIMPLIFIED}.</li> * </ul> * - * @param locale The locale for which the <tt>Language</tt> to return. - * @return The corresponding <tt>Language</tt>, or {@link #UNKNOWN} if not known. + * @param locale The locale for which the <code>Language</code> to return. + * @return The corresponding <code>Language</code>, or {@link #UNKNOWN} if not known. */ public static Language fromLocale(Locale locale) { String str = locale.getLanguage(); @@ -582,7 +582,7 @@ public enum Language { /** * Returns the language from an encoding, or {@link #UNKNOWN} if it cannot be determined. * - * @param encoding The name of the encoding to derive the <tt>Language</tt> from. + * @param encoding The name of the encoding to derive the <code>Language</code> from. * @return the language given by the encoding, or {@link #UNKNOWN} if not determined. */ public static Language fromEncoding(String encoding) { diff --git a/linguistics/src/main/java/com/yahoo/language/Linguistics.java b/linguistics/src/main/java/com/yahoo/language/Linguistics.java index 9006d855faa..ec92c5e857e 100644 --- a/linguistics/src/main/java/com/yahoo/language/Linguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/Linguistics.java @@ -42,15 +42,6 @@ public interface Linguistics { } /** - * The same as new com.yahoo.language.simple.SimpleLinguistics(). Prefer using that directly. - * - * @deprecated use new com.yahoo.language.simple.SimpleLinguistics() - */ - @Deprecated // OK - // TODO: Remove this field on Vespa 7 - Linguistics SIMPLE = new SimpleLinguistics(); - - /** * Returns a thread-unsafe stemmer or lemmatizer. * This is used at query time to do stemming of search terms to indexes which contains text tokenized * with stemming turned on @@ -98,13 +89,4 @@ public interface Linguistics { /** Returns a thread-unsafe character classes instance. */ CharacterClasses getCharacterClasses(); - /** - * Returns the name and version of a processor component returned by - * this instance. - * - * @deprecated do not use - */ - @Deprecated // OK - Tuple2<String, Version> getVersion(Linguistics.Component component); - } diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java index 38181261d6a..1c7c71c00b6 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java @@ -1,14 +1,43 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.opennlp; +import com.google.inject.Inject; +import com.yahoo.language.detect.Detector; import com.yahoo.language.process.Tokenizer; +import com.yahoo.language.simple.SimpleDetector; import com.yahoo.language.simple.SimpleLinguistics; +/** + * Returns a linguistics implementation based on OpenNlp, + * and (optionally, default on) Optimaize for language detection. + */ public class OpenNlpLinguistics extends SimpleLinguistics { + private final Detector detector; + + public OpenNlpLinguistics() { + this(true); + } + + @Inject + public OpenNlpLinguistics(OpennlpLinguisticsConfig config) { + this(config.detector().enableOptimaize()); + } + + public OpenNlpLinguistics(boolean enableOptimaize) { + this(enableOptimaize ? new OptimaizeDetector() : new SimpleDetector()); + } + + private OpenNlpLinguistics(Detector detector) { + this.detector = detector; + } + @Override public Tokenizer getTokenizer() { return new OpenNlpTokenizer(getNormalizer(), getTransformer()); } + @Override + public Detector getDetector() { return detector; } + } diff --git a/linguistics/src/main/java/com/yahoo/language/process/StemMode.java b/linguistics/src/main/java/com/yahoo/language/process/StemMode.java index af486f715b0..628f6910c9e 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/StemMode.java +++ b/linguistics/src/main/java/com/yahoo/language/process/StemMode.java @@ -22,26 +22,4 @@ public enum StemMode { this.value = value; } - /** - * Returns the stem mode as an int - * - * @deprecated do not use - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public int getValue() { - return value; - } - - @Deprecated // OK - // TODO: Remove on Vespa 7 - public static StemMode valueOf(int value) { - for (StemMode mode : values()) { - if (mode.value == value) { - return mode; - } - } - return NONE; - } - } diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java index 1edfe5c804e..3de0eb3e997 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java @@ -1,26 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.language.simple; -import com.google.common.base.Optional; -import com.optimaize.langdetect.LanguageDetector; -import com.optimaize.langdetect.LanguageDetectorBuilder; -import com.optimaize.langdetect.i18n.LdLocale; -import com.optimaize.langdetect.ngram.NgramExtractors; -import com.optimaize.langdetect.profiles.LanguageProfile; -import com.optimaize.langdetect.profiles.LanguageProfileReader; -import com.optimaize.langdetect.text.CommonTextObjectFactories; -import com.optimaize.langdetect.text.TextObject; -import com.optimaize.langdetect.text.TextObjectFactory; import com.yahoo.language.Language; import com.yahoo.language.detect.Detection; import com.yahoo.language.detect.Detector; import com.yahoo.language.detect.Hint; import com.yahoo.text.Utf8; -import java.io.IOException; import java.nio.ByteBuffer; -import java.util.List; -import java.util.Locale; /** * Includes functionality for determining the langCode from a sample or from the encoding. @@ -38,55 +25,6 @@ import java.util.Locale; */ public class SimpleDetector implements Detector { - static private Object initGuard = new Object(); - static private TextObjectFactory textObjectFactory = null; - static private LanguageDetector languageDetector = null; - - static private void initOptimaize (boolean useOptimaize) { - if (!useOptimaize) return; - synchronized (initGuard) { - if ((textObjectFactory != null) && (languageDetector != null)) return; - - // origin: https://github.com/optimaize/language-detector - //load all languages: - List<LanguageProfile> languageProfiles; - try { - languageProfiles = new LanguageProfileReader().readAllBuiltIn(); - } catch (IOException e) { - throw new RuntimeException(e); - } - - //build language detector: - languageDetector = LanguageDetectorBuilder.create(NgramExtractors.standard()) - .withProfiles(languageProfiles) - .build(); - - //create a text object factory - textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText(); - } - } - - private final boolean enableOptimaize; - - /** @deprecated use OptimaizeDetector to enable optimaize */ - @Deprecated - SimpleDetector(boolean enableOptimaize) { - initOptimaize(enableOptimaize); - this.enableOptimaize = enableOptimaize; - - } - - @SuppressWarnings("deprecation") - public SimpleDetector() { - this(true); - } - - /** @deprecated use OptimaizeDetector to enable optimaize */ - @Deprecated - public SimpleDetector(SimpleLinguisticsConfig.Detector detector) { - this(detector.enableOptimaize()); - } - @Override public Detection detect(byte[] input, int offset, int length, Hint hint) { return new Detection(guessLanguage(input, offset, length), guessEncoding(input), false); @@ -172,26 +110,10 @@ public class SimpleDetector implements Detector { return Language.THAI; } } - if (enableOptimaize && Language.UNKNOWN.equals(soFar)){ - return detectLangOptimaize(input); - } // got to the end, so return the current best guess return soFar; } - private static Language detectLangOptimaize(String input) { - if (input == null || input.length() == 0) { - return Language.UNKNOWN; - } - TextObject textObject = textObjectFactory.forText(input); - Optional<LdLocale> lang = languageDetector.detect(textObject); - if (lang.isPresent()) { - String language = lang.get().getLanguage(); - return Language.fromLocale(new Locale(language)); - } - return Language.UNKNOWN; - } - private boolean isTrailingOctet(byte i) { return ((i >>> 6) & 3) == 2; } diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java index b7bf0215ca4..389926f1c1b 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java @@ -17,7 +17,8 @@ import com.yahoo.language.process.Tokenizer; import com.yahoo.language.process.Transformer; /** - * Factory of pure Java linguistic processor implementations. + * Factory of simple linguistic processor implementations. + * Useful for testing and english-only use cases. * * @author bratseth * @author bjorncs @@ -34,26 +35,9 @@ public class SimpleLinguistics implements Linguistics { @Inject @SuppressWarnings("deprecation") public SimpleLinguistics() { - this(true); - - } - - /** @deprecated use OpenNlpLinguistics to get optimaize */ - @Deprecated // OK - public SimpleLinguistics(boolean enableOptimaize) { - this(new SimpleDetector(enableOptimaize)); - } - - /** @deprecated use OpenNlpLinguistics to get optimaize */ - @Deprecated // OK - public SimpleLinguistics(SimpleLinguisticsConfig config) { - this(new SimpleDetector(config.detector())); - } - - private SimpleLinguistics(Detector detector) { this.normalizer = new SimpleNormalizer(); this.transformer = new SimpleTransformer(); - this.detector = detector; + this.detector = new SimpleDetector(); this.characterClasses = new CharacterClasses(); this.gramSplitter = new GramSplitter(characterClasses); } @@ -82,11 +66,4 @@ public class SimpleLinguistics implements Linguistics { @Override public CharacterClasses getCharacterClasses() { return characterClasses; } - /** @deprecated do not use */ - @Deprecated // OK - @Override - public Tuple2<String, Version> getVersion(Component component) { - return new Tuple2<>("yahoo", new Version(1, 0)); - } - } diff --git a/linguistics/src/main/java/com/yahoo/language/simple/kstem/CharacterUtils.java b/linguistics/src/main/java/com/yahoo/language/simple/kstem/CharacterUtils.java index 7bb13744e20..3b7dcca3bc1 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/kstem/CharacterUtils.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/kstem/CharacterUtils.java @@ -24,16 +24,6 @@ public abstract class CharacterUtils { public static CharacterUtils getInstance() { return JAVA_5; } - - /** - * explicitly returns a version matching java 4 semantics - * @deprecated Only for n-gram backwards compat - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public static CharacterUtils getJava4Instance() { - return JAVA_4; - } /** * Returns the code point at the given index of the {@link CharSequence}. diff --git a/linguistics/src/main/resources/configdefinitions/simple-linguistics.def b/linguistics/src/main/resources/configdefinitions/simple-linguistics.def deleted file mode 100644 index 1ddca52c443..00000000000 --- a/linguistics/src/main/resources/configdefinitions/simple-linguistics.def +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Deprecated: Do not use -namespace=language.simple - -# Enable Optimaize language detector -detector.enableOptimaize bool default=true - diff --git a/linguistics/src/test/java/com/yahoo/language/process/StemModeTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/StemModeTestCase.java deleted file mode 100644 index 6433d0d491a..00000000000 --- a/linguistics/src/test/java/com/yahoo/language/process/StemModeTestCase.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.language.process; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author Simon Thoresen Hult - */ -public class StemModeTestCase { - - @Test - @SuppressWarnings("deprecation") - public void requireThatValueOfWorks() { - for (StemMode mode : StemMode.values()) { - assertEquals(mode, StemMode.valueOf(mode.getValue())); - } - } - - @Test - @SuppressWarnings("deprecation") - public void requireThatValueOfUnknownIsNone() { - assertEquals(StemMode.NONE, StemMode.valueOf(-1)); - } - -} diff --git a/logd/pom.xml b/logd/pom.xml index a1a107bc20c..4e1eb802cfa 100644 --- a/logd/pom.xml +++ b/logd/pom.xml @@ -7,11 +7,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>logd</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/logserver/pom.xml b/logserver/pom.xml index 71eda0fb15b..be2d8ba4cee 100644 --- a/logserver/pom.xml +++ b/logserver/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>logserver</artifactId> <packaging>jar</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>logserver</name> <description>Log server.</description> <dependencies> diff --git a/maven-plugins/pom.xml b/maven-plugins/pom.xml index be03926499f..b363c6adb2f 100644 --- a/maven-plugins/pom.xml +++ b/maven-plugins/pom.xml @@ -5,14 +5,14 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <groupId>com.yahoo.vespa</groupId> <artifactId>bundle-plugins</artifactId> <name>bundle-plugins</name> <packaging>pom</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <description>Parent artifact for Vespa maven plugins.</description> <url>http://yahoo.github.io/vespa</url> diff --git a/messagebus-disc/pom.xml b/messagebus-disc/pom.xml index 7b37dc2d197..843936e3290 100644 --- a/messagebus-disc/pom.xml +++ b/messagebus-disc/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>messagebus-disc</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/messagebus/pom.xml b/messagebus/pom.xml index e949088e737..358ce7cc63f 100644 --- a/messagebus/pom.xml +++ b/messagebus/pom.xml @@ -7,11 +7,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>messagebus</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>jar</packaging> <dependencies> <dependency> diff --git a/messagebus/src/main/java/com/yahoo/messagebus/Routable.java b/messagebus/src/main/java/com/yahoo/messagebus/Routable.java index e761a874ac1..792ec4d27cd 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/Routable.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/Routable.java @@ -60,7 +60,7 @@ public abstract class Routable { /** * <p>This is a convenience method for calling {@link CallStack#pop(Routable)} on the {@link CallStack} of this - * Routable. It equals calling <tt>routable.getCallStack().pop(routable)</tt>.</p> + * Routable. It equals calling <code>routable.getCallStack().pop(routable)</code>.</p> * * @return The handler that was popped. * @see CallStack#pop(Routable) diff --git a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java index 3049639586d..d2d8031a689 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java @@ -258,7 +258,7 @@ public class RoutingNode implements ReplyHandler { } /** - * If a reply has been set containing an error, and {@link #shouldIgnoreResult()} returns <tt>true</tt>, this method + * If a reply has been set containing an error, and {@link #shouldIgnoreResult()} returns <code>true</code>, this method * replaces that reply with one that has no error. * * @return Whether or not the reply was replaced. diff --git a/metrics/pom.xml b/metrics/pom.xml index 55556c2cb37..b99ae535719 100644 --- a/metrics/pom.xml +++ b/metrics/pom.xml @@ -5,13 +5,13 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <groupId>com.yahoo.vespa</groupId> <artifactId>metrics</artifactId> <packaging>jar</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>metrics</name> <description></description> <dependencies> diff --git a/model-evaluation/pom.xml b/model-evaluation/pom.xml index 2993e86bb78..db758814702 100644 --- a/model-evaluation/pom.xml +++ b/model-evaluation/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>model-evaluation</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> diff --git a/model-integration/pom.xml b/model-integration/pom.xml index 5e57ec22709..fb9e119f61d 100644 --- a/model-integration/pom.xml +++ b/model-integration/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>model-integration</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> diff --git a/node-admin/pom.xml b/node-admin/pom.xml index e1231f2585d..fda4acebfa0 100644 --- a/node-admin/pom.xml +++ b/node-admin/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>node-admin</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> diff --git a/node-admin/src/main/application/services.xml b/node-admin/src/main/application/services.xml index f1cfeb69660..db00c686c99 100644 --- a/node-admin/src/main/application/services.xml +++ b/node-admin/src/main/application/services.xml @@ -3,7 +3,7 @@ <services version="1.0" xmlns:preprocess="properties"> <container id="node-admin" version="1.0"> <!-- Please update container test when changing this file --> - <accesslog type="vespa" fileNamePattern="logs/vespa/node-admin/access.log.%Y%m%d%H%M%S" rotationScheme="date" symlinkName="access.log" /> + <accesslog type="vespa" fileNamePattern="logs/vespa/node-admin/access.log.%Y%m%d%H%M%S" symlinkName="access.log" /> <component id="docker-api" class="com.yahoo.vespa.hosted.dockerapi.DockerImpl" bundle="docker-api"/> <component id="metrics-wrapper" class="com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper" bundle="docker-api"/> diff --git a/node-maintainer/pom.xml b/node-maintainer/pom.xml index 720fbed2977..3c51bdc05be 100644 --- a/node-maintainer/pom.xml +++ b/node-maintainer/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>node-maintainer</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> diff --git a/node-repository/pom.xml b/node-repository/pom.xml index 80d5369f293..4202e4ebc23 100644 --- a/node-repository/pom.xml +++ b/node-repository/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>node-repository</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <description>Keeps track of node assignment in a multi-application setup.</description> diff --git a/orchestrator-restapi/pom.xml b/orchestrator-restapi/pom.xml index 310512812e2..4ed57b85670 100644 --- a/orchestrator-restapi/pom.xml +++ b/orchestrator-restapi/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>orchestrator-restapi</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/orchestrator/pom.xml b/orchestrator/pom.xml index 37d308111ae..d237ce3e4cc 100644 --- a/orchestrator/pom.xml +++ b/orchestrator/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>orchestrator</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Host.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Host.java index 8e06f3b342a..bda9505d72b 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Host.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Host.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.orchestrator.status.HostStatus; import java.util.List; public class Host { + private final HostName hostName; private final HostStatus hostStatus; private final ApplicationInstanceReference applicationInstanceReference; @@ -39,4 +40,5 @@ public class Host { public List<ServiceInstance> getServiceInstances() { return serviceInstances; } + } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java index df124f2f690..a639d07e504 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java @@ -33,8 +33,6 @@ public interface Orchestrator { /** * Get orchestrator information related to a host. - * - * @throws HostNameNotFoundException */ Host getHost(HostName hostName) throws HostNameNotFoundException; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java index 77bb1e99e19..33cfa310a68 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java @@ -45,6 +45,7 @@ import java.util.stream.Collectors; * @author smorgrav */ public class OrchestratorImpl implements Orchestrator { + private static final Logger log = Logger.getLogger(OrchestratorImpl.class.getName()); private final Policy policy; @@ -185,10 +186,9 @@ public class OrchestratorImpl implements Orchestrator { return; } - ApplicationApi applicationApi = new ApplicationApiImpl( - nodeGroup, - hostStatusRegistry, - clusterControllerClientFactory); + ApplicationApi applicationApi = new ApplicationApiImpl(nodeGroup, + hostStatusRegistry, + clusterControllerClientFactory); policy.grantSuspensionRequest(context.createSubcontextWithinLock(), applicationApi); } } @@ -386,4 +386,5 @@ public class OrchestratorImpl implements Orchestrator { throw new RuntimeException("Unexpectedly interrupted", e); } } + } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java index e2f371a5ce1..2e85713d323 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java @@ -13,8 +13,9 @@ import java.util.List; * The API a Policy has access to */ public interface ApplicationApi { + /** - * @return The 3-part application ID of the form tenant:name:instance. + * Returns the 3-part application ID of the form tenant:name:instance. */ ApplicationId applicationId(); @@ -33,4 +34,5 @@ public interface ApplicationApi { List<StorageNode> getStorageNodesInGroupInClusterOrder(); List<StorageNode> getUpStorageNodesInGroupInClusterOrder(); List<StorageNode> getStorageNodesAllowedToBeDownInGroupInReverseClusterOrder(); + } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java index 9ec1697a45f..5800b48da75 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java @@ -26,6 +26,7 @@ import java.util.stream.Collectors; import static com.yahoo.vespa.orchestrator.OrchestratorUtil.getHostsUsedByApplicationInstance; public class ApplicationApiImpl implements ApplicationApi { + private final ApplicationInstance applicationInstance; private final NodeGroup nodeGroup; private final MutableStatusRegistry hostStatusService; @@ -145,4 +146,5 @@ public class ApplicationApiImpl implements ApplicationApi { return serviceClustersInGroup; } + } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java index 4aa0f3452e3..b6e7014cac0 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java @@ -20,7 +20,6 @@ import java.util.logging.Logger; /** * @author oyving */ - public class HostedVespaPolicy implements Policy { public static final String APPLICATION_SUSPENDED_CONSTRAINT = "application-suspended"; @@ -29,8 +28,6 @@ public class HostedVespaPolicy implements Policy { public static final String CLUSTER_CONTROLLER_AVAILABLE_CONSTRAINT = "controller-available"; public static final String DEADLINE_CONSTRAINT = "deadline"; - private static final Logger log = Logger.getLogger(HostedVespaPolicy.class.getName()); - private final HostedVespaClusterPolicy clusterPolicy; private final ClusterControllerClientFactory clusterControllerClientFactory; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java index 9938d244657..e2487301326 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; * @author oyving */ public interface Policy { + /** * Decide whether to grant a request for temporarily suspending the services on all hosts in the group. */ @@ -20,9 +21,6 @@ public interface Policy { /** * Give all hosts in a group permission to be removed from the application. - * - * @param context - * @param applicationApi */ void acquirePermissionToRemove(OrchestratorContext context, ApplicationApi applicationApi) throws HostStateChangeDeniedException; @@ -35,4 +33,5 @@ public interface Policy { OrchestratorContext context, ApplicationInstance applicationInstance, HostName hostName, MutableStatusRegistry hostStatusService) throws HostStateChangeDeniedException; + } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java index 7f0227df4c5..6984389b688 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java @@ -146,4 +146,5 @@ public class InstanceResource { throw new WebApplicationException( Response.status(Response.Status.BAD_REQUEST).entity(message).build()); } + } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java index 3360a12c32e..c56ff661bba 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java @@ -215,14 +215,14 @@ public class ZookeeperStatusService implements StatusService { } private class ZkMutableStatusRegistry implements MutableStatusRegistry { + private final Lock lock; private final ApplicationInstanceReference applicationInstanceReference; private final boolean probe; - public ZkMutableStatusRegistry( - Lock lock, - ApplicationInstanceReference applicationInstanceReference, - boolean probe) { + public ZkMutableStatusRegistry(Lock lock, + ApplicationInstanceReference applicationInstanceReference, + boolean probe) { this.lock = lock; this.applicationInstanceReference = applicationInstanceReference; this.probe = probe; @@ -275,9 +275,12 @@ public class ZookeeperStatusService implements StatusService { lock.close(); } catch (RuntimeException e) { // We may want to avoid logging some exceptions that may be expected, like when session expires. - log.log(LogLevel.WARNING, "Failed to close application lock for " + - ZookeeperStatusService.class.getSimpleName() + ", will ignore and continue", e); + log.log(LogLevel.WARNING, + "Failed to close application lock for " + + ZookeeperStatusService.class.getSimpleName() + ", will ignore and continue", + e); } } } + } diff --git a/parent/pom.xml b/parent/pom.xml index 5137a7cf171..11282f3c36c 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -4,7 +4,7 @@ <modelVersion>4.0.0</modelVersion> <artifactId>parent</artifactId> <packaging>pom</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>parent</name> <description>Parent artifact for all Vespa maven projects.</description> <url>https://github.com/vespa-engine</url> @@ -12,7 +12,7 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>container-dependency-versions</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../container-dependency-versions/pom.xml</relativePath> </parent> @@ -99,8 +99,11 @@ <artifactId>maven-compiler-plugin</artifactId> <version>3.8.0</version> <configuration> - <source>1.8</source> - <target>1.8</target> + <jdkToolchain> + <version>11</version> + </jdkToolchain> + <source>11</source> + <target>11</target> <showWarnings>true</showWarnings> <optimize>true</optimize> <showDeprecation>false</showDeprecation> @@ -153,13 +156,16 @@ <doclint>${doclint},-missing</doclint> <quiet>true</quiet> <show>protected</show> + + <!-- TODO: remove this setting when the plugin looks for element-list (JDK 10+) instead of package-list. --> + <detectOfflineLinks>false</detectOfflineLinks> </configuration> <version>3.0.1</version> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-plugin-plugin</artifactId> - <version>3.5.2</version> + <version>${maven-plugin-tools.version}</version> <configuration> <!-- see http://jira.codehaus.org/browse/MNG-5346 --> <skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound> @@ -184,16 +190,8 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-shade-plugin</artifactId> - <version>3.1.1</version> - <dependencies> - <!-- TODO: Remove for shade-plugin 3.1.2 - https://issues.apache.org/jira/browse/MSHADE-289 --> - <dependency> - <groupId>org.ow2.asm</groupId> - <artifactId>asm</artifactId> - <version>${asm.version}</version> - </dependency> - </dependencies> - </plugin> + <version>${maven-shade-plugin.version}</version> + </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-site-plugin</artifactId> @@ -285,28 +283,6 @@ </build> <profiles> <profile> - <!-- TODO: move config into pluginManagement when we don't have to support older jdks --> - <id>jdk10</id> - <activation> - <jdk>[10, )</jdk> - </activation> - <build> - <pluginManagement> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-javadoc-plugin</artifactId> - <configuration> - <quiet>true</quiet> - <!-- Avoid javadoc warning ".. not specified the version of HTML .." --> - <additionalJOption>-html4</additionalJOption> - </configuration> - </plugin> - </plugins> - </pluginManagement> - </build> - </profile> - <profile> <id>attach-sources</id> <activation> <property> @@ -456,6 +432,11 @@ <version>1.3.1</version> </dependency> <dependency> + <groupId>commons-codec</groupId> + <artifactId>commons-codec</artifactId> + <version>1.4</version> + </dependency> + <dependency> <groupId>commons-collections</groupId> <artifactId>commons-collections</artifactId> <version>3.2.1</version> @@ -531,6 +512,16 @@ </dependency> <dependency> <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + <version>${apache.httpclient.version}</version> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpcore</artifactId> + <version>${apache.httpcore.version}</version> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> <artifactId>httpmime</artifactId> <version>4.3.6</version> </dependency> @@ -552,12 +543,12 @@ <dependency> <groupId>org.apache.maven.plugin-tools</groupId> <artifactId>maven-plugin-annotations</artifactId> - <version>3.5</version> + <version>${maven-plugin-tools.version}</version> </dependency> <dependency> <groupId>org.apache.maven</groupId> <artifactId>maven-plugin-api</artifactId> - <version>3.5.0</version> + <version>${maven-plugin-tools.version}</version> </dependency> <dependency> <groupId>org.apache.maven</groupId> @@ -723,7 +714,9 @@ <properties> <antlr.version>3.5.2</antlr.version> <antlr4.version>4.5</antlr4.version> - <asm.version>6.2</asm.version> + <apache.httpclient.version>4.4.1</apache.httpclient.version> + <apache.httpcore.version>4.4.1</apache.httpcore.version> + <asm.version>7.0</asm.version> <jna.version>4.5.2</jna.version> <tensorflow.version>1.12.0</tensorflow.version> <!-- Athenz dependencies. Make sure these dependencies matches those in Vespa's internal repositories --> @@ -736,6 +729,8 @@ xargs perl -pi -e 's/major = [0-9]+, minor = [0-9]+, micro = [0-9]+/major = 2, minor = 9, micro = 1/g' --> <curator.version>2.9.1</curator.version> + <maven-shade-plugin.version>3.2.1</maven-shade-plugin.version> + <maven-plugin-tools.version>3.6.0</maven-plugin-tools.version> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <test.hide>true</test.hide> diff --git a/persistence/src/vespa/persistence/spi/CMakeLists.txt b/persistence/src/vespa/persistence/spi/CMakeLists.txt index f9146c5a833..771da9a7453 100644 --- a/persistence/src/vespa/persistence/spi/CMakeLists.txt +++ b/persistence/src/vespa/persistence/spi/CMakeLists.txt @@ -8,7 +8,6 @@ vespa_add_library(persistence_spi OBJECT context.cpp docentry.cpp exceptions.cpp - metricpersistenceprovider.cpp partitionstate.cpp persistenceprovider.cpp read_consistency.cpp diff --git a/persistence/src/vespa/persistence/spi/metricpersistenceprovider.cpp b/persistence/src/vespa/persistence/spi/metricpersistenceprovider.cpp deleted file mode 100644 index d62285da6d3..00000000000 --- a/persistence/src/vespa/persistence/spi/metricpersistenceprovider.cpp +++ /dev/null @@ -1,301 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "metricpersistenceprovider.h" -#include <vespa/metrics/valuemetric.h> -#include <vespa/metrics/metrictimer.h> -#include <cassert> - -#include <vespa/log/log.h> -LOG_SETUP(".persistence.spi.metrics"); - -#define PRE_PROCESS(opIndex) \ - metrics::MetricTimer metricTimer; - -#define POST_PROCESS(opIndex, result) \ - metricTimer.stop( \ - *_functionMetrics[opIndex]->_metric[result.getErrorCode()]); \ - if (result.hasError()) { \ - LOG(debug, "SPI::%s failed: %s", \ - _functionMetrics[opIndex]->getName().c_str(), \ - result.toString().c_str()); \ - } - -namespace storage { -namespace spi { - -namespace { - typedef MetricPersistenceProvider Impl; -} - -using metrics::DoubleAverageMetric; -using std::make_unique; - -Impl::ResultMetrics::~ResultMetrics() { } - -Impl::ResultMetrics::ResultMetrics(const char* opName) - : metrics::MetricSet(opName, {}, ""), - _metric(Result::ERROR_COUNT) -{ - metrics::Metric::Tags noTags; - _metric[Result::NONE] = make_unique<DoubleAverageMetric>("success", noTags, "", this); - _metric[Result::TRANSIENT_ERROR] = make_unique<DoubleAverageMetric>("transient_error", noTags, "", this); - _metric[Result::PERMANENT_ERROR] = make_unique<DoubleAverageMetric>("permanent_error", noTags, "", this); - _metric[Result::TIMESTAMP_EXISTS] = make_unique<DoubleAverageMetric>("timestamp_exists", noTags, "", this); - _metric[Result::FATAL_ERROR] = make_unique<DoubleAverageMetric>("fatal_error", noTags, "", this); - _metric[Result::RESOURCE_EXHAUSTED] = make_unique<DoubleAverageMetric>("resource_exhausted", noTags, "", this); - // Assert that the above initialized all entries in vector - for (size_t i=0; i<_metric.size(); ++i) assert(_metric[i].get()); -} - -Impl::MetricPersistenceProvider(PersistenceProvider& next) - : metrics::MetricSet("spi", {}, ""), - _next(&next), - _functionMetrics(23) -{ - defineResultMetrics(0, "initialize"); - defineResultMetrics(1, "getPartitionStates"); - defineResultMetrics(2, "listBuckets"); - defineResultMetrics(3, "setClusterState"); - defineResultMetrics(4, "setActiveState"); - defineResultMetrics(5, "getBucketInfo"); - defineResultMetrics(6, "put"); - defineResultMetrics(7, "remove"); - defineResultMetrics(8, "removeIfFound"); - defineResultMetrics(9, "removeEntry"); - defineResultMetrics(10, "update"); - defineResultMetrics(11, "flush"); - defineResultMetrics(12, "get"); - defineResultMetrics(13, "createIterator"); - defineResultMetrics(14, "iterate"); - defineResultMetrics(15, "destroyIterator"); - defineResultMetrics(16, "createBucket"); - defineResultMetrics(17, "deleteBucket"); - defineResultMetrics(18, "getModifiedBuckets"); - defineResultMetrics(19, "maintain"); - defineResultMetrics(20, "split"); - defineResultMetrics(21, "join"); - defineResultMetrics(22, "move"); -} - -Impl::~MetricPersistenceProvider() { } - -void -Impl::defineResultMetrics(int index, const char* name) -{ - _functionMetrics[index] = make_unique<ResultMetrics>(name); - registerMetric(*_functionMetrics[index]); -} - -// Implementation of SPI functions - -Result -Impl::initialize() -{ - PRE_PROCESS(0); - Result r(_next->initialize()); - POST_PROCESS(0, r); - return r; -} - -PartitionStateListResult -Impl::getPartitionStates() const -{ - PRE_PROCESS(1); - PartitionStateListResult r(_next->getPartitionStates()); - POST_PROCESS(1, r); - return r; -} - -BucketIdListResult -Impl::listBuckets(BucketSpace bucketSpace, PartitionId v1) const -{ - PRE_PROCESS(2); - BucketIdListResult r(_next->listBuckets(bucketSpace, v1)); - POST_PROCESS(2, r); - return r; -} - -Result -Impl::setClusterState(BucketSpace bucketSpace, const ClusterState& v1) -{ - PRE_PROCESS(3); - Result r(_next->setClusterState(bucketSpace, v1)); - POST_PROCESS(3, r); - return r; -} - -Result -Impl::setActiveState(const Bucket& v1, BucketInfo::ActiveState v2) -{ - PRE_PROCESS(4); - Result r(_next->setActiveState(v1, v2)); - POST_PROCESS(4, r); - return r; -} - -BucketInfoResult -Impl::getBucketInfo(const Bucket& v1) const -{ - PRE_PROCESS(5); - BucketInfoResult r(_next->getBucketInfo(v1)); - POST_PROCESS(5, r); - return r; -} - -Result -Impl::put(const Bucket& v1, Timestamp v2, const DocumentSP& v3, Context& v4) -{ - PRE_PROCESS(6); - Result r(_next->put(v1, v2, v3, v4)); - POST_PROCESS(6, r); - return r; -} - -RemoveResult -Impl::remove(const Bucket& v1, Timestamp v2, const DocumentId& v3, Context& v4) -{ - PRE_PROCESS(7); - RemoveResult r(_next->remove(v1, v2, v3, v4)); - POST_PROCESS(7, r); - return r; -} - -RemoveResult -Impl::removeIfFound(const Bucket& v1, Timestamp v2, const DocumentId& v3, - Context& v4) -{ - PRE_PROCESS(8); - RemoveResult r(_next->removeIfFound(v1, v2, v3, v4)); - POST_PROCESS(8, r); - return r; -} - -Result -Impl::removeEntry(const Bucket& v1, Timestamp v2, Context& v3) -{ - PRE_PROCESS(9); - Result r(_next->removeEntry(v1, v2, v3)); - POST_PROCESS(9, r); - return r; -} - -UpdateResult -Impl::update(const Bucket& v1, Timestamp v2, const DocumentUpdateSP& v3, Context& v4) -{ - PRE_PROCESS(10); - UpdateResult r(_next->update(v1, v2, v3, v4)); - POST_PROCESS(10, r); - return r; -} - -Result -Impl::flush(const Bucket& v1, Context& v2) -{ - PRE_PROCESS(11); - Result r(_next->flush(v1, v2)); - POST_PROCESS(11, r); - return r; -} - -GetResult -Impl::get(const Bucket& v1, const document::FieldSet& v2, const DocumentId& v3, Context& v4) const -{ - PRE_PROCESS(12); - GetResult r(_next->get(v1, v2, v3, v4)); - POST_PROCESS(12, r); - return r; -} - -CreateIteratorResult -Impl::createIterator(const Bucket& v1, const document::FieldSet& v2, - const Selection& v3, IncludedVersions v4, Context& v5) -{ - PRE_PROCESS(13); - CreateIteratorResult r(_next->createIterator(v1, v2, v3, v4, v5)); - POST_PROCESS(13, r); - return r; -} - -IterateResult -Impl::iterate(IteratorId v1, uint64_t v2, Context& v3) const -{ - PRE_PROCESS(14); - IterateResult r(_next->iterate(v1, v2, v3)); - POST_PROCESS(14, r); - return r; -} - -Result -Impl::destroyIterator(IteratorId v1, Context& v2) -{ - PRE_PROCESS(15); - Result r(_next->destroyIterator(v1, v2)); - POST_PROCESS(15, r); - return r; -} - -Result -Impl::createBucket(const Bucket& v1, Context& v2) -{ - PRE_PROCESS(16); - Result r(_next->createBucket(v1, v2)); - POST_PROCESS(16, r); - return r; -} - -Result -Impl::deleteBucket(const Bucket& v1, Context& v2) -{ - PRE_PROCESS(17); - Result r(_next->deleteBucket(v1, v2)); - POST_PROCESS(17, r); - return r; -} - -BucketIdListResult -Impl::getModifiedBuckets(BucketSpace bucketSpace) const -{ - PRE_PROCESS(18); - BucketIdListResult r(_next->getModifiedBuckets(bucketSpace)); - POST_PROCESS(18, r); - return r; -} - -Result -Impl::maintain(const Bucket& v1, MaintenanceLevel v2) -{ - PRE_PROCESS(19); - Result r(_next->maintain(v1, v2)); - POST_PROCESS(19, r); - return r; -} - -Result -Impl::split(const Bucket& v1, const Bucket& v2, const Bucket& v3, Context& v4) -{ - PRE_PROCESS(20); - Result r(_next->split(v1, v2, v3, v4)); - POST_PROCESS(20, r); - return r; -} - -Result -Impl::join(const Bucket& v1, const Bucket& v2, const Bucket& v3, Context& v4) -{ - PRE_PROCESS(21); - Result r(_next->join(v1, v2, v3, v4)); - POST_PROCESS(21, r); - return r; -} - -Result -Impl::move(const Bucket& v1, PartitionId v2, Context& v3) -{ - PRE_PROCESS(22); - Result r(_next->move(v1, v2, v3)); - POST_PROCESS(22, r); - return r; -} - -} // spi -} // storage diff --git a/persistence/src/vespa/persistence/spi/metricpersistenceprovider.h b/persistence/src/vespa/persistence/spi/metricpersistenceprovider.h deleted file mode 100644 index b804fd21550..00000000000 --- a/persistence/src/vespa/persistence/spi/metricpersistenceprovider.h +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * SPI implementation wrapper to add metrics. - */ - -#pragma once - -#include "persistenceprovider.h" -#include <vespa/metrics/metricset.h> -#include <vespa/metrics/valuemetric.h> - -namespace storage::spi { - -class MetricPersistenceProvider : public PersistenceProvider, - public metrics::MetricSet -{ - struct ResultMetrics : public metrics::MetricSet { - std::vector<std::unique_ptr<metrics::DoubleAverageMetric> > _metric; - - ResultMetrics(const char* opName); - ~ResultMetrics(); - }; - PersistenceProvider* _next; - std::vector<std::unique_ptr<ResultMetrics>> _functionMetrics; - -public: - typedef std::unique_ptr<MetricPersistenceProvider> UP; - - MetricPersistenceProvider(PersistenceProvider&); - ~MetricPersistenceProvider(); - - void setNextProvider(PersistenceProvider& p) { _next = &p; } - - // Implementation of the PersistenceProvider API - Result initialize() override; - PartitionStateListResult getPartitionStates() const override; - BucketIdListResult listBuckets(BucketSpace bucketSpace, PartitionId) const override; - Result setClusterState(BucketSpace bucketSpace, const ClusterState&) override; - Result setActiveState(const Bucket&, BucketInfo::ActiveState) override; - BucketInfoResult getBucketInfo(const Bucket&) const override; - Result put(const Bucket&, Timestamp, const DocumentSP&, Context&) override; - RemoveResult remove(const Bucket&, Timestamp, const DocumentId&, Context&) override; - RemoveResult removeIfFound(const Bucket&, Timestamp, const DocumentId&, Context&) override; - Result removeEntry(const Bucket&, Timestamp, Context&) override; - UpdateResult update(const Bucket&, Timestamp, const DocumentUpdateSP&, Context&) override; - Result flush(const Bucket&, Context&) override; - GetResult get(const Bucket&, const document::FieldSet&, const DocumentId&, Context&) const override; - CreateIteratorResult createIterator(const Bucket&, const document::FieldSet&, const Selection&, - IncludedVersions, Context&) override; - IterateResult iterate(IteratorId, uint64_t maxByteSize, Context&) const override; - Result destroyIterator(IteratorId, Context&) override; - Result createBucket(const Bucket&, Context&) override; - Result deleteBucket(const Bucket&, Context&) override; - BucketIdListResult getModifiedBuckets(BucketSpace bucketSpace) const override; - Result maintain(const Bucket&, MaintenanceLevel level) override; - Result split(const Bucket& source, const Bucket& target1, const Bucket& target2, Context&) override; - Result join(const Bucket& source1, const Bucket& source2, const Bucket& target, Context&) override; - Result move(const Bucket&, PartitionId target, Context&) override; - -private: - void defineResultMetrics(int index, const char* name); -}; - -} - @@ -5,7 +5,7 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>vespa</artifactId> <packaging>pom</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <description>Aggregator pom for vespa.</description> <url>https://github.com/vespa-engine</url> diff --git a/predicate-search-core/pom.xml b/predicate-search-core/pom.xml index 8ced58b840e..0707c7ef064 100644 --- a/predicate-search-core/pom.xml +++ b/predicate-search-core/pom.xml @@ -6,11 +6,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>predicate-search-core</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/predicate-search/pom.xml b/predicate-search/pom.xml index 289250012a0..fa1bee2fe28 100644 --- a/predicate-search/pom.xml +++ b/predicate-search/pom.xml @@ -6,11 +6,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>predicate-search</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/processing/pom.xml b/processing/pom.xml index 309e6e6fec7..3194c0642db 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>processing</artifactId> <packaging>jar</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>junit</groupId> diff --git a/provided-dependencies/pom.xml b/provided-dependencies/pom.xml index 9c2cc236c2a..7318e992fc7 100755 --- a/provided-dependencies/pom.xml +++ b/provided-dependencies/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>provided-dependencies</artifactId> <packaging>jar</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <description>Dependencies that are installed and should not be included in artifacts with dependencies.</description> <dependencies> <!-- jdisc --> diff --git a/searchcommon/src/tests/schema/load-save-cfg/indexschema.cfg b/searchcommon/src/tests/schema/load-save-cfg/indexschema.cfg index 989f30f7499..c0998bcf597 100644 --- a/searchcommon/src/tests/schema/load-save-cfg/indexschema.cfg +++ b/searchcommon/src/tests/schema/load-save-cfg/indexschema.cfg @@ -8,17 +8,6 @@ indexfield[2].datatype STRING indexfield[2].prefix true indexfield[2].phrases false indexfield[2].positions false -indexfield[3].name e -indexfield[3].datatype BOOLEANTREE -indexfield[3].collectiontype SINGLE -indexfield[4].name f -indexfield[4].indextype RISE -indexfield[4].datatype STRING -indexfield[4].collectiontype WEIGHTEDSET -indexfield[5].name g -indexfield[5].indextype RISE -indexfield[5].datatype INT64 -indexfield[5].collectiontype WEIGHTEDSET fieldset[1] fieldset[0].name default fieldset[0].field[2] diff --git a/searchcommon/src/vespa/searchcommon/common/schemaconfigurer.cpp b/searchcommon/src/vespa/searchcommon/common/schemaconfigurer.cpp index 401003cb74b..d56f3c747c1 100644 --- a/searchcommon/src/vespa/searchcommon/common/schemaconfigurer.cpp +++ b/searchcommon/src/vespa/searchcommon/common/schemaconfigurer.cpp @@ -32,8 +32,6 @@ convertIndexDataType(const IndexschemaConfig::Indexfield::Datatype &type) return DataType::STRING; case IndexschemaConfig::Indexfield::INT64: return DataType::INT64; - case IndexschemaConfig::Indexfield::BOOLEANTREE: - return DataType::BOOLEANTREE; } return DataType::STRING; } @@ -144,20 +142,12 @@ SchemaBuilder::build(const IndexschemaConfig &cfg, Schema &schema) { for (size_t i = 0; i < cfg.indexfield.size(); ++i) { const IndexschemaConfig::Indexfield & f = cfg.indexfield[i]; - if ((f.datatype == IndexschemaConfig::Indexfield::BOOLEANTREE && - f.collectiontype == IndexschemaConfig::Indexfield::SINGLE) || - (f.indextype == IndexschemaConfig::Indexfield::RISE)) - { - LOG(warning, "Your field '%s' is a rise index. Those are no longer supported as of Vespa-5.89.\n" - " Redeploy and follow instructions to mitigate.", f.name.c_str()); - } else { - schema.addIndexField(Schema::IndexField(f.name, convertIndexDataType(f.datatype), - convertIndexCollectionType(f.collectiontype)). - setPrefix(f.prefix). - setPhrases(f.phrases). - setPositions(f.positions). - setAvgElemLen(f.averageelementlen)); - } + schema.addIndexField(Schema::IndexField(f.name, convertIndexDataType(f.datatype), + convertIndexCollectionType(f.collectiontype)). + setPrefix(f.prefix). + setPhrases(f.phrases). + setPositions(f.positions). + setAvgElemLen(f.averageelementlen)); } for (size_t i = 0; i < cfg.fieldset.size(); ++i) { const IndexschemaConfig::Fieldset &fs = cfg.fieldset[i]; diff --git a/searchcommon/src/vespa/searchcommon/common/undefinedvalues.h b/searchcommon/src/vespa/searchcommon/common/undefinedvalues.h index 478cc680ea0..3185cad34bd 100644 --- a/searchcommon/src/vespa/searchcommon/common/undefinedvalues.h +++ b/searchcommon/src/vespa/searchcommon/common/undefinedvalues.h @@ -11,17 +11,17 @@ namespace attribute { // for all integers template <typename T> -T getUndefined() { +constexpr T getUndefined() { return std::numeric_limits<T>::min(); } template <> -inline float getUndefined<float>() { +inline constexpr float getUndefined<float>() { return -std::numeric_limits<float>::quiet_NaN(); } template <> -inline double getUndefined<double>() { +inline constexpr double getUndefined<double>() { return -std::numeric_limits<double>::quiet_NaN(); } diff --git a/searchcore/pom.xml b/searchcore/pom.xml index 448209b1fd6..1254e9db39a 100644 --- a/searchcore/pom.xml +++ b/searchcore/pom.xml @@ -8,12 +8,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>searchcore</artifactId> <packaging>jar</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>${project.artifactId}</name> <dependencies> <dependency> diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index c0c706383f6..40c2733d230 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -1108,13 +1108,13 @@ Test::requireThatPositionsAreUsed() EXPECT_EQUAL(1u, rep->docsums[0].docid); EXPECT_EQUAL(gid1, rep->docsums[0].gid); EXPECT_TRUE(assertSlime("{sp2:'1047758'" - ",sp2x:'<position x=\"1002\" y=\"1003\" latlong=\"N0.001003;E0.001002\" />'" + ",sp2x:{x:1002, y:1003, latlong:'N0.001003;E0.001002'}" ",ap2:[1047806,1048322]" - ",ap2x:'<position x=\"1006\" y=\"1007\" latlong=\"N0.001007;E0.001006\" />" - "<position x=\"1008\" y=\"1009\" latlong=\"N0.001009;E0.001008\" />'" + ",ap2x:[{x:1006, y:1007, latlong:'N0.001007;E0.001006'}," + "{x:1008, y:1009, latlong:'N0.001009;E0.001008'}]" ",wp2:[{item:1048370,weight:43},{item:1048382,weight:44}]" - ",wp2x:'<position x=\"1012\" y=\"1013\" latlong=\"N0.001013;E0.001012\" />" - "<position x=\"1014\" y=\"1015\" latlong=\"N0.001015;E0.001014\" />'}", + ",wp2x:[{ x:1012, y:1013, latlong:'N0.001013;E0.001012'}," + "{ x:1014, y:1015, latlong:'N0.001015;E0.001014'}]}", *rep, 0, false)); } diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index 45c9ae71bc9..32cf0584768 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -7,9 +7,7 @@ #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/initializer/task_runner.h> #include <vespa/searchcore/proton/metrics/attribute_metrics.h> -#include <vespa/searchcore/proton/metrics/attribute_metrics_collection.h> -#include <vespa/searchcore/proton/metrics/documentdb_metrics_collection.h> -#include <vespa/searchcore/proton/metrics/legacy_attribute_metrics.h> +#include <vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h> #include <vespa/searchcore/proton/metrics/metricswireservice.h> #include <vespa/searchcore/proton/reference/i_document_db_reference_resolver.h> #include <vespa/searchcore/proton/reprocessing/i_reprocessing_task.h> @@ -108,7 +106,7 @@ struct MyMetricsWireService : public DummyWireService { std::set<vespalib::string> _attributes; MyMetricsWireService() : _attributes() {} - virtual void addAttribute(const AttributeMetricsCollection &, LegacyAttributeMetrics *, const std::string &name) override { + virtual void addAttribute(AttributeMetrics &, const std::string &name) override { _attributes.insert(name); } }; @@ -142,7 +140,7 @@ struct MyStoreOnlyContext MySyncProxy _syncProxy; MyGetSerialNum _getSerialNum; MyFileHeaderContext _fileHeader; - DocumentDBMetricsCollection _metrics; + DocumentDBTaggedMetrics _metrics; std::mutex _configMutex; HwInfo _hwInfo; StoreOnlyContext _ctx; @@ -182,8 +180,6 @@ struct MyFastAccessContext { MyStoreOnlyContext _storeOnlyCtx; AttributeMetrics _attributeMetrics; - LegacyAttributeMetrics _legacyAttributeMetrics; - AttributeMetricsCollection _attributeMetricsCollection; MyMetricsWireService _wireService; FastAccessContext _ctx; MyFastAccessContext(IThreadingService &writeService, @@ -203,10 +199,9 @@ MyFastAccessContext::MyFastAccessContext(IThreadingService &writeService, Thread std::shared_ptr<BucketDBOwner> bucketDB, IBucketDBHandlerInitializer & bucketDBHandlerInitializer) : _storeOnlyCtx(writeService, summaryExecutor, bucketDB, bucketDBHandlerInitializer), - _attributeMetrics(NULL), _legacyAttributeMetrics(NULL), - _attributeMetricsCollection(_attributeMetrics, _legacyAttributeMetrics), + _attributeMetrics(NULL), _wireService(), - _ctx(_storeOnlyCtx._ctx, _attributeMetricsCollection, NULL, _wireService) + _ctx(_storeOnlyCtx._ctx, _attributeMetrics, _wireService) {} MyFastAccessContext::~MyFastAccessContext() = default; diff --git a/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp b/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp index e10fd41ffed..c798de01804 100644 --- a/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp +++ b/searchcore/src/tests/proton/metrics/metrics_engine/metrics_engine_test.cpp @@ -1,12 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Unit tests for metrics_engine. #include <vespa/metrics/metricset.h> -#include <vespa/searchcore/proton/metrics/attribute_metrics_collection.h> #include <vespa/searchcore/proton/metrics/attribute_metrics.h> -#include <vespa/searchcore/proton/metrics/documentdb_metrics_collection.h> - - #include <vespa/searchcore/proton/metrics/metrics_engine.h> #include <vespa/vespalib/testkit/testapp.h> @@ -23,56 +18,31 @@ struct AttributeMetricsFixture { MetricsEngine engine; DummyMetricSet parent; AttributeMetrics metrics; - LegacyAttributeMetrics legacyMetrics; - LegacyAttributeMetrics totalLegacyMetrics; AttributeMetricsFixture() : engine(), parent("parent"), - metrics(&parent), - legacyMetrics(nullptr), - totalLegacyMetrics(nullptr) + metrics(&parent) {} void addAttribute(const vespalib::string &attrName) { - engine.addAttribute(AttributeMetricsCollection(metrics, legacyMetrics), &totalLegacyMetrics, attrName); + engine.addAttribute(metrics, attrName); } void removeAttribute(const vespalib::string &attrName) { - engine.removeAttribute(AttributeMetricsCollection(metrics, legacyMetrics), &totalLegacyMetrics, attrName); + engine.removeAttribute(metrics, attrName); } void cleanAttributes() { - engine.cleanAttributes(AttributeMetricsCollection(metrics, legacyMetrics), &totalLegacyMetrics); + engine.cleanAttributes(metrics); } void assertRegisteredMetrics(size_t expNumMetrics) const { EXPECT_EQUAL(expNumMetrics, parent.getRegisteredMetrics().size()); - EXPECT_EQUAL(expNumMetrics, legacyMetrics.list.getRegisteredMetrics().size()); - EXPECT_EQUAL(expNumMetrics, totalLegacyMetrics.list.getRegisteredMetrics().size()); } void assertMetricsExists(const vespalib::string &attrName) { EXPECT_TRUE(metrics.get(attrName) != nullptr); - EXPECT_TRUE(legacyMetrics.list.get(attrName) != nullptr); - EXPECT_TRUE(totalLegacyMetrics.list.get(attrName) != nullptr); } void assertMetricsNotExists(const vespalib::string &attrName) { EXPECT_TRUE(metrics.get(attrName) == nullptr); - EXPECT_TRUE(legacyMetrics.list.get(attrName) == nullptr); - EXPECT_TRUE(totalLegacyMetrics.list.get(attrName) == nullptr); } }; -TEST("require that the metric proton.diskusage is the sum of the documentDB diskusage metrics") -{ - MetricsEngine metrics_engine; - - DocumentDBMetricsCollection metrics1("type1", 1); - DocumentDBMetricsCollection metrics2("type2", 1); - metrics1.getLegacyMetrics().index.diskUsage.addValue(100); - metrics2.getLegacyMetrics().index.diskUsage.addValue(1000); - - metrics_engine.addDocumentDBMetrics(metrics1); - metrics_engine.addDocumentDBMetrics(metrics2); - - EXPECT_EQUAL(1100, metrics_engine.legacyRoot().diskUsage.getLongValue("value")); -} - TEST_F("require that attribute metrics can be added", AttributeMetricsFixture) { TEST_DO(f.assertRegisteredMetrics(0)); diff --git a/searchcore/src/tests/proton/server/CMakeLists.txt b/searchcore/src/tests/proton/server/CMakeLists.txt index 31f67bdd0ac..51595daf407 100644 --- a/searchcore/src/tests/proton/server/CMakeLists.txt +++ b/searchcore/src/tests/proton/server/CMakeLists.txt @@ -1,12 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_legacy_attribute_metrics_test_app TEST - SOURCES - legacy_attribute_metrics_test.cpp - DEPENDS - searchcore_server - searchcore_proton_metrics -) -vespa_add_test(NAME searchcore_legacy_attribute_metrics_test_app COMMAND searchcore_legacy_attribute_metrics_test_app) vespa_add_executable(searchcore_documentretriever_test_app TEST SOURCES documentretriever_test.cpp diff --git a/searchcore/src/tests/proton/server/legacy_attribute_metrics_test.cpp b/searchcore/src/tests/proton/server/legacy_attribute_metrics_test.cpp deleted file mode 100644 index bf68deafb96..00000000000 --- a/searchcore/src/tests/proton/server/legacy_attribute_metrics_test.cpp +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("attribute_metrics_test"); -#include <vespa/vespalib/testkit/testapp.h> - -#include <vespa/searchcore/proton/metrics/legacy_attribute_metrics.h> - -using namespace proton; - -class Test : public vespalib::TestApp -{ -public: - int Main() override; -}; - -int -Test::Main() -{ - TEST_INIT("attribute_metrics_test"); - { - LegacyAttributeMetrics attrMetrics(0); - EXPECT_EQUAL(0u, attrMetrics.list.release().size()); - { - LegacyAttributeMetrics::List::Entry *e1 = attrMetrics.list.add("foo"); - LegacyAttributeMetrics::List::Entry *e2 = attrMetrics.list.add("bar"); - LegacyAttributeMetrics::List::Entry *e3 = attrMetrics.list.add("foo"); - EXPECT_TRUE(e1 != nullptr); - EXPECT_TRUE(e2 != nullptr); - EXPECT_TRUE(e3 == nullptr); - } - { - const LegacyAttributeMetrics &constMetrics = attrMetrics; - LegacyAttributeMetrics::List::Entry *e1 = constMetrics.list.get("foo"); - LegacyAttributeMetrics::List::Entry *e2 = constMetrics.list.get("bar"); - LegacyAttributeMetrics::List::Entry *e3 = constMetrics.list.get("baz"); - EXPECT_TRUE(e1 != nullptr); - EXPECT_TRUE(e2 != nullptr); - EXPECT_TRUE(e3 == nullptr); - } - EXPECT_EQUAL(2u, attrMetrics.list.release().size()); - { - const LegacyAttributeMetrics &constMetrics = attrMetrics; - LegacyAttributeMetrics::List::Entry *e1 = constMetrics.list.get("foo"); - LegacyAttributeMetrics::List::Entry *e2 = constMetrics.list.get("bar"); - LegacyAttributeMetrics::List::Entry *e3 = constMetrics.list.get("baz"); - EXPECT_TRUE(e1 == nullptr); - EXPECT_TRUE(e2 == nullptr); - EXPECT_TRUE(e3 == nullptr); - } - EXPECT_EQUAL(0u, attrMetrics.list.release().size()); - } - TEST_DONE(); -} - -TEST_APPHOOK(Test); diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index 2a673c2379e..1852266b321 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -66,10 +66,6 @@ flush.memory.each.diskbloatfactor double default=0.2 ## Unit is seconds with 1 day being the default. flush.memory.maxage.time double default=86400.0 -## Max diff in serial number allowed before that takes precedence. -## TODO Deprecated and ignored. Remove soon. -flush.memory.maxage.serial long default=1000000 - ## When resource limit for memory is reached we choose a conservative mode for the flush strategy. ## In this case this factor is multiplied with 'maxmemory' and 'each.maxmemory' to calculate conservative values to use instead. flush.memory.conservative.memorylimitfactor double default=0.5 @@ -160,10 +156,6 @@ index.cache.size long default=0 restart ## Control io options during flushing of attributes. attribute.write.io enum {NORMAL, OSYNC, DIRECTIO} default=DIRECTIO restart -## Control options for io during search. -## Dictionary is always MMAP. -search.io enum {NORMAL, DIRECTIO, MMAP } default=MMAP restart - ## Multiple optional options for use with mmap search.mmap.options[] enum {MLOCK, POPULATE, HUGETLB} restart @@ -255,24 +247,12 @@ summary.log.chunk.compression.level int default=9 ## Max size in bytes per chunk. summary.log.chunk.maxbytes int default=65536 -## Max number of documents in each chunk. -## TODO Deprecated and ignored. Remove soon. -summary.log.chunk.maxentries int default=256 - ## Skip crc32 check on read. summary.log.chunk.skipcrconread bool default=false -## Control how compaction is done, write to the front or to new const file. -## TODO: Remove, will always be false -summary.log.compact2activefile bool default=false - ## Max size per summary file. summary.log.maxfilesize long default=1000000000 -## Max number of removes per summary file. -## TODO Deprecated and ignored. Remove soon. -summary.log.maxentriesperfile long default=20000000 - ## Max disk bloat factor. This will trigger compacting. summary.log.maxdiskbloatfactor double default=0.1 @@ -283,11 +263,6 @@ summary.log.maxbucketspread double default=2.5 ## Value in the range [0.0, 1.0] summary.log.minfilesizefactor double default=0.2 -## Number of threads used for compressing incoming documents/compacting. -## Deprecated. Use feeding.concurrency instead. -## TODO Remove -summary.log.numthreads int default=8 restart - ## Control io options during flush of stored documents. summary.write.io enum {NORMAL, OSYNC, DIRECTIO} default=DIRECTIO @@ -302,10 +277,6 @@ summary.read.mmap.options[] enum {MLOCK, POPULATE, HUGETLB} restart ## Advise to give to os when mapping memory. summary.read.mmap.advise enum {NORMAL, RANDOM, SEQUENTIAL} default=NORMAL restart -## Enable compact for bucket oriented access. -## TODO: Unused, always bucket order. -summary.compact2buckets bool default=true restart - ## The name of the input document type documentdb[].inputdoctypename string ## The configid used to subscribe to config for this database. @@ -343,14 +314,6 @@ pruneremoveddocumentsinterval double default=0.0 ## Default value is 2 weeks (1209600 seconds). pruneremoveddocumentsage double default=1209600.0 -## NOT USED. TODO: Remove when going to Vespa 7. -## Interval between wiping of old removed document fields (in seconds). -wipeoldremovedfieldsinterval double default=21600.0 - -## NOT USED. TODO: Remove when going to Vespa 7. -## Age of removed document fields before they can be wiped (in seconds). -wipeoldremovedfieldsage double default=1209600.0 - ## Timeout of recovery rpc calls in rowcolumn mode ## ## Default value is 60 seconds diff --git a/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt index 6e01bae1866..6077a6ddd87 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt @@ -4,7 +4,6 @@ vespa_add_library(searchcore_proton_metrics STATIC attribute_metrics.cpp content_proton_metrics.cpp documentdb_job_trackers.cpp - documentdb_metrics_collection.cpp documentdb_tagged_metrics.cpp executor_metrics.cpp executor_threading_service_metrics.cpp @@ -13,10 +12,6 @@ vespa_add_library(searchcore_proton_metrics STATIC job_tracker.cpp job_tracked_flush_target.cpp job_tracked_flush_task.cpp - legacy_attribute_metrics.cpp - legacy_documentdb_metrics.cpp - legacy_proton_metrics.cpp - legacy_sessionmanager_metrics.cpp memory_usage_metrics.cpp metrics_engine.cpp resource_usage_metrics.cpp diff --git a/searchcore/src/vespa/searchcore/proton/metrics/attribute_metrics_collection.h b/searchcore/src/vespa/searchcore/proton/metrics/attribute_metrics_collection.h deleted file mode 100644 index 52a49688b64..00000000000 --- a/searchcore/src/vespa/searchcore/proton/metrics/attribute_metrics_collection.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -namespace proton { - -class AttributeMetrics; -class LegacyAttributeMetrics; - -/** - * A collection of references to all the metrics for a set of attributes. - */ -class AttributeMetricsCollection -{ -private: - AttributeMetrics &_metrics; - LegacyAttributeMetrics &_legacyMetrics; - -public: - AttributeMetricsCollection(AttributeMetrics &metrics, - LegacyAttributeMetrics &legacyMetrics) - : _metrics(metrics), - _legacyMetrics(legacyMetrics) - { - } - AttributeMetrics &getMetrics() const { return _metrics; } - LegacyAttributeMetrics &getLegacyMetrics() const { return _legacyMetrics; } -}; - -} // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_metrics_collection.cpp b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_metrics_collection.cpp deleted file mode 100644 index 80a1508614b..00000000000 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_metrics_collection.cpp +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "documentdb_metrics_collection.h" - -namespace proton { - -DocumentDBMetricsCollection::DocumentDBMetricsCollection(const vespalib::string &docTypeName, size_t maxNumThreads) - : _metrics(docTypeName, maxNumThreads), - _taggedMetrics(docTypeName) -{} - -DocumentDBMetricsCollection::~DocumentDBMetricsCollection() {} - -} // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_metrics_collection.h b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_metrics_collection.h deleted file mode 100644 index 8fa15dffc9d..00000000000 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_metrics_collection.h +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "documentdb_tagged_metrics.h" -#include "legacy_documentdb_metrics.h" - -namespace proton { - -/** - * A collection of all the metrics for a document db (both tagged and no-tagged). - */ -class DocumentDBMetricsCollection -{ -private: - LegacyDocumentDBMetrics _metrics; - DocumentDBTaggedMetrics _taggedMetrics; - -public: - DocumentDBMetricsCollection(const vespalib::string &docTypeName, size_t maxNumThreads); - ~DocumentDBMetricsCollection(); - LegacyDocumentDBMetrics &getLegacyMetrics() { return _metrics; } - DocumentDBTaggedMetrics &getTaggedMetrics() { return _taggedMetrics; } -}; - -} // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp index 13116f5a0ff..043d3b3314e 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp @@ -234,7 +234,7 @@ DocumentDBTaggedMetrics::DocumentsMetrics::DocumentsMetrics(metrics::MetricSet * DocumentDBTaggedMetrics::DocumentsMetrics::~DocumentsMetrics() = default; -DocumentDBTaggedMetrics::DocumentDBTaggedMetrics(const vespalib::string &docTypeName) +DocumentDBTaggedMetrics::DocumentDBTaggedMetrics(const vespalib::string &docTypeName, size_t maxNumThreads_) : MetricSet("documentdb", {{"documenttype", docTypeName}}, "Document DB metrics", nullptr), job(this), attribute(this), @@ -247,7 +247,8 @@ DocumentDBTaggedMetrics::DocumentDBTaggedMetrics(const vespalib::string &docType sessionCache(this), documents(this), totalMemoryUsage(this), - totalDiskUsage("disk_usage", {}, "The total disk usage (in bytes) for this document db", this) + totalDiskUsage("disk_usage", {}, "The total disk usage (in bytes) for this document db", this), + maxNumThreads(maxNumThreads_) { } diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h index 94a69be2c7f..bfb954ecd25 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h @@ -191,8 +191,9 @@ struct DocumentDBTaggedMetrics : metrics::MetricSet DocumentsMetrics documents; MemoryUsageMetrics totalMemoryUsage; metrics::LongValueMetric totalDiskUsage; + size_t maxNumThreads; - DocumentDBTaggedMetrics(const vespalib::string &docTypeName); + DocumentDBTaggedMetrics(const vespalib::string &docTypeName, size_t maxNumThreads_); ~DocumentDBTaggedMetrics(); }; diff --git a/searchcore/src/vespa/searchcore/proton/metrics/legacy_attribute_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/legacy_attribute_metrics.cpp deleted file mode 100644 index 7d182e5ccc5..00000000000 --- a/searchcore/src/vespa/searchcore/proton/metrics/legacy_attribute_metrics.cpp +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "legacy_attribute_metrics.h" - -namespace proton { - -LegacyAttributeMetrics::List::Entry::Entry(const std::string &name) - : metrics::MetricSet(name, {}, "Attribute vector metrics", 0), - memoryUsage("memoryusage", {}, "Memory usage", this), - bitVectors("bitvectors", {}, "Number of bitvectors", this) -{ -} - -LegacyAttributeMetrics::List::Entry * -LegacyAttributeMetrics::List::add(const std::string &name) -{ - if (metrics.find(name) != metrics.end()) { - return nullptr; - } - auto &pos = metrics[name]; - pos = std::make_unique<Entry>(name); - return pos.get(); -} - -LegacyAttributeMetrics::List::Entry * -LegacyAttributeMetrics::List::get(const std::string &name) const -{ - const auto pos = metrics.find(name); - if (pos == metrics.end()) { - return nullptr; - } - return pos->second.get(); -} - -LegacyAttributeMetrics::List::Entry::UP -LegacyAttributeMetrics::List::remove(const std::string &name) -{ - auto pos = metrics.find(name); - if (pos == metrics.end()) { - return Entry::UP(); - } - Entry::UP retval = std::move(pos->second); - metrics.erase(name); - return retval; -} - -std::vector<LegacyAttributeMetrics::List::Entry::UP> -LegacyAttributeMetrics::List::release() -{ - std::vector<Entry::UP> entries; - for (auto &pos: metrics) { - entries.push_back(std::move(pos.second)); - } - metrics.clear(); - return entries; -} - -LegacyAttributeMetrics::List::List(metrics::MetricSet *parent) - : metrics::MetricSet("list", {}, "Metrics per attribute vector", parent), - metrics() -{ -} - -LegacyAttributeMetrics::List::~List() = default; - -LegacyAttributeMetrics::LegacyAttributeMetrics(metrics::MetricSet *parent) - : metrics::MetricSet("attributes", {}, "Attribute metrics", parent), - list(this), - memoryUsage("memoryusage", {}, "Memory usage for attributes", this), - bitVectors("bitvectors", {}, "Number of bitvectors for attributes", this) -{ -} - -LegacyAttributeMetrics::~LegacyAttributeMetrics() = default; - -} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/metrics/legacy_attribute_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/legacy_attribute_metrics.h deleted file mode 100644 index 073acabf614..00000000000 --- a/searchcore/src/vespa/searchcore/proton/metrics/legacy_attribute_metrics.h +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/metrics/metricset.h> -#include <vespa/metrics/valuemetric.h> -#include <map> - -namespace proton { - -struct LegacyAttributeMetrics : metrics::MetricSet { - - // The metric set also owns the actual metrics for individual - // attribute vectors. Another way to do this would be to let the - // attribute vectors own their own metrics, but this would - // generate more dependencies and reduce locality of code changes. - - struct List : metrics::MetricSet { - struct Entry : metrics::MetricSet { - using UP = std::unique_ptr<Entry>; - metrics::LongValueMetric memoryUsage; - metrics::LongValueMetric bitVectors; - Entry(const std::string &name); - }; - Entry *add(const std::string &name); - Entry *get(const std::string &name) const; - Entry::UP remove(const std::string &name); - std::vector<Entry::UP> release(); - - // per attribute metrics will be wired in here (by the metrics engine) - List(metrics::MetricSet *parent); - ~List() override; - - private: - std::map<std::string, Entry::UP> metrics; - }; - - List list; - metrics::LongValueMetric memoryUsage; - metrics::LongValueMetric bitVectors; - - LegacyAttributeMetrics(metrics::MetricSet *parent); - ~LegacyAttributeMetrics(); -}; - -} // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/metrics/legacy_documentdb_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/legacy_documentdb_metrics.cpp deleted file mode 100644 index d4180c4ec5f..00000000000 --- a/searchcore/src/vespa/searchcore/proton/metrics/legacy_documentdb_metrics.cpp +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "legacy_documentdb_metrics.h" -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/util/exceptions.h> - -using vespalib::asciistream; -using vespalib::make_string; -using metrics::MetricSet; - -namespace proton { - -using matching::MatchingStats; - -LegacyDocumentDBMetrics::IndexMetrics::IndexMetrics(MetricSet *parent) - : MetricSet("index", {}, "Index metrics", parent), - memoryUsage("memoryusage", {}, "Memory usage for memory indexes", this), - docsInMemory("docsinmemory", {}, "Number of documents in memory", this), - diskUsage("diskusage", {}, "Disk usage for disk indexes", this) -{ } - -LegacyDocumentDBMetrics::IndexMetrics::~IndexMetrics() {} - -LegacyDocumentDBMetrics::DocstoreMetrics::DocstoreMetrics(MetricSet *parent) - : MetricSet("docstore", {}, "Document store metrics", parent), - memoryUsage("memoryusage", {}, "Memory usage for docstore", this), - cacheLookups("cachelookups", {}, "Number of lookups in summary cache", this), - cacheHitRate("cachehitrate", {}, "Rate of cache hits in summary cache", this), - cacheElements("cacheelements", {}, "Number of elements in summary cache", this), - cacheMemoryUsed("cachememoryused", {}, "Memory used by summary cache", this) -{ } - -LegacyDocumentDBMetrics::DocstoreMetrics::~DocstoreMetrics() {} - -void -LegacyDocumentDBMetrics::MatchingMetrics::update(const MatchingStats &stats) -{ - docsMatched.inc(stats.docsMatched()); - docsRanked.inc(stats.docsRanked()); - docsReRanked.inc(stats.docsReRanked()); - softDoomFactor.set(stats.softDoomFactor()); - queries.inc(stats.queries()); - queryCollateralTime.addValueBatch(stats.queryCollateralTimeAvg(), stats.queryCollateralTimeCount(), - stats.queryCollateralTimeMin(), stats.queryCollateralTimeMax()); - queryLatency.addValueBatch(stats.queryLatencyAvg(), stats.queryLatencyCount(), - stats.queryLatencyMin(), stats.queryLatencyMax()); -} - -LegacyDocumentDBMetrics::MatchingMetrics::MatchingMetrics(MetricSet *parent) - : MetricSet("matching", {}, "Matching metrics", parent), - docsMatched("docsmatched", {}, "Number of documents matched", this), - docsRanked("docsranked", {}, "Number of documents ranked (first phase)", this), - docsReRanked("docsreranked", {}, "Number of documents re-ranked (second phase)", this), - queries("queries", {}, "Number of queries executed", this), - softDoomFactor("softdoomfactor", {}, "Factor used to compute soft-timeout", this), - queryCollateralTime("querycollateraltime", {}, "Average time spent setting up and tearing down queries", this), - queryLatency("querylatency", {}, "Average latency when matching a query", this) -{ } - -LegacyDocumentDBMetrics::MatchingMetrics::~MatchingMetrics() {} - -LegacyDocumentDBMetrics::MatchingMetrics::RankProfileMetrics::RankProfileMetrics( - const std::string &name, size_t numDocIdPartitions, MetricSet *parent) - : MetricSet(name, {}, "Rank profile metrics", parent), - queries("queries", {}, "Number of queries executed", this), - limited_queries("limitedqueries", {}, "Number of queries limited in match phase", this), - matchTime("match_time", {}, "Average time for matching a query", this), - groupingTime("grouping_time", {}, "Average time spent on grouping", this), - rerankTime("rerank_time", {}, "Average time spent on 2nd phase ranking", this) -{ - for (size_t i=0; i < numDocIdPartitions; i++) { - vespalib::string s(make_string("docid_part%02ld", i)); - partitions.push_back(DocIdPartition::UP(new DocIdPartition(s, this))); - } -} - -LegacyDocumentDBMetrics::MatchingMetrics::RankProfileMetrics::~RankProfileMetrics() {} - -LegacyDocumentDBMetrics::MatchingMetrics::RankProfileMetrics::DocIdPartition::DocIdPartition(const std::string &name, MetricSet *parent) : - MetricSet(name, {}, "DocId Partition profile metrics", parent), - docsMatched("docsmatched", {}, "Number of documents matched", this), - docsRanked("docsranked", {}, "Number of documents ranked (first phase)", this), - docsReRanked("docsreranked", {}, "Number of documents re-ranked (second phase)", this), - active_time("activetime", {}, "Time spent doing actual work", this), - wait_time("waittime", {}, "Time spent waiting for other external threads and resources", this) -{ } - -LegacyDocumentDBMetrics::MatchingMetrics::RankProfileMetrics::DocIdPartition::~DocIdPartition() {} - -void -LegacyDocumentDBMetrics::MatchingMetrics::RankProfileMetrics::DocIdPartition::update(const MatchingStats::Partition &stats) -{ - docsMatched.inc(stats.docsMatched()); - docsRanked.inc(stats.docsRanked()); - docsReRanked.inc(stats.docsReRanked()); - active_time.addValueBatch(stats.active_time_avg(), stats.active_time_count(), - stats.active_time_min(), stats.active_time_max()); - wait_time.addValueBatch(stats.wait_time_avg(), stats.wait_time_count(), - stats.wait_time_min(), stats.wait_time_max()); -} - -void -LegacyDocumentDBMetrics::MatchingMetrics::RankProfileMetrics::update(const MatchingStats &stats) -{ - queries.inc(stats.queries()); - limited_queries.inc(stats.limited_queries()); - matchTime.addValueBatch(stats.matchTimeAvg(), stats.matchTimeCount(), - stats.matchTimeMin(), stats.matchTimeMax()); - groupingTime.addValueBatch(stats.groupingTimeAvg(), stats.groupingTimeCount(), - stats.groupingTimeMin(), stats.groupingTimeMax()); - rerankTime.addValueBatch(stats.rerankTimeAvg(), stats.rerankTimeCount(), - stats.rerankTimeMin(), stats.rerankTimeMax()); - if (stats.getNumPartitions() > 0) { - if (stats.getNumPartitions() <= partitions.size()) { - for (size_t i(0), m(stats.getNumPartitions()); i < m; i++) { - DocIdPartition & partition(*partitions[i]); - const MatchingStats::Partition & s(stats.getPartition(i)); - partition.update(s); - } - } else { - vespalib::string msg(make_string("Num partitions used '%ld' is larger than number of partitions '%ld' configured.", - stats.getNumPartitions(), partitions.size())); - throw vespalib::IllegalStateException(msg, VESPA_STRLOC); - } - } -} - -LegacyDocumentDBMetrics::SubDBMetrics::DocumentMetaStoreMetrics::DocumentMetaStoreMetrics(MetricSet *parent) - : MetricSet("docmetastore", {}, "Document meta store metrics", parent), - lidLimit("lidlimit", {}, "The size of the allocated lid space", this), - usedLids("usedlids", {}, "The number of lids used", this), - lowestFreeLid("lowestfreelid", {}, "The lowest free lid", this), - highestUsedLid("highestusedlid", {}, "The highest used lid", this), - lidBloatFactor("lidbloatfactor", {}, "The bloat factor of this lid space, indicating the total amount of holes in the allocated lid space " - "((lidlimit - usedlids) / lidlimit)", this), - lidFragmentationFactor("lid_fragmentation_factor", {}, - "The fragmentation factor of this lid space, indicating the amount of holes in the currently used part of the lid space " - "((highestusedlid - usedlids) / highestusedlid)", this) -{ -} - -LegacyDocumentDBMetrics::SubDBMetrics::DocumentMetaStoreMetrics::~DocumentMetaStoreMetrics() {} - -LegacyDocumentDBMetrics::SubDBMetrics::SubDBMetrics(const vespalib::string &name, MetricSet *parent) - : MetricSet(name, {}, "Sub database metrics", parent), - attributes(this), - docMetaStore(this) -{ } - -LegacyDocumentDBMetrics::SubDBMetrics::~SubDBMetrics() {} - -LegacyDocumentDBMetrics::LegacyDocumentDBMetrics(const std::string &docTypeName, size_t maxNumThreads) - : MetricSet(make_string("%s", docTypeName.c_str()), {}, "Document DB Metrics", 0), - index(this), - attributes(this), - docstore(this), - matching(this), - executor("executor", this), - indexExecutor("indexexecutor", this), - summaryExecutor("summaryexecutor", this), - sessionManager(this), - ready("ready", this), - notReady("notready", this), - removed("removed", this), - memoryUsage("memoryusage", {}, "Memory usage for this Document DB", this), - numDocs("numdocs", {}, "Number of ready/indexed documents in this Document DB (aka number of documents in the 'ready' sub db)", this), - numActiveDocs("numactivedocs", {}, "Number of active/searchable documents in this Document DB (aka number of active/searchable documents in the 'ready' sub db)", this), - numIndexedDocs("numindexeddocs", {}, "Number of ready/indexed documents in this Document DB (aka number of documents in the 'ready' sub db)", this), - numStoredDocs("numstoreddocs", {}, "Total number of documents stored in this Document DB (aka number of documents in the 'ready' and 'notready' sub dbs)", this), - numRemovedDocs("numremoveddocs", {}, "Number of removed documents in this Document DB (aka number of documents in the 'removed' sub db)", this), - numBadConfigs("numBadConfigs", {}, "Number of bad configs for this Document DB", this), - _maxNumThreads(maxNumThreads) -{ - memoryUsage.addMetricToSum(index.memoryUsage); - memoryUsage.addMetricToSum(attributes.memoryUsage); - memoryUsage.addMetricToSum(docstore.memoryUsage); -} - -LegacyDocumentDBMetrics::~LegacyDocumentDBMetrics() { } - -} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/metrics/legacy_documentdb_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/legacy_documentdb_metrics.h deleted file mode 100644 index c61d371ae0b..00000000000 --- a/searchcore/src/vespa/searchcore/proton/metrics/legacy_documentdb_metrics.h +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "executor_metrics.h" -#include "legacy_attribute_metrics.h" -#include "legacy_sessionmanager_metrics.h" -#include <vespa/metrics/summetric.h> -#include <vespa/searchcore/proton/matching/matching_stats.h> - -namespace proton { - -/** - * Metric set for all legacy metrics reported for a document db. - * - * All these metrics have the document type name as part of the metric name, - * which is not flexible for setting up default metric graph dashboards. - * - * TODO: Remove on Vespa 7 - * - * @deprecated Use DocumentDBTaggedMetrics for all new metrics. - */ -struct LegacyDocumentDBMetrics : metrics::MetricSet -{ - struct IndexMetrics : metrics::MetricSet { - metrics::LongValueMetric memoryUsage; - metrics::LongValueMetric docsInMemory; - metrics::LongValueMetric diskUsage; - - IndexMetrics(metrics::MetricSet *parent); - ~IndexMetrics(); - }; - - struct DocstoreMetrics : metrics::MetricSet { - metrics::LongValueMetric memoryUsage; - metrics::LongCountMetric cacheLookups; - metrics::LongAverageMetric cacheHitRate; - metrics::LongValueMetric cacheElements; - metrics::LongValueMetric cacheMemoryUsed; - - DocstoreMetrics(metrics::MetricSet *parent); - ~DocstoreMetrics(); - }; - - struct MatchingMetrics : metrics::MetricSet { - metrics::LongCountMetric docsMatched; - metrics::LongCountMetric docsRanked; - metrics::LongCountMetric docsReRanked; - metrics::LongCountMetric queries; - metrics::DoubleValueMetric softDoomFactor; - metrics::DoubleAverageMetric queryCollateralTime; - metrics::DoubleAverageMetric queryLatency; - - struct RankProfileMetrics : metrics::MetricSet { - struct DocIdPartition : metrics::MetricSet { - metrics::LongCountMetric docsMatched; - metrics::LongCountMetric docsRanked; - metrics::LongCountMetric docsReRanked; - metrics::DoubleAverageMetric active_time; - metrics::DoubleAverageMetric wait_time; - - using UP = std::unique_ptr<DocIdPartition>; - DocIdPartition(const std::string &name, metrics::MetricSet *parent); - ~DocIdPartition(); - void update(const matching::MatchingStats::Partition &stats); - }; - using DocIdPartitions = std::vector<DocIdPartition::UP>; - using UP = std::unique_ptr<RankProfileMetrics>; - - metrics::LongCountMetric queries; - metrics::LongCountMetric limited_queries; - metrics::DoubleAverageMetric matchTime; - metrics::DoubleAverageMetric groupingTime; - metrics::DoubleAverageMetric rerankTime; - DocIdPartitions partitions; - - RankProfileMetrics(const std::string &name, - size_t numDocIdPartitions, - metrics::MetricSet *parent); - ~RankProfileMetrics(); - void update(const matching::MatchingStats &stats); - - }; - using RankProfileMap = std::map<std::string, RankProfileMetrics::UP>; - RankProfileMap rank_profiles; - - void update(const matching::MatchingStats &stats); - MatchingMetrics(metrics::MetricSet *parent); - ~MatchingMetrics(); - }; - - struct SubDBMetrics : metrics::MetricSet - { - struct DocumentMetaStoreMetrics : metrics::MetricSet - { - metrics::LongValueMetric lidLimit; - metrics::LongValueMetric usedLids; - metrics::LongValueMetric lowestFreeLid; - metrics::LongValueMetric highestUsedLid; - metrics::DoubleValueMetric lidBloatFactor; - metrics::DoubleValueMetric lidFragmentationFactor; - - DocumentMetaStoreMetrics(metrics::MetricSet *parent); - ~DocumentMetaStoreMetrics(); - }; - - LegacyAttributeMetrics attributes; - DocumentMetaStoreMetrics docMetaStore; - SubDBMetrics(const vespalib::string &name, metrics::MetricSet *parent); - ~SubDBMetrics(); - }; - - IndexMetrics index; - LegacyAttributeMetrics attributes; - DocstoreMetrics docstore; - MatchingMetrics matching; - ExecutorMetrics executor; - ExecutorMetrics indexExecutor; - ExecutorMetrics summaryExecutor; - LegacySessionManagerMetrics sessionManager; - SubDBMetrics ready; - SubDBMetrics notReady; - SubDBMetrics removed; - metrics::SumMetric<metrics::LongValueMetric> memoryUsage; - metrics::LongValueMetric numDocs; - metrics::LongValueMetric numActiveDocs; - metrics::LongValueMetric numIndexedDocs; - metrics::LongValueMetric numStoredDocs; - metrics::LongValueMetric numRemovedDocs; - metrics::LongValueMetric numBadConfigs; - size_t _maxNumThreads; - - LegacyDocumentDBMetrics(const std::string &docTypeName, size_t maxNumThreads); - ~LegacyDocumentDBMetrics(); -}; - -} // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/metrics/legacy_proton_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/legacy_proton_metrics.cpp deleted file mode 100644 index 5c0285967e9..00000000000 --- a/searchcore/src/vespa/searchcore/proton/metrics/legacy_proton_metrics.cpp +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "legacy_proton_metrics.h" - -namespace proton { - -LegacyProtonMetrics::DocumentTypeMetrics::DocumentTypeMetrics(metrics::MetricSet *parent) - : metrics::MetricSet("doctypes", {}, "Metrics per document type", parent) -{ -} - -LegacyProtonMetrics::DocumentTypeMetrics::~DocumentTypeMetrics() { } - -LegacyProtonMetrics::LegacyProtonMetrics() - : metrics::MetricSet("proton", {}, "Search engine metrics", 0), - docTypes(this), - executor("executor", this), - flushExecutor("flushexecutor", this), - matchExecutor("matchexecutor", this), - summaryExecutor("summaryexecutor", this), - memoryUsage("memoryusage", {{"logdefault"}}, "Total tracked memory usage", this), - diskUsage("diskusage", {{"logdefault"}}, "Total tracked disk usage for disk indexes", this), - docsInMemory("docsinmemory", {{"logdefault"}}, "Total Number of documents in memory", this), - numDocs("numdocs", {{"logdefault"}}, "Total number of ready/indexed documents among all document dbs (equal as numindexeddocs)", this), - numActiveDocs("numactivedocs", {{"logdefault"}}, - "Total number of active/searchable documents among all document dbs", this), - numIndexedDocs("numindexeddocs", {{"logdefault"}}, - "Total number of ready/indexed documents among all document dbs (equal as numdocs)", this), - numStoredDocs("numstoreddocs", {{"logdefault"}}, - "Total number of stored documents among all document dbs", this), - numRemovedDocs("numremoveddocs", {{"logdefault"}}, - "Total number of removed documents among all document dbs", this) -{ - // supply start value to support sum without any document types - metrics::LongValueMetric start("start", {}, "", 0); - memoryUsage.setStartValue(start); - diskUsage.setStartValue(start); - docsInMemory.setStartValue(start); - numDocs.setStartValue(start); - numActiveDocs.setStartValue(start); - numIndexedDocs.setStartValue(start); - numStoredDocs.setStartValue(start); - numRemovedDocs.setStartValue(start); -} - -LegacyProtonMetrics::~LegacyProtonMetrics() {} - -} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/metrics/legacy_proton_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/legacy_proton_metrics.h deleted file mode 100644 index 03b73761ce5..00000000000 --- a/searchcore/src/vespa/searchcore/proton/metrics/legacy_proton_metrics.h +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/metrics/metrics.h> -#include "executor_metrics.h" - -namespace proton { - -/** - * Metric set for all legacy metrics reported by proton. - * - * @deprecated Use ContentProtonMetrics for all new metrics. - */ -struct LegacyProtonMetrics : metrics::MetricSet -{ - struct DocumentTypeMetrics : metrics::MetricSet { - // documentdb metrics will be wired in here (by the metrics engine) - DocumentTypeMetrics(metrics::MetricSet *parent); - ~DocumentTypeMetrics(); - }; - - DocumentTypeMetrics docTypes; - ExecutorMetrics executor; - ExecutorMetrics flushExecutor; - ExecutorMetrics matchExecutor; - ExecutorMetrics summaryExecutor; - metrics::SumMetric<metrics::LongValueMetric> memoryUsage; - metrics::SumMetric<metrics::LongValueMetric> diskUsage; - metrics::SumMetric<metrics::LongValueMetric> docsInMemory; - metrics::SumMetric<metrics::LongValueMetric> numDocs; - metrics::SumMetric<metrics::LongValueMetric> numActiveDocs; - metrics::SumMetric<metrics::LongValueMetric> numIndexedDocs; - metrics::SumMetric<metrics::LongValueMetric> numStoredDocs; - metrics::SumMetric<metrics::LongValueMetric> numRemovedDocs; - // transport metrics will be wired in here - - LegacyProtonMetrics(); - ~LegacyProtonMetrics(); -}; - -} // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/metrics/legacy_sessionmanager_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/legacy_sessionmanager_metrics.cpp deleted file mode 100644 index 6d780739848..00000000000 --- a/searchcore/src/vespa/searchcore/proton/metrics/legacy_sessionmanager_metrics.cpp +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "legacy_sessionmanager_metrics.h" - -namespace proton { - -LegacySessionManagerMetrics::LegacySessionManagerMetrics(metrics::MetricSet *parent) - : metrics::MetricSet("sessionmanager", {}, "Grouping session manager metrics", parent), - numInsert("numinsert", {}, "Number of inserted sessions", this), - numPick("numpick", {}, "Number if picked sessions", this), - numDropped("numdropped", {}, "Number of dropped cached sessions", this), - numCached("numcached", {}, "Number of currently cached sessions", this), - numTimedout("numtimedout", {}, "Number of timed out sessions", this) -{ -} - -LegacySessionManagerMetrics::~LegacySessionManagerMetrics() = default; - -void -LegacySessionManagerMetrics::update(const proton::matching::SessionManager::Stats &stats) -{ - numInsert.inc(stats.numInsert); - numPick.inc(stats.numPick); - numDropped.inc(stats.numDropped); - numCached.set(stats.numCached); - numTimedout.inc(stats.numTimedout); -} - -} diff --git a/searchcore/src/vespa/searchcore/proton/metrics/legacy_sessionmanager_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/legacy_sessionmanager_metrics.h deleted file mode 100644 index 3e8e3f54c02..00000000000 --- a/searchcore/src/vespa/searchcore/proton/metrics/legacy_sessionmanager_metrics.h +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/metrics/countmetric.h> -#include <vespa/metrics/metricset.h> -#include <vespa/metrics/valuemetric.h> -#include <vespa/searchcore/proton/matching/sessionmanager.h> - -namespace proton { - -/** - * Legacy metrics for session manager cache. - * TODO: Remove on Vespa 7 - */ -struct LegacySessionManagerMetrics : metrics::MetricSet -{ - metrics::LongCountMetric numInsert; - metrics::LongCountMetric numPick; - metrics::LongCountMetric numDropped; - metrics::LongValueMetric numCached; - metrics::LongCountMetric numTimedout; - - void update(const proton::matching::SessionManager::Stats &stats); - LegacySessionManagerMetrics(metrics::MetricSet *parent); - ~LegacySessionManagerMetrics(); -}; - -} diff --git a/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp b/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp index 3073b5df3c9..d34ec8d05a8 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "attribute_metrics.h" +#include "documentdb_tagged_metrics.h" #include "metrics_engine.h" -#include "attribute_metrics_collection.h" -#include "documentdb_metrics_collection.h" #include <vespa/metrics/jsonwriter.h> #include <vespa/metrics/metricmanager.h> @@ -13,7 +13,6 @@ namespace proton { MetricsEngine::MetricsEngine() : _root(), - _legacyRoot(), _manager(std::make_unique<metrics::MetricManager>()), _metrics_producer(*_manager) { } @@ -26,7 +25,6 @@ MetricsEngine::start(const config::ConfigUri &) { metrics::MetricLockGuard guard(_manager->getMetricLock()); _manager->registerMetric(guard, _root); - _manager->registerMetric(guard, _legacyRoot); } // Storage doesnt snapshot unset metrics to save memory. Currently @@ -56,114 +54,33 @@ void MetricsEngine::addExternalMetrics(metrics::Metric &child) { metrics::MetricLockGuard guard(_manager->getMetricLock()); - _legacyRoot.registerMetric(child); + _root.registerMetric(child); } void MetricsEngine::removeExternalMetrics(metrics::Metric &child) { metrics::MetricLockGuard guard(_manager->getMetricLock()); - _legacyRoot.unregisterMetric(child); -} - -namespace { - -void -addLegacyDocumentDBMetrics(LegacyProtonMetrics &legacyRoot, - LegacyDocumentDBMetrics &metrics) -{ - legacyRoot.docTypes.registerMetric(metrics); - // cannot use sum of sum due to metric clone issues - legacyRoot.memoryUsage.addMetricToSum(metrics.index.memoryUsage); - legacyRoot.memoryUsage.addMetricToSum(metrics.attributes.memoryUsage); - legacyRoot.memoryUsage.addMetricToSum(metrics.docstore.memoryUsage); - legacyRoot.diskUsage.addMetricToSum(metrics.index.diskUsage); - legacyRoot.docsInMemory.addMetricToSum(metrics.index.docsInMemory); - legacyRoot.numDocs.addMetricToSum(metrics.numDocs); - legacyRoot.numActiveDocs.addMetricToSum(metrics.numActiveDocs); - legacyRoot.numIndexedDocs.addMetricToSum(metrics.numIndexedDocs); - legacyRoot.numStoredDocs.addMetricToSum(metrics.numStoredDocs); - legacyRoot.numRemovedDocs.addMetricToSum(metrics.numRemovedDocs); -} - -void -removeLegacyDocumentDBMetrics(LegacyProtonMetrics &legacyRoot, - LegacyDocumentDBMetrics &metrics) -{ - legacyRoot.docTypes.unregisterMetric(metrics); - // cannot use sum of sum due to metric clone issues - legacyRoot.memoryUsage.removeMetricFromSum(metrics.index.memoryUsage); - legacyRoot.memoryUsage.removeMetricFromSum(metrics.attributes.memoryUsage); - legacyRoot.memoryUsage.removeMetricFromSum(metrics.docstore.memoryUsage); - legacyRoot.diskUsage.removeMetricFromSum(metrics.index.diskUsage); - legacyRoot.docsInMemory.removeMetricFromSum(metrics.index.docsInMemory); - legacyRoot.numDocs.removeMetricFromSum(metrics.numDocs); - legacyRoot.numActiveDocs.removeMetricFromSum(metrics.numActiveDocs); - legacyRoot.numIndexedDocs.removeMetricFromSum(metrics.numIndexedDocs); - legacyRoot.numStoredDocs.removeMetricFromSum(metrics.numStoredDocs); - legacyRoot.numRemovedDocs.removeMetricFromSum(metrics.numRemovedDocs); -} - + _root.unregisterMetric(child); } void -MetricsEngine::addDocumentDBMetrics(DocumentDBMetricsCollection &child) +MetricsEngine::addDocumentDBMetrics(DocumentDBTaggedMetrics &child) { metrics::MetricLockGuard guard(_manager->getMetricLock()); - addLegacyDocumentDBMetrics(_legacyRoot, child.getLegacyMetrics()); - - _root.registerMetric(child.getTaggedMetrics()); + _root.registerMetric(child); } void -MetricsEngine::removeDocumentDBMetrics(DocumentDBMetricsCollection &child) +MetricsEngine::removeDocumentDBMetrics(DocumentDBTaggedMetrics &child) { metrics::MetricLockGuard guard(_manager->getMetricLock()); - removeLegacyDocumentDBMetrics(_legacyRoot, child.getLegacyMetrics()); - - _root.unregisterMetric(child.getTaggedMetrics()); + _root.unregisterMetric(child); } namespace { void -doAddAttribute(LegacyAttributeMetrics &attributes, - const std::string &name) -{ - LegacyAttributeMetrics::List::Entry *entry = attributes.list.add(name); - if (entry != nullptr) { - LOG(debug, "doAddAttribute(): name='%s', attributes=%p", - name.c_str(), (void*)&attributes); - attributes.list.registerMetric(*entry); - } else { - LOG(warning, "multiple attributes have the same name: '%s'", name.c_str()); - } -} - -void -doRemoveAttribute(LegacyAttributeMetrics &attributes, - const std::string &name) -{ - LegacyAttributeMetrics::List::Entry::UP entry = attributes.list.remove(name); - if (entry.get() != 0) { - LOG(debug, "doRemoveAttribute(): name='%s', attributes=%p", - name.c_str(), (void*)&attributes); - attributes.list.unregisterMetric(*entry); - } else { - LOG(debug, "Could not remove attribute with name '%s', not found", name.c_str()); - } -} - -void -doCleanAttributes(LegacyAttributeMetrics &attributes) -{ - std::vector<LegacyAttributeMetrics::List::Entry::UP> entries = attributes.list.release(); - for (size_t i = 0; i < entries.size(); ++i) { - attributes.list.unregisterMetric(*entries[i]); - } -} - -void doAddAttribute(AttributeMetrics &attributes, const std::string &attrName) { auto entry = attributes.add(attrName); @@ -197,41 +114,26 @@ doCleanAttributes(AttributeMetrics &attributes) } void -MetricsEngine::addAttribute(const AttributeMetricsCollection &subAttributes, - LegacyAttributeMetrics *totalAttributes, +MetricsEngine::addAttribute(AttributeMetrics &subAttributes, const std::string &name) { metrics::MetricLockGuard guard(_manager->getMetricLock()); - doAddAttribute(subAttributes.getMetrics(), name); - doAddAttribute(subAttributes.getLegacyMetrics(), name); - if (totalAttributes != NULL) { - doAddAttribute(*totalAttributes, name); - } + doAddAttribute(subAttributes, name); } void -MetricsEngine::removeAttribute(const AttributeMetricsCollection &subAttributes, - LegacyAttributeMetrics *totalAttributes, +MetricsEngine::removeAttribute(AttributeMetrics &subAttributes, const std::string &name) { metrics::MetricLockGuard guard(_manager->getMetricLock()); - doRemoveAttribute(subAttributes.getMetrics(), name); - doRemoveAttribute(subAttributes.getLegacyMetrics(), name); - if (totalAttributes != NULL) { - doRemoveAttribute(*totalAttributes, name); - } + doRemoveAttribute(subAttributes, name); } void -MetricsEngine::cleanAttributes(const AttributeMetricsCollection &subAttributes, - LegacyAttributeMetrics *totalAttributes) +MetricsEngine::cleanAttributes(AttributeMetrics &subAttributes) { metrics::MetricLockGuard guard(_manager->getMetricLock()); - doCleanAttributes(subAttributes.getMetrics()); - doCleanAttributes(subAttributes.getLegacyMetrics()); - if (totalAttributes != NULL) { - doCleanAttributes(*totalAttributes); - } + doCleanAttributes(subAttributes); } namespace { @@ -263,18 +165,17 @@ cleanRankProfilesIn(MatchingMetricsType &matchingMetrics) } void -MetricsEngine::addRankProfile(DocumentDBMetricsCollection &owner, const std::string &name, size_t numDocIdPartitions) { +MetricsEngine::addRankProfile(DocumentDBTaggedMetrics &owner, const std::string &name, size_t numDocIdPartitions) +{ metrics::MetricLockGuard guard(_manager->getMetricLock()); - size_t adjustedNumDocIdPartitions = std::min(numDocIdPartitions, owner.getLegacyMetrics()._maxNumThreads); - addRankProfileTo(owner.getTaggedMetrics().matching, name, adjustedNumDocIdPartitions); - addRankProfileTo(owner.getLegacyMetrics().matching, name, adjustedNumDocIdPartitions); + size_t adjustedNumDocIdPartitions = std::min(numDocIdPartitions, owner.maxNumThreads); + addRankProfileTo(owner.matching, name, adjustedNumDocIdPartitions); } void -MetricsEngine::cleanRankProfiles(DocumentDBMetricsCollection &owner) { +MetricsEngine::cleanRankProfiles(DocumentDBTaggedMetrics &owner) { metrics::MetricLockGuard guard(_manager->getMetricLock()); - cleanRankProfilesIn(owner.getTaggedMetrics().matching); - cleanRankProfilesIn(owner.getLegacyMetrics().matching); + cleanRankProfilesIn(owner.matching); } void @@ -283,4 +184,4 @@ MetricsEngine::stop() _manager->stop(); } -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.h b/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.h index 44be43c11b3..dc8872c6c8b 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.h @@ -3,7 +3,6 @@ #pragma once #include "content_proton_metrics.h" -#include "legacy_proton_metrics.h" #include "metricswireservice.h" #include <vespa/metrics/state_api_adapter.h> @@ -17,13 +16,13 @@ namespace config { } namespace proton { -class DocumentDBMetricsCollection; +class AttributeMetrics; +class DocumentDBTaggedMetrics; class MetricsEngine : public MetricsWireService { private: ContentProtonMetrics _root; - LegacyProtonMetrics _legacyRoot; std::unique_ptr<metrics::MetricManager> _manager; metrics::StateApiAdapter _metrics_producer; @@ -33,24 +32,20 @@ public: MetricsEngine(); virtual ~MetricsEngine(); ContentProtonMetrics &root() { return _root; } - LegacyProtonMetrics &legacyRoot() { return _legacyRoot; } void start(const config::ConfigUri & configUri); void addMetricsHook(metrics::UpdateHook &hook); void removeMetricsHook(metrics::UpdateHook &hook); void addExternalMetrics(metrics::Metric &child); void removeExternalMetrics(metrics::Metric &child); - void addDocumentDBMetrics(DocumentDBMetricsCollection &child); - void removeDocumentDBMetrics(DocumentDBMetricsCollection &child); - void addAttribute(const AttributeMetricsCollection &subAttributes, - LegacyAttributeMetrics *totalAttributes, + void addDocumentDBMetrics(DocumentDBTaggedMetrics &child); + void removeDocumentDBMetrics(DocumentDBTaggedMetrics &child); + void addAttribute(AttributeMetrics &subAttributes, const std::string &name) override; - void removeAttribute(const AttributeMetricsCollection &subAttributes, - LegacyAttributeMetrics *totalAttributes, + void removeAttribute(AttributeMetrics &subAttributes, const std::string &name) override; - void cleanAttributes(const AttributeMetricsCollection &subAttributes, - LegacyAttributeMetrics *totalAttributes) override; - void addRankProfile(DocumentDBMetricsCollection &owner, const std::string &name, size_t numDocIdPartitions) override; - void cleanRankProfiles(DocumentDBMetricsCollection &owner) override; + void cleanAttributes(AttributeMetrics &subAttributes) override; + void addRankProfile(DocumentDBTaggedMetrics &owner, const std::string &name, size_t numDocIdPartitions) override; + void cleanRankProfiles(DocumentDBTaggedMetrics &owner) override; void stop(); vespalib::MetricsProducer &metrics_producer() { return _metrics_producer; } diff --git a/searchcore/src/vespa/searchcore/proton/metrics/metricswireservice.h b/searchcore/src/vespa/searchcore/proton/metrics/metricswireservice.h index faa172544f7..70602adbd7b 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/metricswireservice.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/metricswireservice.h @@ -5,33 +5,30 @@ #include <string> namespace proton { -class AttributeMetricsCollection; -class DocumentDBMetricsCollection; -class LegacyAttributeMetrics; + +class AttributeMetrics; +class DocumentDBTaggedMetrics; struct MetricsWireService { - virtual void addAttribute(const AttributeMetricsCollection &subAttributes, - LegacyAttributeMetrics *totalAttributes, + virtual void addAttribute(AttributeMetrics &subAttributes, const std::string &name) = 0; - virtual void removeAttribute(const AttributeMetricsCollection &subAttributes, - LegacyAttributeMetrics *totalAttributes, + virtual void removeAttribute(AttributeMetrics &subAttributes, const std::string &name) = 0; - virtual void cleanAttributes(const AttributeMetricsCollection &subAttributes, - LegacyAttributeMetrics *totalAttributes) = 0; - virtual void addRankProfile(DocumentDBMetricsCollection &owner, + virtual void cleanAttributes(AttributeMetrics &subAttributes) = 0; + virtual void addRankProfile(DocumentDBTaggedMetrics &owner, const std::string &name, size_t numDocIdPartitions) = 0; - virtual void cleanRankProfiles(DocumentDBMetricsCollection &owner) = 0; + virtual void cleanRankProfiles(DocumentDBTaggedMetrics &owner) = 0; virtual ~MetricsWireService() {} }; struct DummyWireService : public MetricsWireService { - virtual void addAttribute(const AttributeMetricsCollection &, LegacyAttributeMetrics *, const std::string &) override {} - virtual void removeAttribute(const AttributeMetricsCollection &, LegacyAttributeMetrics *, const std::string &) override {} - virtual void cleanAttributes(const AttributeMetricsCollection &, LegacyAttributeMetrics *) override {} - virtual void addRankProfile(DocumentDBMetricsCollection &, const std::string &, size_t) override {} - virtual void cleanRankProfiles(DocumentDBMetricsCollection &) override {} + virtual void addAttribute(AttributeMetrics &, const std::string &) override {} + virtual void removeAttribute(AttributeMetrics &, const std::string &) override {} + virtual void cleanAttributes(AttributeMetrics &) override {} + virtual void addRankProfile(DocumentDBTaggedMetrics &, const std::string &, size_t) override {} + virtual void cleanRankProfiles(DocumentDBTaggedMetrics &) override {} }; -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 0f6f4fdda11..6a1d053745f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -20,7 +20,6 @@ #include <vespa/searchcore/proton/feedoperation/noopoperation.h> #include <vespa/searchcore/proton/index/index_writer.h> #include <vespa/searchcore/proton/initializer/task_runner.h> -#include <vespa/searchcore/proton/metrics/attribute_metrics_collection.h> #include <vespa/searchcore/proton/metrics/metricswireservice.h> #include <vespa/searchcore/proton/reference/i_document_db_reference_resolver.h> #include <vespa/searchcore/proton/reference/i_document_db_reference_registry.h> @@ -155,7 +154,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _writeFilter(), _feedHandler(_writeService, tlsSpec, docTypeName, _state, *this, _writeFilter, *this, tlsDirectWriter), _subDBs(*this, *this, _feedHandler, _docTypeName, _writeService, warmupExecutor, - sharedExecutor, fileHeaderContext, metricsWireService, getMetricsCollection(), + sharedExecutor, fileHeaderContext, metricsWireService, getMetrics(), queryLimiter, clock, _configMutex, _baseDir, makeSubDBConfig(protonCfg), hwInfo), _maintenanceController(_writeService.master(), sharedExecutor, _docTypeName), _visibility(_feedHandler, _writeService, _feedView), @@ -547,12 +546,9 @@ DocumentDB::close() stopMaintenance(); // The attributes in the ready sub db is also the total set of attributes. - DocumentDBTaggedMetrics &metrics = getMetricsCollection().getTaggedMetrics(); - LegacyDocumentDBMetrics &legacyMetrics = getMetricsCollection().getLegacyMetrics(); - AttributeMetricsCollection ready(metrics.ready.attributes, legacyMetrics.ready.attributes); - AttributeMetricsCollection notReady(metrics.notReady.attributes, legacyMetrics.notReady.attributes); - _metricsWireService.cleanAttributes(ready, &legacyMetrics.attributes); - _metricsWireService.cleanAttributes(notReady, nullptr); + DocumentDBTaggedMetrics &metrics = getMetrics(); + _metricsWireService.cleanAttributes(metrics.ready.attributes); + _metricsWireService.cleanAttributes(metrics.notReady.attributes); _writeService.sync(); masterExecute([this] () { closeSubDBs(); } ); _writeService.sync(); @@ -1030,7 +1026,7 @@ DocumentDB::notifyAllBucketsChanged() } void -DocumentDB::updateMetrics(DocumentDBMetricsCollection &metrics) +DocumentDB::updateMetrics(DocumentDBTaggedMetrics &metrics) { if (_state.getState() < DDBState::State::REPLAY_TRANSACTION_LOG) { return; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index f9b4e5ad1f6..716a1058348 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -26,7 +26,7 @@ #include <vespa/searchcore/proton/common/doctypename.h> #include <vespa/searchcore/proton/common/monitored_refcount.h> #include <vespa/searchcore/proton/metrics/documentdb_job_trackers.h> -#include <vespa/searchcore/proton/metrics/documentdb_metrics_collection.h> +#include <vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h> #include <vespa/searchcore/proton/persistenceengine/bucket_guard.h> #include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h> #include <vespa/searchcore/proton/index/indexmanager.h> @@ -66,7 +66,7 @@ class DocumentDB : public DocumentDBConfigOwner, { private: class MetricsUpdateHook : public metrics::UpdateHook { - DocumentDBMetricsCollection _metrics; + DocumentDBTaggedMetrics _metrics; DocumentDB &_db; public: MetricsUpdateHook(DocumentDB &s, const std::string &doc_type, size_t maxNumThreads) @@ -74,7 +74,7 @@ private: _metrics(doc_type, maxNumThreads), _db(s) {} void updateMetrics(const MetricLockGuard & ) override { _db.updateMetrics(_metrics); } - DocumentDBMetricsCollection &getMetrics() { return _metrics; } + DocumentDBTaggedMetrics &getMetrics() { return _metrics; } }; struct DocumentStoreCacheStats { @@ -289,11 +289,11 @@ public: void close(); /** - * Obtain the metrics collection for this document db. + * Obtain the metrics for this document db. * * @return document db metrics **/ - DocumentDBMetricsCollection &getMetricsCollection() { return _metricsHook.getMetrics(); } + DocumentDBTaggedMetrics &getMetrics() { return _metricsHook.getMetrics(); } /** * Obtain the metrics update hook for this document db. @@ -424,7 +424,7 @@ public: * the metric manager). Do not call this function in multiple * threads at once. **/ - void updateMetrics(DocumentDBMetricsCollection &metrics); + void updateMetrics(DocumentDBTaggedMetrics &metrics); /** * Implement search::transactionlog::SyncProxy API. diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp index 71baabd9d6f..e01fb048266 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp @@ -13,7 +13,6 @@ #include <vespa/searchcore/proton/matching/matching_stats.h> #include <vespa/searchcore/proton/matching/matching_stats.h> #include <vespa/searchcore/proton/metrics/documentdb_job_trackers.h> -#include <vespa/searchcore/proton/metrics/documentdb_metrics_collection.h> #include <vespa/searchcore/proton/metrics/executor_threading_service_stats.h> #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/docstore/cachestats.h> @@ -71,17 +70,12 @@ updateDiskUsageMetric(metrics::LongValueMetric &metric, uint64_t diskUsage, Tota } void -updateIndexMetrics(DocumentDBMetricsCollection &metrics, const search::SearchableStats &stats, TotalStats &totalStats) +updateIndexMetrics(DocumentDBTaggedMetrics &metrics, const search::SearchableStats &stats, TotalStats &totalStats) { - DocumentDBTaggedMetrics::IndexMetrics &indexMetrics = metrics.getTaggedMetrics().index; + DocumentDBTaggedMetrics::IndexMetrics &indexMetrics = metrics.index; updateDiskUsageMetric(indexMetrics.diskUsage, stats.sizeOnDisk(), totalStats); updateMemoryUsageMetrics(indexMetrics.memoryUsage, stats.memoryUsage(), totalStats); indexMetrics.docsInMemory.set(stats.docsInMemory()); - - LegacyDocumentDBMetrics::IndexMetrics &legacyIndexMetrics = metrics.getLegacyMetrics().index; - legacyIndexMetrics.memoryUsage.set(stats.memoryUsage().allocatedBytes()); - legacyIndexMetrics.docsInMemory.set(stats.docsInMemory()); - legacyIndexMetrics.diskUsage.set(stats.sizeOnDisk()); } struct TempAttributeMetric @@ -153,22 +147,6 @@ fillTempAttributeMetrics(TempAttributeMetrics &totalMetrics, } void -updateLegacyAttributeMetrics(LegacyAttributeMetrics &metrics, const TempAttributeMetrics &tmpMetrics) -{ - for (const auto &attr : tmpMetrics.attrs) { - LegacyAttributeMetrics::List::Entry *entry = metrics.list.get(attr.first); - if (entry) { - entry->memoryUsage.set(attr.second.memoryUsage.allocatedBytes()); - entry->bitVectors.set(attr.second.bitVectors); - } else { - LOG(debug, "Could not update metrics for attribute: '%s'", attr.first.c_str()); - } - } - metrics.memoryUsage.set(tmpMetrics.total.memoryUsage.allocatedBytes()); - metrics.bitVectors.set(tmpMetrics.total.bitVectors); -} - -void updateAttributeMetrics(AttributeMetrics &metrics, const TempAttributeMetrics &tmpMetrics) { for (const auto &attr : tmpMetrics.attrs) { @@ -180,60 +158,43 @@ updateAttributeMetrics(AttributeMetrics &metrics, const TempAttributeMetrics &tm } void -updateAttributeMetrics(DocumentDBMetricsCollection &metrics, const DocumentSubDBCollection &subDbs, TotalStats &totalStats) +updateAttributeMetrics(DocumentDBTaggedMetrics &metrics, const DocumentSubDBCollection &subDbs, TotalStats &totalStats) { TempAttributeMetrics totalMetrics; TempAttributeMetrics readyMetrics; TempAttributeMetrics notReadyMetrics; fillTempAttributeMetrics(totalMetrics, readyMetrics, notReadyMetrics, subDbs); - updateLegacyAttributeMetrics(metrics.getLegacyMetrics().attributes, totalMetrics); - updateLegacyAttributeMetrics(metrics.getLegacyMetrics().ready.attributes, readyMetrics); - updateLegacyAttributeMetrics(metrics.getLegacyMetrics().notReady.attributes, notReadyMetrics); - - updateAttributeMetrics(metrics.getTaggedMetrics().ready.attributes, readyMetrics); - updateAttributeMetrics(metrics.getTaggedMetrics().notReady.attributes, notReadyMetrics); - updateMemoryUsageMetrics(metrics.getTaggedMetrics().attribute.totalMemoryUsage, totalMetrics.total.memoryUsage, totalStats); -} - -void -updateLegacyRankProfileMetrics(LegacyDocumentDBMetrics::MatchingMetrics &matchingMetrics, - const vespalib::string &rankProfileName, - const MatchingStats &stats) -{ - auto itr = matchingMetrics.rank_profiles.find(rankProfileName); - assert(itr != matchingMetrics.rank_profiles.end()); - itr->second->update(stats); + updateAttributeMetrics(metrics.ready.attributes, readyMetrics); + updateAttributeMetrics(metrics.notReady.attributes, notReadyMetrics); + updateMemoryUsageMetrics(metrics.attribute.totalMemoryUsage, totalMetrics.total.memoryUsage, totalStats); } void -updateMatchingMetrics(DocumentDBMetricsCollection &metrics, const IDocumentSubDB &ready) +updateMatchingMetrics(DocumentDBTaggedMetrics &metrics, const IDocumentSubDB &ready) { MatchingStats totalStats; - for (const auto &rankProfile : metrics.getTaggedMetrics().matching.rank_profiles) { + for (const auto &rankProfile : metrics.matching.rank_profiles) { MatchingStats matchingStats = ready.getMatcherStats(rankProfile.first); rankProfile.second->update(matchingStats); - updateLegacyRankProfileMetrics(metrics.getLegacyMetrics().matching, rankProfile.first, matchingStats); totalStats.add(matchingStats); } - metrics.getTaggedMetrics().matching.update(totalStats); - metrics.getLegacyMetrics().matching.update(totalStats); + metrics.matching.update(totalStats); } void -updateSessionCacheMetrics(DocumentDBMetricsCollection &metrics, proton::matching::SessionManager &sessionManager) +updateSessionCacheMetrics(DocumentDBTaggedMetrics &metrics, proton::matching::SessionManager &sessionManager) { auto searchStats = sessionManager.getSearchStats(); - metrics.getTaggedMetrics().sessionCache.search.update(searchStats); + metrics.sessionCache.search.update(searchStats); auto groupingStats = sessionManager.getGroupingStats(); - metrics.getTaggedMetrics().sessionCache.grouping.update(groupingStats); - metrics.getLegacyMetrics().sessionManager.update(groupingStats); + metrics.sessionCache.grouping.update(groupingStats); } void -updateDocumentsMetrics(DocumentDBMetricsCollection &metrics, const DocumentSubDBCollection &subDbs) +updateDocumentsMetrics(DocumentDBTaggedMetrics &metrics, const DocumentSubDBCollection &subDbs) { DocumentMetaStoreReadGuards dms(subDbs); uint32_t active = dms.numActiveDocs(); @@ -241,18 +202,11 @@ updateDocumentsMetrics(DocumentDBMetricsCollection &metrics, const DocumentSubDB uint32_t total = dms.numTotalDocs(); uint32_t removed = dms.numRemovedDocs(); - auto &docsMetrics = metrics.getTaggedMetrics().documents; + auto &docsMetrics = metrics.documents; docsMetrics.active.set(active); docsMetrics.ready.set(ready); docsMetrics.total.set(total); docsMetrics.removed.set(removed); - - auto &legacyMetrics = metrics.getLegacyMetrics(); - legacyMetrics.numDocs.set(ready); - legacyMetrics.numActiveDocs.set(active); - legacyMetrics.numIndexedDocs.set(ready); - legacyMetrics.numStoredDocs.set(total); - legacyMetrics.numRemovedDocs.set(removed); } void @@ -283,28 +237,6 @@ updateCountMetric(uint64_t currVal, uint64_t lastVal, metrics::LongCountMetric & } void -updateLegacyDocstoreMetrics(LegacyDocumentDBMetrics::DocstoreMetrics &metrics, - const DocumentSubDBCollection &sub_dbs, - CacheStats &lastCacheStats) -{ - size_t memoryUsage = 0; - CacheStats cache_stats; - for (const auto subDb : sub_dbs) { - const ISummaryManager::SP &summaryMgr = subDb->getSummaryManager(); - if (summaryMgr) { - cache_stats += summaryMgr->getBackingStore().getCacheStats(); - memoryUsage += summaryMgr->getBackingStore().memoryUsed(); - } - } - metrics.memoryUsage.set(memoryUsage); - updateCountMetric(cache_stats.lookups(), lastCacheStats.lookups(), metrics.cacheLookups); - updateDocumentStoreCacheHitRate(cache_stats, lastCacheStats, metrics.cacheHitRate); - metrics.cacheElements.set(cache_stats.elements); - metrics.cacheMemoryUsed.set(cache_stats.memory_used); - lastCacheStats = cache_stats; -} - -void updateDocumentStoreMetrics(DocumentDBTaggedMetrics::SubDBMetrics::DocumentStoreMetrics &metrics, const IDocumentSubDB *subDb, CacheStats &lastCacheStats, @@ -353,37 +285,20 @@ updateLidSpaceMetrics(MetricSetType &metrics, const search::IDocumentMetaStore & } void -DocumentDBMetricsUpdater::updateMetrics(DocumentDBMetricsCollection &metrics) +DocumentDBMetricsUpdater::updateMetrics(DocumentDBTaggedMetrics &metrics) { TotalStats totalStats; ExecutorThreadingServiceStats threadingServiceStats = _writeService.getStats(); - updateLegacyMetrics(metrics.getLegacyMetrics(), threadingServiceStats); updateIndexMetrics(metrics, _subDBs.getReadySubDB()->getSearchableStats(), totalStats); updateAttributeMetrics(metrics, _subDBs, totalStats); updateMatchingMetrics(metrics, *_subDBs.getReadySubDB()); updateSessionCacheMetrics(metrics, _sessionManager); updateDocumentsMetrics(metrics, _subDBs); - updateDocumentStoreMetrics(metrics.getTaggedMetrics(), _subDBs, _lastDocStoreCacheStats, totalStats); - updateMiscMetrics(metrics.getTaggedMetrics(), threadingServiceStats); - - metrics.getTaggedMetrics().totalMemoryUsage.update(totalStats.memoryUsage); - metrics.getTaggedMetrics().totalDiskUsage.set(totalStats.diskUsage); -} - -void -DocumentDBMetricsUpdater::updateLegacyMetrics(LegacyDocumentDBMetrics &metrics, const ExecutorThreadingServiceStats &threadingServiceStats) -{ - metrics.executor.update(threadingServiceStats.getMasterExecutorStats()); - metrics.summaryExecutor.update(threadingServiceStats.getSummaryExecutorStats()); - metrics.indexExecutor.update(threadingServiceStats.getIndexExecutorStats()); - updateLegacyDocstoreMetrics(metrics.docstore, _subDBs, _lastDocStoreCacheStats.total); - - DocumentMetaStoreReadGuards dmss(_subDBs); - updateLidSpaceMetrics(metrics.ready.docMetaStore, dmss.readydms->get()); - updateLidSpaceMetrics(metrics.notReady.docMetaStore, dmss.notreadydms->get()); - updateLidSpaceMetrics(metrics.removed.docMetaStore, dmss.remdms->get()); + updateDocumentStoreMetrics(metrics, _subDBs, _lastDocStoreCacheStats, totalStats); + updateMiscMetrics(metrics, threadingServiceStats); - metrics.numBadConfigs.set(_state.getDelayedConfig() ? 1u : 0u); + metrics.totalMemoryUsage.update(totalStats.memoryUsage); + metrics.totalDiskUsage.set(totalStats.diskUsage); } void diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.h b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.h index e0042207060..7c27b273c59 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.h @@ -11,11 +11,9 @@ namespace matching { class SessionManager; } class AttributeUsageFilter; class DDBState; class DocumentDBJobTrackers; -class DocumentDBMetricsCollection; class DocumentSubDBCollection; class ExecutorThreadingService; class ExecutorThreadingServiceStats; -class LegacyDocumentDBMetrics; /** * Class used to update metrics for a document db. @@ -24,11 +22,10 @@ class DocumentDBMetricsUpdater { public: struct DocumentStoreCacheStats { - search::CacheStats total; search::CacheStats readySubDb; search::CacheStats notReadySubDb; search::CacheStats removedSubDb; - DocumentStoreCacheStats() : total(), readySubDb(), notReadySubDb(), removedSubDb() {} + DocumentStoreCacheStats() : readySubDb(), notReadySubDb(), removedSubDb() {} }; private: @@ -41,7 +38,6 @@ private: // Last updated document store cache statistics. Necessary due to metrics implementation is upside down. DocumentStoreCacheStats _lastDocStoreCacheStats; - void updateLegacyMetrics(LegacyDocumentDBMetrics &metrics, const ExecutorThreadingServiceStats &threadingServiceStats); void updateMiscMetrics(DocumentDBTaggedMetrics &metrics, const ExecutorThreadingServiceStats &threadingServiceStats); void updateAttributeResourceUsageMetrics(DocumentDBTaggedMetrics::AttributeMetrics &metrics); @@ -54,7 +50,7 @@ public: const DDBState &state); ~DocumentDBMetricsUpdater(); - void updateMetrics(DocumentDBMetricsCollection &metrics); + void updateMetrics(DocumentDBTaggedMetrics &metrics); }; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index 0429ccb4a61..1d1bbf21d70 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -193,7 +193,7 @@ deriveConfig(const ProtonConfig::Summary & summary, const ProtonConfig::Flush::M logConfig.setMaxFileSize(log.maxfilesize) .setMaxDiskBloatFactor(std::min(flush.diskbloatfactor, flush.each.diskbloatfactor)) .setMaxBucketSpread(log.maxbucketspread).setMinFileSizeFactor(log.minfilesizefactor) - .compact2ActiveFile(log.compact2activefile).compactCompression(deriveCompression(log.compact.compression)) + .compactCompression(deriveCompression(log.compact.compression)) .setFileConfig(fileConfig).disableCrcOnRead(chunk.skipcrconread); return LogDocumentStore::Config(config, logConfig); } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp index 78f49599322..22b50a65a41 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp @@ -7,8 +7,7 @@ #include "i_document_subdb_owner.h" #include "maintenancecontroller.h" #include "searchabledocsubdb.h" - -#include <vespa/searchcore/proton/metrics/documentdb_metrics_collection.h> +#include <vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h> using proton::matching::SessionManager; using search::GrowStrategy; @@ -37,7 +36,7 @@ DocumentSubDBCollection::DocumentSubDBCollection( vespalib::ThreadStackExecutorBase &sharedExecutor, const search::common::FileHeaderContext &fileHeaderContext, MetricsWireService &metricsWireService, - DocumentDBMetricsCollection &metrics, + DocumentDBTaggedMetrics &metrics, matching::QueryLimiter &queryLimiter, const vespalib::Clock &clock, std::mutex &configMutex, @@ -76,13 +75,11 @@ DocumentSubDBCollection::DocumentSubDBCollection( cfg.getNumSearchThreads()), SearchableDocSubDB::Context(FastAccessDocSubDB::Context (context, - AttributeMetricsCollection(metrics.getTaggedMetrics().ready.attributes, - metrics.getLegacyMetrics().ready.attributes), - &metrics.getLegacyMetrics().attributes, - metricsWireService), - queryLimiter, - clock, - warmupExecutor))); + metrics.ready.attributes, + metricsWireService), + queryLimiter, + clock, + warmupExecutor))); _subDBs.push_back (new StoreOnlyDocSubDB(StoreOnlyDocSubDB::Config(docTypeName, "1.removed", @@ -105,9 +102,7 @@ DocumentSubDBCollection::DocumentSubDBCollection( true, true), FastAccessDocSubDB::Context(context, - AttributeMetricsCollection(metrics.getTaggedMetrics().notReady.attributes, - metrics.getLegacyMetrics().notReady.attributes), - nullptr, + metrics.notReady.attributes, metricsWireService))); } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h index c4ddd6d0e72..68f790307ce 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h @@ -26,7 +26,7 @@ namespace searchcorespi { namespace proton { class DocumentDBConfig; -class DocumentDBMetricsCollection; +class DocumentDBTaggedMetrics; class MaintenanceController; class MetricsWireService; class ICommitable; @@ -102,7 +102,7 @@ public: vespalib::ThreadStackExecutorBase &sharedExecutor, const search::common::FileHeaderContext &fileHeaderContext, MetricsWireService &metricsWireService, - DocumentDBMetricsCollection &metrics, + DocumentDBTaggedMetrics &metrics, matching::QueryLimiter & queryLimiter, const vespalib::Clock &clock, std::mutex &configMutex, diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp index bf83c889f27..531873fcd19 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp @@ -13,7 +13,7 @@ #include <vespa/searchcore/proton/attribute/attribute_populator.h> #include <vespa/searchcore/proton/attribute/filter_attribute_manager.h> #include <vespa/searchcore/proton/attribute/sequential_attributes_initializer.h> -#include <vespa/searchcore/proton/metrics/legacy_documentdb_metrics.h> +#include <vespa/searchcore/proton/matching/sessionmanager.h> #include <vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h> #include <vespa/searchcore/proton/reprocessing/document_reprocessing_handler.h> #include <vespa/searchcore/proton/reprocessing/reprocess_documents_task.h> @@ -94,8 +94,7 @@ FastAccessDocSubDB::setupAttributeManager(AttributeManager::SP attrMgrResult) attrMgrResult->getAttributeListAll(list); for (const auto &attr : list) { const AttributeVector &v = *attr; - _metricsWireService.addAttribute(_subAttributeMetrics, _totalAttributeMetrics, - v.getName()); + _metricsWireService.addAttribute(_subAttributeMetrics, v.getName()); } } _initAttrMgr = attrMgrResult; @@ -175,11 +174,11 @@ FastAccessDocSubDB::reconfigureAttributeMetrics(const proton::IAttributeManager } for (const auto &attrName : toAdd) { LOG(debug, "reconfigureAttributeMetrics(): addAttribute='%s'", attrName.c_str()); - _metricsWireService.addAttribute(_subAttributeMetrics, _totalAttributeMetrics, attrName); + _metricsWireService.addAttribute(_subAttributeMetrics, attrName); } for (const auto &attrName : toRemove) { LOG(debug, "reconfigureAttributeMetrics(): removeAttribute='%s'", attrName.c_str()); - _metricsWireService.removeAttribute(_subAttributeMetrics, _totalAttributeMetrics, attrName); + _metricsWireService.removeAttribute(_subAttributeMetrics, attrName); } } @@ -203,7 +202,6 @@ FastAccessDocSubDB::FastAccessDocSubDB(const Config &cfg, const Context &ctx) _initAttrMgr(), _fastAccessFeedView(), _subAttributeMetrics(ctx._subAttributeMetrics), - _totalAttributeMetrics(ctx._totalAttributeMetrics), _addMetrics(cfg._addMetrics), _metricsWireService(ctx._metricsWireService), _docIdLimit(0) diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h index 7c7ee3fab56..972c496508a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h @@ -5,7 +5,7 @@ #include "storeonlydocsubdb.h" #include <vespa/searchcore/proton/attribute/attributemanager.h> #include <vespa/searchcore/proton/common/docid_limit.h> -#include <vespa/searchcore/proton/metrics/attribute_metrics_collection.h> +#include <vespa/searchcore/proton/metrics/attribute_metrics.h> #include <vespa/searchcore/proton/metrics/metricswireservice.h> namespace proton { @@ -45,16 +45,13 @@ public: struct Context { const StoreOnlyDocSubDB::Context _storeOnlyCtx; - const AttributeMetricsCollection &_subAttributeMetrics; - LegacyAttributeMetrics *_totalAttributeMetrics; + const AttributeMetrics &_subAttributeMetrics; MetricsWireService &_metricsWireService; Context(const StoreOnlyDocSubDB::Context &storeOnlyCtx, - const AttributeMetricsCollection &subAttributeMetrics, - LegacyAttributeMetrics *totalAttributeMetrics, + const AttributeMetrics &subAttributeMetrics, MetricsWireService &metricsWireService) : _storeOnlyCtx(storeOnlyCtx), _subAttributeMetrics(subAttributeMetrics), - _totalAttributeMetrics(totalAttributeMetrics), _metricsWireService(metricsWireService) { } }; @@ -67,8 +64,7 @@ private: const bool _fastAccessAttributesOnly; AttributeManager::SP _initAttrMgr; Configurer::FeedViewVarHolder _fastAccessFeedView; - AttributeMetricsCollection _subAttributeMetrics; - LegacyAttributeMetrics *_totalAttributeMetrics; + AttributeMetrics _subAttributeMetrics; std::shared_ptr<initializer::InitializerTask> createAttributeManagerInitializer(const DocumentDBConfig &configSnapshot, diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 2174412bedf..09e5a64185a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -90,11 +90,11 @@ size_t deriveCompactionCompressionThreads(const ProtonConfig &proton, const HwInfo::Cpu &cpuInfo) { size_t scaledCores = (size_t)std::ceil(cpuInfo.cores() * proton.feeding.concurrency); - size_t threads = std::max(scaledCores, size_t(proton.summary.log.numthreads)); + size_t threads = std::max(scaledCores, 8ul); // We need at least 1 guaranteed free worker in order to ensure progress so #documentsdbs + 1 should suffice, // but we will not be cheap and give #documentsdbs * 2 - return std::max(threads, proton.documentdb.size() * 2);; + return std::max(threads, proton.documentdb.size() * 2); } const vespalib::string CUSTOM_COMPONENT_API_PATH = "/state/v1/custom/component"; @@ -541,7 +541,7 @@ Proton::addDocumentDB(const document::DocumentType &docType, if (!_isInitializing || _initDocumentDbsInSequence) { ret->waitForOnlineState(); } - _metricsEngine->addDocumentDBMetrics(ret->getMetricsCollection()); + _metricsEngine->addDocumentDBMetrics(ret->getMetrics()); _metricsEngine->addMetricsHook(ret->getMetricsUpdateHook()); _documentDBMap[docTypeName] = ret; if (_persistenceEngine) { @@ -597,7 +597,7 @@ Proton::removeDocumentDB(const DocTypeName &docTypeName) _summaryEngine->removeSearchHandler(docTypeName); _flushEngine->removeFlushHandler(docTypeName); _metricsEngine->removeMetricsHook(old->getMetricsUpdateHook()); - _metricsEngine->removeDocumentDBMetrics(old->getMetricsCollection()); + _metricsEngine->removeDocumentDBMetrics(old->getMetrics()); _diskMemUsageSampler->notifier().removeDiskMemUsageListener(old->diskMemUsageListener()); // Caller should have removed & drained relevant timer tasks old->close(); @@ -659,11 +659,10 @@ int countOpenFiles() } void -updateExecutorMetrics(ExecutorMetrics &metrics, ExecutorMetrics &legacyMetrics, +updateExecutorMetrics(ExecutorMetrics &metrics, const vespalib::ThreadStackExecutor::Stats &stats) { metrics.update(stats); - legacyMetrics.update(stats); } } @@ -690,16 +689,15 @@ Proton::updateMetrics(const vespalib::MonitorGuard &) } { ContentProtonMetrics::ProtonExecutorMetrics &metrics = _metricsEngine->root().executor; - LegacyProtonMetrics &legacyMetrics = _metricsEngine->legacyRoot(); - updateExecutorMetrics(metrics.proton, legacyMetrics.executor, _executor.getStats()); + updateExecutorMetrics(metrics.proton, _executor.getStats()); if (_flushEngine) { - updateExecutorMetrics(metrics.flush, legacyMetrics.flushExecutor, _flushEngine->getExecutorStats()); + updateExecutorMetrics(metrics.flush, _flushEngine->getExecutorStats()); } if (_matchEngine) { - updateExecutorMetrics(metrics.match, legacyMetrics.matchExecutor, _matchEngine->getExecutorStats()); + updateExecutorMetrics(metrics.match, _matchEngine->getExecutorStats()); } if (_summaryEngine) { - updateExecutorMetrics(metrics.docsum, legacyMetrics.summaryExecutor, _summaryEngine->getExecutorStats()); + updateExecutorMetrics(metrics.docsum, _summaryEngine->getExecutorStats()); } if (_sharedExecutor) { metrics.shared.update(_sharedExecutor->getStats()); diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index 91394e429a4..c0c9b961ac8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -10,7 +10,7 @@ #include <vespa/searchcore/proton/flushengine/threadedflushtarget.h> #include <vespa/searchcore/proton/index/index_manager_initializer.h> #include <vespa/searchcore/proton/index/index_writer.h> -#include <vespa/searchcore/proton/metrics/legacy_documentdb_metrics.h> +#include <vespa/searchcore/proton/matching/sessionmanager.h> #include <vespa/searchcore/proton/reference/document_db_reference.h> #include <vespa/searchcore/proton/reference/gid_to_lid_change_handler.h> #include <vespa/searchcorespi/plugin/iindexmanagerfactory.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index b222aab1397..99186f89c6d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -17,7 +17,7 @@ #include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h> #include <vespa/searchcore/proton/flushengine/threadedflushtarget.h> #include <vespa/searchcore/proton/index/index_writer.h> -#include <vespa/searchcore/proton/metrics/documentdb_metrics_collection.h> +#include <vespa/searchcore/proton/matching/sessionmanager.h> #include <vespa/searchcore/proton/metrics/metricswireservice.h> #include <vespa/searchcore/proton/reference/dummy_gid_to_lid_change_handler.h> #include <vespa/searchlib/attribute/configconverter.h> @@ -80,7 +80,7 @@ StoreOnlyDocSubDB::Context::Context(IDocumentSubDBOwner &owner, vespalib::ThreadStackExecutorBase &sharedExecutor, std::shared_ptr<BucketDBOwner> bucketDB, bucketdb::IBucketDBHandlerInitializer & bucketDBHandlerInitializer, - DocumentDBMetricsCollection &metrics, + DocumentDBTaggedMetrics &metrics, std::mutex &configMutex, const HwInfo &hwInfo) : _owner(owner), diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h index db9cbf7415a..d377166226e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h @@ -23,7 +23,7 @@ namespace proton { -class DocumentDBMetricsCollection; +class DocumentDBTaggedMetrics; class DocumentMetaStoreInitializerResult; class FeedHandler; class IDocumentSubDBOwner; @@ -113,7 +113,7 @@ public: vespalib::ThreadStackExecutorBase &_sharedExecutor; std::shared_ptr<BucketDBOwner> _bucketDB; bucketdb::IBucketDBHandlerInitializer &_bucketDBHandlerInitializer; - DocumentDBMetricsCollection &_metrics; + DocumentDBTaggedMetrics &_metrics; std::mutex &_configMutex; const HwInfo &_hwInfo; @@ -126,7 +126,7 @@ public: std::shared_ptr<BucketDBOwner> bucketDB, bucketdb::IBucketDBHandlerInitializer & bucketDBHandlerInitializer, - DocumentDBMetricsCollection &metrics, + DocumentDBTaggedMetrics &metrics, std::mutex &configMutex, const HwInfo &hwInfo); ~Context(); @@ -154,7 +154,7 @@ private: protected: searchcorespi::index::IThreadingService &_writeService; vespalib::ThreadStackExecutorBase &_sharedExecutor; - DocumentDBMetricsCollection &_metrics; + DocumentDBTaggedMetrics &_metrics; vespalib::VarHolder<ISearchHandler::SP> _iSearchView; vespalib::VarHolder<IFeedView::SP> _iFeedView; std::mutex &_configMutex; diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index 39bf1603e58..79c633b9617 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -1294,7 +1294,6 @@ "public final boolean equals(java.lang.Object)", "public final java.lang.String toString()", "public abstract java.lang.StringBuilder toString(java.lang.StringBuilder, com.yahoo.searchlib.rankingexpression.rule.SerializationContext, java.util.Deque, com.yahoo.searchlib.rankingexpression.rule.CompositeNode)", - "public java.lang.String toString(com.yahoo.searchlib.rankingexpression.rule.SerializationContext, java.util.Deque, com.yahoo.searchlib.rankingexpression.rule.CompositeNode)", "public abstract com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public abstract com.yahoo.searchlib.rankingexpression.evaluation.Value evaluate(com.yahoo.searchlib.rankingexpression.evaluation.Context)" ], diff --git a/searchlib/pom.xml b/searchlib/pom.xml index 597d7e665ee..1d7862d541e 100644 --- a/searchlib/pom.xml +++ b/searchlib/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>searchlib</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>searchlib</name> <description>Search library functions.</description> <dependencies> diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Grouping.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Grouping.java index a9e2bc73d6a..c13bde4b633 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Grouping.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Grouping.java @@ -209,7 +209,7 @@ public class Grouping extends Identifiable { * * @param level The level to add. * @return This, to allow chaining. - * @throws NullPointerException If <tt>level</tt> argument is null. + * @throws NullPointerException If <code>level</code> argument is null. */ public Grouping addLevel(GroupingLevel level) { level.getClass(); // throws NullPointerException @@ -231,7 +231,7 @@ public class Grouping extends Identifiable { * * @param root The group to set as root. * @return This, to allow chaining. - * @throws NullPointerException If <tt>root</tt> argument is null. + * @throws NullPointerException If <code>root</code> argument is null. */ public Grouping setRoot(Group root) { root.getClass(); // throws NullPointerException @@ -249,7 +249,7 @@ public class Grouping extends Identifiable { } /** - * <p>Sets whether or not grouping should be forced to execute in a single pass. If false, this <tt>Grouping</tt> + * <p>Sets whether or not grouping should be forced to execute in a single pass. If false, this <code>Grouping</code> * might still execute in a single pass due to other constraints.</p> * * @param forceSinglePass True to force execution in single pass. diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/ForceLoad.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/ForceLoad.java index 1144b01f638..ca53c6133be 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/ForceLoad.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/ForceLoad.java @@ -37,7 +37,6 @@ public class ForceLoad { "BitFunctionNode", "FixedWidthBucketFunctionNode", "RangeBucketPreDefFunctionNode", - "GetYMUMChecksumFunctionNode", "DocumentFieldNode", "NullResultNode", "FunctionNode", diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/GetYMUMChecksumFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/GetYMUMChecksumFunctionNode.java deleted file mode 100644 index 9fb47d0401c..00000000000 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/GetYMUMChecksumFunctionNode.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchlib.expression; - -import com.yahoo.vespa.objects.Deserializer; -import com.yahoo.vespa.objects.ObjectVisitor; -import com.yahoo.vespa.objects.Serializer; - -/** - * This node is a request to retrieve the YMUM checksum of a document. - * - * @author baldersheim - * @author Simon Thoresen Hult - */ -public class GetYMUMChecksumFunctionNode extends DocumentAccessorNode { - - public static final int classId = registerClass(0x4000 + 74, GetYMUMChecksumFunctionNode.class); - private IntegerResultNode result = new IntegerResultNode(0); - - @Override - public ResultNode getResult() { - return result; - } - - @Override - protected int onGetClassId() { - return classId; - } - - @Override - protected void onSerialize(Serializer buf) { - super.onSerialize(buf); - result.serialize(buf); - } - - @Override - protected void onDeserialize(Deserializer buf) { - super.onDeserialize(buf); - result.deserialize(buf); - } - - @Override - public GetYMUMChecksumFunctionNode clone() { - GetYMUMChecksumFunctionNode obj = (GetYMUMChecksumFunctionNode)super.clone(); - if (result != null) { - obj.result = (IntegerResultNode)result.clone(); - } - return obj; - } - - @Override - public void visitMembers(ObjectVisitor visitor) { - super.visitMembers(visitor); - visitor.visit("result", result); - } - - @Override - protected boolean equalsExpression(ExpressionNode obj) { - return equals(result, ((GetYMUMChecksumFunctionNode)obj).result); - } -} diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ExpressionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ExpressionNode.java index c432e49fe2b..dba0da7301d 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ExpressionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ExpressionNode.java @@ -45,20 +45,6 @@ public abstract class ExpressionNode implements Serializable { public abstract StringBuilder toString(StringBuilder builder, SerializationContext context, Deque<String> path, CompositeNode parent); /** - * Returns a script instance of this based on the supplied script functions. - * @deprecated use the faster one that takes and returns a StringBuilder instead. - * - * @param context the serialization context - * @param path the call path to this, used for cycle detection, or null if this is a root - * @param parent the parent node of this, or null if it a root - * @return the main script, referring to script instances. - */ - @Deprecated - public String toString(SerializationContext context, Deque<String> path, CompositeNode parent) { - return toString(new StringBuilder(), context, path, parent).toString(); - } - - /** * Returns the type this will return if evaluated with the given context. * * @param context the variable type bindings to use for this evaluation diff --git a/searchlib/src/test/files/testSpecialNodes b/searchlib/src/test/files/testSpecialNodes Binary files differindex 149cc17f99b..d1a65b16666 100644 --- a/searchlib/src/test/files/testSpecialNodes +++ b/searchlib/src/test/files/testSpecialNodes diff --git a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupingSerializationTest.java b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupingSerializationTest.java index 969bc318391..b9d3bc3bd4d 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupingSerializationTest.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/GroupingSerializationTest.java @@ -72,7 +72,6 @@ public class GroupingSerializationTest { t.assertMatch(new AttributeNode("testattribute")); t.assertMatch(new DocumentFieldNode("testdocumentfield")); t.assertMatch(new GetDocIdNamespaceSpecificFunctionNode(new IntegerResultNode(7))); - t.assertMatch(new GetYMUMChecksumFunctionNode()); } } @@ -363,11 +362,13 @@ public class GroupingSerializationTest { @Override public void close() throws IOException { - boolean moreDataAvailable = in.read() != -1; + int bytesLeft = 0; + while (in.read() != -1) + bytesLeft++; in.close(); - if (moreDataAvailable) { - fail("The file was not fully consumed. Did you forget to deserialize an object on Java side?"); - } + if (bytesLeft > 0) + fail(FILE_PATH + "/" + fileName + " has " + bytesLeft + " bytes left. " + + "Did you forget to deserialize an object on Java side?"); } private static String toHexString(byte[] data) { diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/ExpressionTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/ExpressionTestCase.java index 1fa012c83f7..d282dc17aaa 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/ExpressionTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/ExpressionTestCase.java @@ -401,25 +401,6 @@ public class ExpressionTestCase { } @Test - public void testGetYMUMChecksumFunctionNode() { - GetYMUMChecksumFunctionNode a = new GetYMUMChecksumFunctionNode(); - assertTrue(a.getResult() instanceof IntegerResultNode); - assertSerialize(a); - try { - new GetYMUMChecksumFunctionNode().prepare(); - fail("Should not be able to prepare documentfieldnode"); - } catch (RuntimeException e) { - // expected - } - try { - new GetYMUMChecksumFunctionNode().execute(); - fail("Should not be able to execute documentfieldnode"); - } catch (RuntimeException e) { - // expected - } - } - - @Test public void testIntegerResultNode() { IntegerResultNode a = new IntegerResultNode(7); assertEquals(a.getInteger(), 7); diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp index eb49a556bdc..9e75a85d289 100644 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp @@ -206,7 +206,7 @@ TEST("testGrowing") { LogDataStore::Config config; //(100000, 0.1, 3.0, 0.2, 8, true, CompressionConfig::LZ4, // WriteableFileChunk::Config(CompressionConfig(CompressionConfig::LZ4, 9, 60), 1000)); config.setMaxFileSize(100000).setMaxDiskBloatFactor(0.1).setMaxBucketSpread(3.0).setMinFileSizeFactor(0.2) - .compact2ActiveFile(true).compactCompression({CompressionConfig::LZ4}) + .compactCompression({CompressionConfig::LZ4}) .setFileConfig({{CompressionConfig::LZ4, 9, 60}, 1000}); vespalib::ThreadStackExecutor executor(8, 128*1024); DummyFileHeaderContext fileHeaderContext; @@ -1054,7 +1054,6 @@ TEST("require that config equality operator detects inequality") { EXPECT_FALSE(C() == C().setMinFileSizeFactor(0.3)); EXPECT_FALSE(C() == C().setFileConfig(WriteableFileChunk::Config({}, 70))); EXPECT_FALSE(C() == C().disableCrcOnRead(true)); - EXPECT_FALSE(C() == C().compact2ActiveFile(false)); EXPECT_FALSE(C() == C().compactCompression({CompressionConfig::ZSTD})); } diff --git a/searchlib/src/tests/grouping/grouping_serialization_test.cpp b/searchlib/src/tests/grouping/grouping_serialization_test.cpp index cdf8eb61381..e430559d74e 100644 --- a/searchlib/src/tests/grouping/grouping_serialization_test.cpp +++ b/searchlib/src/tests/grouping/grouping_serialization_test.cpp @@ -146,7 +146,6 @@ TEST_F("testSpecialNodes", Fixture("testSpecialNodes")) { f.checkObject(GetDocIdNamespaceSpecificFunctionNode( ResultNode::UP(new Int64ResultNode(7)))); } - f.checkObject(GetYMUMChecksumFunctionNode()); } TEST_F("testFunctionNodes", Fixture("testFunctionNodes")) { diff --git a/searchlib/src/tests/grouping/grouping_test.cpp b/searchlib/src/tests/grouping/grouping_test.cpp index 084f13795f7..fea18619ef9 100644 --- a/searchlib/src/tests/grouping/grouping_test.cpp +++ b/searchlib/src/tests/grouping/grouping_test.cpp @@ -1930,10 +1930,10 @@ Test::testAttributeMapLookup() ctx.add(StringAttrBuilder("key3").add("k3").add("k2").sp()); testAggregationSimple(ctx, SumAggregationResult(), Int64ResultNode(10 + undefinedInteger), "smap{\"k1\"}.weight"); testAggregationSimple(ctx, SumAggregationResult(), Int64ResultNode(20 + undefinedInteger), "smap{\"k2\"}.weight"); - testAggregationSimple(ctx, SumAggregationResult(), Int64ResultNode(2 * undefinedInteger), "smap{\"k5\"}.weight"); + testAggregationSimple(ctx, SumAggregationResult(), Int64ResultNode(0), "smap{\"k5\"}.weight"); testAggregationSimple(ctx, SumAggregationResult(), Int64ResultNode(210), "smap{attribute(key1)}.weight"); testAggregationSimple(ctx, SumAggregationResult(), Int64ResultNode(120), "smap{attribute(key2)}.weight"); - testAggregationSimple(ctx, SumAggregationResult(), Int64ResultNode(2 * undefinedInteger), "smap{attribute(key3)}.weight"); + testAggregationSimple(ctx, SumAggregationResult(), Int64ResultNode(0), "smap{attribute(key3)}.weight"); testAggregationSimple(ctx, MinAggregationResult(), Int64ResultNode(10), "smap{attribute(key1)}.weight"); testAggregationSimple(ctx, MinAggregationResult(), Int64ResultNode(20), "smap{attribute(key2)}.weight"); testAggregationSimple(ctx, MaxAggregationResult(), Int64ResultNode(200), "smap{attribute(key1)}.weight"); diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp index 148d77738f9..590764bf469 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp @@ -35,7 +35,6 @@ LogDataStore::Config::Config() _maxBucketSpread(2.5), _minFileSizeFactor(0.2), _skipCrcOnRead(false), - _compact2ActiveFile(true), _compactCompression(CompressionConfig::LZ4), _fileConfig() { } @@ -46,7 +45,6 @@ LogDataStore::Config::operator == (const Config & rhs) const { (_maxDiskBloatFactor == rhs._maxDiskBloatFactor) && (_maxFileSize == rhs._maxFileSize) && (_minFileSizeFactor == rhs._minFileSizeFactor) && - (_compact2ActiveFile == rhs._compact2ActiveFile) && (_skipCrcOnRead == rhs._skipCrcOnRead) && (_compactCompression == rhs._compactCompression) && (_fileConfig == rhs._fileConfig); @@ -449,8 +447,7 @@ void LogDataStore::flushActiveAndWait(SerialNum syncToken) { } bool LogDataStore::shouldCompactToActiveFile(size_t compactedSize) const { - return _config.compact2ActiveFile() - || (_config.getMinFileSizeFactor() * _config.getMaxFileSize() > compactedSize); + return (_config.getMinFileSizeFactor() * _config.getMaxFileSize() > compactedSize); } void LogDataStore::setNewFileChunk(const LockGuard & guard, FileChunk::UP file) @@ -748,9 +745,7 @@ LogDataStore::verifyModificationTime(const NameIdSet & partList) if ( ! FastOS_File::Stat(idxName.c_str(), &idxStat)) { throw runtime_error(make_string("Failed to Stat '%s'\nDirectory =\n%s", idxName.c_str(), ls(partList).c_str())); } - ns_log::Logger::LogLevel logLevel = _config.compact2ActiveFile() - ? ns_log::Logger::warning - : ns_log::Logger::debug; + ns_log::Logger::LogLevel logLevel = ns_log::Logger::debug; if ((datStat._modifiedTimeNS < prevDatStat._modifiedTimeNS) && hasNonHeaderData(datName)) { VLOG(logLevel, "Older file '%s' is newer (%ld) than file '%s' (%ld)\nDirectory =\n%s", prevDatNam.c_str(), prevDatStat._modifiedTimeNS, diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.h b/searchlib/src/vespa/searchlib/docstore/logdatastore.h index 52d78cac2bd..91143ecd1d2 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.h +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.h @@ -52,12 +52,10 @@ public: double getMinFileSizeFactor() const { return _minFileSizeFactor; } bool crcOnReadDisabled() const { return _skipCrcOnRead; } - bool compact2ActiveFile() const { return _compact2ActiveFile; } const CompressionConfig & compactCompression() const { return _compactCompression; } const WriteableFileChunk::Config & getFileConfig() const { return _fileConfig; } Config & disableCrcOnRead(bool v) { _skipCrcOnRead = v; return *this;} - Config & compact2ActiveFile(bool v) { _compact2ActiveFile = v; return *this; } bool operator == (const Config &) const; private: @@ -66,7 +64,6 @@ public: double _maxBucketSpread; double _minFileSizeFactor; bool _skipCrcOnRead; - bool _compact2ActiveFile; CompressionConfig _compactCompression; WriteableFileChunk::Config _fileConfig; }; diff --git a/searchsummary/pom.xml b/searchsummary/pom.xml index 417e25ff944..cb110f107a9 100644 --- a/searchsummary/pom.xml +++ b/searchsummary/pom.xml @@ -7,11 +7,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>searchsummary</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp index a188ab6e60a..4e3540fb573 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp @@ -3,6 +3,7 @@ #include "docsumwriter.h" #include "docsumstate.h" #include "docsum_field_writer_state.h" +#include <vespa/searchcommon/common/undefinedvalues.h> #include <vespa/searchlib/common/transport.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchlib/attribute/iattributemanager.h> @@ -78,8 +79,14 @@ DynamicDocsumWriter::resolveInputClass(ResolveClassInfo &rci, uint32_t id) const } } -static void convertEntry(GetDocsumsState *state, const ResConfigEntry *resCfg, const ResEntry *entry, - Inserter &inserter, Slime &slime) +constexpr uint32_t default_32bits_int = search::attribute::getUndefined<int32_t>(); +constexpr uint64_t default_64bits_int = search::attribute::getUndefined<int64_t>(); + +static void convertEntry(GetDocsumsState *state, + const ResConfigEntry *resCfg, + const ResEntry *entry, + Inserter &inserter, + Slime &slime) { using vespalib::slime::BinaryFormat; const char *ptr; @@ -90,30 +97,40 @@ static void convertEntry(GetDocsumsState *state, const ResConfigEntry *resCfg, c case RES_INT: case RES_SHORT: case RES_BYTE: - inserter.insertLong(entry->_intval); + if (entry->_intval != default_32bits_int) { + inserter.insertLong(entry->_intval); + } break; case RES_BOOL: inserter.insertBool(entry->_intval != 0); break; case RES_FLOAT: case RES_DOUBLE: - inserter.insertDouble(entry->_doubleval); + if (! std::isnan(entry->_doubleval)) { + inserter.insertDouble(entry->_doubleval); + } break; case RES_INT64: - inserter.insertLong(entry->_int64val); + if (entry->_int64val != default_64bits_int) { + inserter.insertLong(entry->_int64val); + } break; case RES_STRING: case RES_LONG_STRING: case RES_FEATUREDATA: case RES_XMLSTRING: entry->_resolve_field(&ptr, &len, &state->_docSumFieldSpace); - inserter.insertString(Memory(ptr, len)); + if (len != 0) { + inserter.insertString(Memory(ptr, len)); + } break; case RES_DATA: case RES_TENSOR: case RES_LONG_DATA: entry->_resolve_field(&ptr, &len, &state->_docSumFieldSpace); - inserter.insertData(Memory(ptr, len)); + if (len != 0) { + inserter.insertData(Memory(ptr, len)); + } break; case RES_JSONSTRING: entry->_resolve_field(&ptr, &len, &state->_docSumFieldSpace); @@ -162,10 +179,10 @@ DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, uint32_t docid, const Memory field_name(outCfg->_bindname.data(), outCfg->_bindname.size()); ObjectInserter inserter(docsum, field_name); if (writer != nullptr) { - //TODO: Need to add test for writer->isDefaultValue - writer->insertField(docid, &gres, state, outCfg->_type, inserter); + if (! writer->isDefaultValue(docid, state)) { + writer->insertField(docid, &gres, state, outCfg->_type, inserter); + } } else { - //TODO: Need to add similar test as writer->isDefaultValue if (rci.inputClass == rci.outputClass) { convertEntry(state, outCfg, gres.GetEntry(i), inserter, slime); } else { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp index 9c82c00c3ef..6b003553f49 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp @@ -3,8 +3,11 @@ #include "positionsdfw.h" #include "docsumstate.h" #include <vespa/searchlib/attribute/iattributemanager.h> +#include <vespa/searchcommon/attribute/attributecontent.h> #include <vespa/searchlib/common/location.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/data/slime/cursor.h> +#include <vespa/vespalib/data/slime/inserter.h> #include <cmath> #include <climits> @@ -16,6 +19,7 @@ namespace search::docsummary { using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; using search::attribute::BasicType; +using search::attribute::IntegerContent; using search::common::Location; AbsDistanceDFW::AbsDistanceDFW(const vespalib::string & attrName) : @@ -31,13 +35,9 @@ AbsDistanceDFW::findMinDistance(uint32_t docid, GetDocsumsState *state) uint64_t absdist = std::numeric_limits<int64_t>::max(); int32_t docx = 0; int32_t docy = 0; - std::vector<IAttributeVector::largeint_t> pos(16); - uint32_t numValues = attribute.get(docid, &pos[0], pos.size()); - if (numValues > pos.size()) { - pos.resize(numValues); - numValues = attribute.get(docid, &pos[0], pos.size()); - assert(numValues <= pos.size()); - } + IntegerContent pos; + pos.fill(attribute, docid); + uint32_t numValues = pos.size(); for (uint32_t i = 0; i < numValues; i++) { int64_t docxy(pos[i]); vespalib::geo::ZCurve::decode(docxy, &docx, &docy); @@ -112,19 +112,72 @@ PositionsDFW::PositionsDFW(const vespalib::string & attrName) : } namespace { + +void +insertPos(int64_t docxy, vespalib::slime::Inserter &target) +{ + + int32_t docx = 0; + int32_t docy = 0; + vespalib::geo::ZCurve::decode(docxy, &docx, &docy); + if (docx == 0 && docy == INT_MIN) { + LOG(spam, "skipping empty zcurve value"); + return; + } + vespalib::slime::Cursor &obj = target.insertObject(); + obj.setLong("y", docy); + obj.setLong("x", docx); + + double degrees_ns = docy; + degrees_ns /= 1000000.0; + double degrees_ew = docx; + degrees_ew /= 1000000.0; + + vespalib::asciistream latlong; + latlong << vespalib::FloatSpec::fixed; + if (degrees_ns < 0) { + latlong << "S" << (-degrees_ns); + } else { + latlong << "N" << degrees_ns; + } + latlong << ";"; + if (degrees_ew < 0) { + latlong << "W" << (-degrees_ew); + } else { + latlong << "E" << degrees_ew; + } + obj.setString("latlong", vespalib::Memory(latlong.str())); +} + +void +insertFromAttr(const attribute::IAttributeVector &attribute, uint32_t docid, vespalib::slime::Inserter &target) +{ + IntegerContent pos; + pos.fill(attribute, docid); + uint32_t numValues = pos.size(); + LOG(debug, "docid=%d, numValues=%d", docid, numValues); + if (numValues > 0) { + if (attribute.getCollectionType() == attribute::CollectionType::SINGLE) { + insertPos(pos[0], target); + } else { + vespalib::slime::Cursor &arr = target.insertArray(); + for (uint32_t i = 0; i < numValues; i++) { + vespalib::slime::ArrayInserter ai(arr); + insertPos(pos[i], ai); + } + } + } +} + vespalib::asciistream formatField(const attribute::IAttributeVector &attribute, uint32_t docid, ResType type) { vespalib::asciistream target; int32_t docx = 0; int32_t docy = 0; - std::vector<IAttributeVector::largeint_t> pos(16); - uint32_t numValues = attribute.get(docid, &pos[0], pos.size()); - if (numValues > pos.size()) { - pos.resize(numValues); - numValues = attribute.get(docid, &pos[0], pos.size()); - assert(numValues <= pos.size()); - } + IntegerContent pos; + pos.fill(attribute, docid); + uint32_t numValues = pos.size(); LOG(debug, "docid=%d, numValues=%d", docid, numValues); bool isShort = !IDocsumFieldWriter::IsBinaryCompatible(type, RES_LONG_STRING); @@ -167,6 +220,10 @@ formatField(const attribute::IAttributeVector &attribute, uint32_t docid, ResTyp void PositionsDFW::insertField(uint32_t docid, GetDocsumsState * dsState, ResType type, vespalib::slime::Inserter &target) { + if (type == RES_XMLSTRING) { + insertFromAttr(vec(*dsState), docid, target); + return; + } vespalib::asciistream val(formatField(vec(*dsState), docid, type)); target.insertString(vespalib::Memory(val.c_str(), val.size())); } diff --git a/security-utils/pom.xml b/security-utils/pom.xml index 450428c5c14..0a26c73cf70 100644 --- a/security-utils/pom.xml +++ b/security-utils/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>security-utils</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <!-- provided --> <dependency> diff --git a/security-utils/src/main/java/com/yahoo/security/X509CertificateBuilder.java b/security-utils/src/main/java/com/yahoo/security/X509CertificateBuilder.java index 54d7d39253e..4452a4a7604 100644 --- a/security-utils/src/main/java/com/yahoo/security/X509CertificateBuilder.java +++ b/security-utils/src/main/java/com/yahoo/security/X509CertificateBuilder.java @@ -23,9 +23,9 @@ import java.security.PrivateKey; import java.security.PublicKey; import java.security.SecureRandom; import java.security.cert.X509Certificate; -import java.sql.Date; import java.time.Instant; import java.util.ArrayList; +import java.util.Date; import java.util.List; import static com.yahoo.security.SubjectAlternativeName.Type.DNS_NAME; diff --git a/service-monitor/README b/service-monitor/README index 4d69c8ef3be..26f94fb8026 100644 --- a/service-monitor/README +++ b/service-monitor/README @@ -1 +1,2 @@ -a service that gives a list of node/service and their status(up/down) for all instances in a zone +A service which provides the up/down status of all service instances ("nodes in a cluster") +in all applications in a system. diff --git a/service-monitor/pom.xml b/service-monitor/pom.xml index 168af826b40..c6504ff9315 100644 --- a/service-monitor/pom.xml +++ b/service-monitor/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>service-monitor</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>${project.artifactId}</name> <description>Service monitor component for hosted vespa.</description> diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ZoneApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ZoneApplication.java index 70354c0f16d..bcf5f096e7f 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ZoneApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/ZoneApplication.java @@ -21,6 +21,7 @@ import java.util.Objects; * 2) ZoneApplication has multiple clusters */ public class ZoneApplication { + private ZoneApplication() {} private static final ApplicationId ZONE_APPLICATION_ID = InfraApplication @@ -103,4 +104,5 @@ public class ZoneApplication { ApplicationInstanceGenerator.getClusterId(serviceInfo), ApplicationInstanceGenerator.toServiceType(serviceInfo)); } + } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java index a9071e3ab94..b62552188e1 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java @@ -17,8 +17,8 @@ import java.util.stream.Collectors; * also gives ServiceStatus on each service, and there may be * artificial applications like the config server "application". */ -// @Immutable public class ServiceModel { + private final Map<ApplicationInstanceReference, ApplicationInstance> applications; public ServiceModel(Map<ApplicationInstanceReference, ApplicationInstance> applications) { @@ -39,4 +39,5 @@ public class ServiceModel { .flatMap(cluster -> cluster.serviceInstances().stream()) .collect(Collectors.groupingBy(ServiceInstance::hostName, Collectors.toList())); } + } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java index ab32af39580..5ed34673da5 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java @@ -21,4 +21,5 @@ public interface ServiceMonitor { Map<ApplicationInstanceReference, ApplicationInstance> getAllApplicationInstances(); ServiceModel getServiceModelSnapshot(); + } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceStatusProvider.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceStatusProvider.java index 88c72a7d47a..30a20cf9980 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceStatusProvider.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceStatusProvider.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.applicationmodel.ServiceType; * @author hakonhall */ public interface ServiceStatusProvider { + /** * Get the {@link ServiceStatus} of a particular service. * @@ -22,4 +23,5 @@ public interface ServiceStatusProvider { ClusterId clusterId, ServiceType serviceType, ConfigId configId); + } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokApi.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokApi.java index dff605b888d..b0f0d734fe4 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokApi.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/SlobrokApi.java @@ -7,9 +7,11 @@ import com.yahoo.jrt.slobrok.api.Mirror; import java.util.List; public interface SlobrokApi extends ServiceStatusProvider { + /** * Get all Slobrok entries that has a name matching pattern as described in * Mirror::lookup. */ List<Mirror.Entry> lookup(ApplicationId application, String pattern); + } diff --git a/serviceview/pom.xml b/serviceview/pom.xml index 4357cc697b8..1d1ad765d89 100644 --- a/serviceview/pom.xml +++ b/serviceview/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>serviceview</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>serviceview</name> <description>REST API that is shared between the Vespa Config Server and others.</description> <dependencies> diff --git a/simplemetrics/pom.xml b/simplemetrics/pom.xml index 374ea77a2eb..f65aeaabef1 100644 --- a/simplemetrics/pom.xml +++ b/simplemetrics/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>simplemetrics</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <description> </description> <dependencies> <dependency> diff --git a/socket_test/pom.xml b/socket_test/pom.xml index 0c6d5cd77f4..402fe812af2 100644 --- a/socket_test/pom.xml +++ b/socket_test/pom.xml @@ -7,11 +7,11 @@ <modelVersion>4.0.0</modelVersion> <groupId>com.yahoo.vespa</groupId> <artifactId>socket_test</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <packaging>jar</packaging> diff --git a/standalone-container/pom.xml b/standalone-container/pom.xml index 73d05d35df7..744d6854ab2 100644 --- a/standalone-container/pom.xml +++ b/standalone-container/pom.xml @@ -6,11 +6,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>standalone-container</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> diff --git a/standalone-container/src/main/sh/standalone-container.sh b/standalone-container/src/main/sh/standalone-container.sh index 9d74f96bb03..3398aba28f8 100755 --- a/standalone-container/src/main/sh/standalone-container.sh +++ b/standalone-container/src/main/sh/standalone-container.sh @@ -160,9 +160,14 @@ StartCommand() { -XX:+HeapDumpOnOutOfMemoryError \ -XX:HeapDumpPath="$VESPA_HOME/var/crash" \ -XX:+ExitOnOutOfMemoryError \ + --illegal-access=warn \ + --add-opens=java.base/java.io=ALL-UNNAMED \ + --add-opens=java.base/java.lang=ALL-UNNAMED \ + --add-opens=java.base/java.net=ALL-UNNAMED \ + --add-opens=java.base/jdk.internal.loader=ALL-UNNAMED \ -Djava.library.path="$VESPA_HOME/lib64" \ -Djava.awt.headless=true \ - -Dsun.rmi.dgc.client.gcInterval=3600000 \ + -Dsun.rmi.dgc.client.gcInterval=3600000 \ -Dsun.net.client.defaultConnectTimeout=5000 \ -Dsun.net.client.defaultReadTimeout=60000 \ -Djavax.net.ssl.keyStoreType=JKS \ diff --git a/statistics/pom.xml b/statistics/pom.xml index f164f3ba1eb..8206d97ad34 100644 --- a/statistics/pom.xml +++ b/statistics/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>statistics</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>jar</packaging> <dependencies> <dependency> diff --git a/storage/pom.xml b/storage/pom.xml index e0b21d5a8a9..ba47fe84eae 100644 --- a/storage/pom.xml +++ b/storage/pom.xml @@ -7,11 +7,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>storage</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/storage/src/tests/distributor/bucketdbupdatertest.cpp b/storage/src/tests/distributor/bucketdbupdatertest.cpp index 56f88b7f98f..53f80854bef 100644 --- a/storage/src/tests/distributor/bucketdbupdatertest.cpp +++ b/storage/src/tests/distributor/bucketdbupdatertest.cpp @@ -61,7 +61,6 @@ class BucketDBUpdaterTest : public CppUnit::TestFixture, CPPUNIT_TEST(testDistributorChangeWithGrouping); CPPUNIT_TEST(testNormalUsageInitializing); // Check that we send request bucket info when storage node is initializing, and send another when it's up. CPPUNIT_TEST(testFailedRequestBucketInfo); - CPPUNIT_TEST(testEncodeErrorHandling); CPPUNIT_TEST(testBitChange); // Check what happens when distribution bits change CPPUNIT_TEST(testNodeDown); CPPUNIT_TEST(testStorageNodeInMaintenanceClearsBucketsForNode); @@ -123,7 +122,6 @@ protected: void testDistributorChangeWithGrouping(); void testNormalUsageInitializing(); void testFailedRequestBucketInfo(); - void testEncodeErrorHandling(); void testNoResponses(); void testBitChange(); void testInconsistentChecksum(); @@ -811,29 +809,6 @@ BucketDBUpdaterTest::testFailedRequestBucketInfo() } void -BucketDBUpdaterTest::testEncodeErrorHandling() -{ - setSystemState(lib::ClusterState("distributor:1 .0.s:i storage:1")); - - CPPUNIT_ASSERT_EQUAL(_bucketSpaces.size(), _sender.commands.size()); - - // Not yet passing on system state. - CPPUNIT_ASSERT_EQUAL(size_t(0), _senderDown.commands.size()); - for (uint32_t i = 0; i < _bucketSpaces.size(); ++i) { - std::shared_ptr<api::RequestBucketInfoReply> reply = - getFakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"), - *((RequestBucketInfoCommand*)_sender.commands[i].get()), - 0, - 10); - - reply->setResult(api::ReturnCode::ENCODE_ERROR); - getBucketDBUpdater().onRequestBucketInfoReply(reply); - } - CPPUNIT_ASSERT_EQUAL(std::string("Set system state"), - _senderDown.getCommands()); -} - -void BucketDBUpdaterTest::testDownWhileInit() { setStorageNodes(3); diff --git a/storage/src/tests/storageserver/communicationmanagertest.cpp b/storage/src/tests/storageserver/communicationmanagertest.cpp index 3b72585b076..6af2733f3b9 100644 --- a/storage/src/tests/storageserver/communicationmanagertest.cpp +++ b/storage/src/tests/storageserver/communicationmanagertest.cpp @@ -29,7 +29,7 @@ struct CommunicationManagerTest : public CppUnit::TestFixture { void testRepliesAreDequeuedInFifoOrder(); void bucket_space_config_can_be_updated_live(); void unmapped_bucket_space_documentapi_request_returns_error_reply(); - void unmapped_bucket_space_for_get_documentapi_request_returns_empty_reply(); + void unmapped_bucket_space_for_get_documentapi_request_returns_error_reply(); static constexpr uint32_t MESSAGE_WAIT_TIME_SEC = 60; @@ -54,7 +54,7 @@ struct CommunicationManagerTest : public CppUnit::TestFixture { CPPUNIT_TEST(testRepliesAreDequeuedInFifoOrder); CPPUNIT_TEST(bucket_space_config_can_be_updated_live); CPPUNIT_TEST(unmapped_bucket_space_documentapi_request_returns_error_reply); - CPPUNIT_TEST(unmapped_bucket_space_for_get_documentapi_request_returns_empty_reply); + CPPUNIT_TEST(unmapped_bucket_space_for_get_documentapi_request_returns_error_reply); CPPUNIT_TEST_SUITE_END(); }; @@ -349,11 +349,7 @@ void CommunicationManagerTest::unmapped_bucket_space_documentapi_request_returns CPPUNIT_ASSERT_EQUAL(uint64_t(1), f.comm_mgr->metrics().bucketSpaceMappingFailures.getValue()); } -// Legacy DocumentAPI routing protocols will send Gets to _all_ clusters even -// if they do not contain a particular document type. By sending an empty reply -// we signal a mergeable "not found" to the sender rather than a non-mergeable -// fatal error. -void CommunicationManagerTest::unmapped_bucket_space_for_get_documentapi_request_returns_empty_reply() { +void CommunicationManagerTest::unmapped_bucket_space_for_get_documentapi_request_returns_error_reply() { CommunicationManagerFixture f; BucketspacesConfigBuilder config; @@ -363,9 +359,9 @@ void CommunicationManagerTest::unmapped_bucket_space_for_get_documentapi_request f.comm_mgr->handleMessage(f.documentapi_get_message_for_space("fluff")); CPPUNIT_ASSERT_EQUAL(size_t(1), f.reply_handler.replies.size()); auto& reply = *f.reply_handler.replies[0]; - CPPUNIT_ASSERT(!reply.hasErrors()); - auto& get_reply = dynamic_cast<documentapi::GetDocumentReply&>(reply); - CPPUNIT_ASSERT(!get_reply.hasDocument()); + CPPUNIT_ASSERT(reply.hasErrors()); + CPPUNIT_ASSERT_EQUAL(static_cast<uint32_t>(api::ReturnCode::REJECTED), reply.getError(0).getCode()); + CPPUNIT_ASSERT_EQUAL(uint64_t(1), f.comm_mgr->metrics().bucketSpaceMappingFailures.getValue()); } } // storage diff --git a/storage/src/tests/visiting/visitortest.cpp b/storage/src/tests/visiting/visitortest.cpp index 11e5e355f0c..46b0f4d830d 100644 --- a/storage/src/tests/visiting/visitortest.cpp +++ b/storage/src/tests/visiting/visitortest.cpp @@ -30,10 +30,6 @@ using msg_ptr_vector = std::vector<api::StorageMessage::SP>; struct TestParams { - TestParams& iteratorsPerBucket(uint32_t n) { - _iteratorsPerBucket = n; - return *this; - } TestParams& maxVisitorMemoryUsage(uint32_t bytes) { _maxVisitorMemoryUsage = bytes; return *this; @@ -47,7 +43,6 @@ struct TestParams return *this; } - uint32_t _iteratorsPerBucket {1}; uint32_t _maxVisitorMemoryUsage {UINT32_MAX}; uint32_t _parallelBuckets {1}; mbus::Error _autoReplyError; @@ -62,7 +57,6 @@ private: CPPUNIT_TEST(testNormalUsage); CPPUNIT_TEST(testFailedCreateIterator); CPPUNIT_TEST(testFailedGetIter); - CPPUNIT_TEST(iterators_per_bucket_config_is_ignored_and_hardcoded_to_1); CPPUNIT_TEST(testDocumentAPIClientError); CPPUNIT_TEST(testNoDocumentAPIResendingForFailedVisitor); CPPUNIT_TEST(testIteratorCreatedForFailedVisitor); @@ -90,7 +84,6 @@ public: void testNormalUsage(); void testFailedCreateIterator(); void testFailedGetIter(); - void iterators_per_bucket_config_is_ignored_and_hardcoded_to_1(); void testDocumentAPIClientError(); void testNoDocumentAPIResendingForFailedVisitor(); void testIteratorCreatedForFailedVisitor(); @@ -184,9 +177,6 @@ VisitorTest::initializeTest(const TestParams& params) vdstestlib::DirConfig config(getStandardConfig(true, "visitortest")); config.getConfig("stor-visitor").set("visitorthreads", "1"); config.getConfig("stor-visitor").set( - "iterators_per_bucket", - std::to_string(params._iteratorsPerBucket)); - config.getConfig("stor-visitor").set( "defaultparalleliterators", std::to_string(params._parallelBuckets)); config.getConfig("stor-visitor").set( @@ -596,33 +586,6 @@ VisitorTest::testFailedGetIter() CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); } -void VisitorTest::iterators_per_bucket_config_is_ignored_and_hardcoded_to_1() { - initializeTest(TestParams().iteratorsPerBucket(20)); - auto cmd = makeCreateVisitor(); - _top->sendDown(cmd); - sendCreateIteratorReply(); - - auto getIterCmd = fetchSingleCommand<GetIterCommand>(*_bottom); - CPPUNIT_ASSERT_EQUAL(spi::IteratorId(1234), - getIterCmd->getIteratorId()); - sendGetIterReply(*getIterCmd); - - CPPUNIT_ASSERT_EQUAL(size_t(0), _bottom->getNumCommands()); - - std::vector<document::Document::SP> docs; - std::vector<document::DocumentId> docIds; - std::vector<std::string> infoMessages; - getMessagesAndReply(_documents.size(), getSession(0), docs, docIds, infoMessages); - CPPUNIT_ASSERT_EQUAL(size_t(0), infoMessages.size()); - CPPUNIT_ASSERT_EQUAL(size_t(0), docIds.size()); - - auto destroyIterCmd = fetchSingleCommand<DestroyIteratorCommand>(*_bottom); - - verifyCreateVisitorReply(api::ReturnCode::OK); - CPPUNIT_ASSERT(waitUntilNoActiveVisitors()); - CPPUNIT_ASSERT_EQUAL(0L, getFailedVisitorDestinationReplyCount()); -} - void VisitorTest::testDocumentAPIClientError() { @@ -709,7 +672,7 @@ VisitorTest::testNoDocumentAPIResendingForFailedVisitor() void VisitorTest::testIteratorCreatedForFailedVisitor() { - initializeTest(TestParams().iteratorsPerBucket(1).parallelBuckets(2)); + initializeTest(TestParams().parallelBuckets(2)); std::shared_ptr<api::CreateVisitorCommand> cmd( makeCreateVisitor()); cmd->addBucketToBeVisited(document::BucketId(16, 4)); @@ -928,8 +891,7 @@ void VisitorTest::testNoMoreIteratorsSentWhileMemoryUsedAboveLimit() { initializeTest(TestParams().maxVisitorMemoryUsage(1) - .parallelBuckets(1) - .iteratorsPerBucket(1)); + .parallelBuckets(1)); std::shared_ptr<api::CreateVisitorCommand> cmd( makeCreateVisitor()); _top->sendDown(cmd); diff --git a/storage/src/vespa/storage/config/stor-server.def b/storage/src/vespa/storage/config/stor-server.def index fbc29e0234b..36ca88dd7de 100644 --- a/storage/src/vespa/storage/config/stor-server.def +++ b/storage/src/vespa/storage/config/stor-server.def @@ -13,11 +13,6 @@ cluster_name string default="storage" restart ## to identify the node, and to decide what data should be on it. node_index int default=0 restart -## The maximum amount of memory to use in the storage node. -## Currently default is 2 GB. -## DEPRECATED! -memorytouse long default=2147483647 restart - ## Set whether this is a distributor or a storage node. This will decide what ## storage links are set up. is_distributor bool restart diff --git a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp index 471ce7e2b27..1996ae9d2af 100644 --- a/storage/src/vespa/storage/distributor/pendingclusterstate.cpp +++ b/storage/src/vespa/storage/distributor/pendingclusterstate.cpp @@ -233,12 +233,7 @@ PendingClusterState::onRequestBucketInfoReply(const std::shared_ptr<api::Request const BucketSpaceAndNode bucketSpaceAndNode = iter->second; api::ReturnCode result(reply->getResult()); - if (result == api::ReturnCode::Result::ENCODE_ERROR) { - // Handle failure to encode bucket space due to use of old storage api - // protocol. Pretend that request succeeded with no buckets returned. - // TODO remove this workaround for Vespa 7 - LOG(debug, "Got ENCODE_ERROR, pretending success with no buckets"); - } else if (!result.success()) { + if (!result.success()) { framework::MilliSecTime resendTime(_clock); resendTime += framework::MilliSecTime(100); _delayedRequests.emplace_back(resendTime, bucketSpaceAndNode); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index e7323d07480..76e04852178 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -34,8 +34,7 @@ FileStorManager(const config::ConfigUri & configUri, const spi::PartitionStateLi _partitions(partitions), _providerCore(provider), _providerErrorWrapper(_providerCore), - _providerMetric(new spi::MetricPersistenceProvider(_providerErrorWrapper)), - _provider(_providerMetric.get()), + _provider(&_providerErrorWrapper), _bucketIdFactory(_component.getBucketIdFactory()), _configUri(configUri), _disks(), @@ -47,7 +46,6 @@ FileStorManager(const config::ConfigUri & configUri, const spi::PartitionStateLi _threadMonitor(), _closed(false) { - _metrics->registerMetric(*_providerMetric), _configFetcher.subscribe(_configUri.getConfigId(), this); _configFetcher.start(); _component.registerMetric(*_metrics); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h index 5c52e6c6a23..8c7e206237b 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h @@ -14,7 +14,6 @@ #include <vespa/vespalib/util/sync.h> #include <vespa/document/bucket/bucketid.h> #include <vespa/persistence/spi/persistenceprovider.h> -#include <vespa/persistence/spi/metricpersistenceprovider.h> #include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storage/common/messagesender.h> #include <vespa/storage/common/servicelayercomponent.h> @@ -56,7 +55,6 @@ class FileStorManager : public StorageLinkQueued, const spi::PartitionStateList& _partitions; spi::PersistenceProvider& _providerCore; ProviderErrorWrapper _providerErrorWrapper; - spi::MetricPersistenceProvider::UP _providerMetric; spi::PersistenceProvider* _provider; const document::BucketIdFactory& _bucketIdFactory; diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp index 5a1c2aed113..7fb85ef0ecc 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp +++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp @@ -259,16 +259,8 @@ void CommunicationManager::fail_with_unresolvable_bucket_space( LOG(debug, "Could not map DocumentAPI message to internal bucket: %s", error_message.c_str()); MBUS_TRACE(msg->getTrace(), 6, "Communication manager: Failing message as its document type has no known bucket space mapping"); std::unique_ptr<mbus::Reply> reply; - if (msg->getType() == documentapi::DocumentProtocol::MESSAGE_GETDOCUMENT) { - // HACK: to avoid breaking legacy routing of GetDocumentMessages to _all_ clusters - // regardless of them having a document type or not, we remap missing bucket spaces - // to explicit Not Found replies (empty document GetDocumentReply). - // TODO remove this workaround for Vespa 7 - reply = std::make_unique<documentapi::GetDocumentReply>(std::shared_ptr<document::Document>()); - } else { - reply = std::make_unique<mbus::EmptyReply>(); - reply->addError(mbus::Error(documentapi::DocumentProtocol::ERROR_REJECTED, error_message)); - } + reply = std::make_unique<mbus::EmptyReply>(); + reply->addError(mbus::Error(documentapi::DocumentProtocol::ERROR_REJECTED, error_message)); msg->swapState(*reply); _metrics.bucketSpaceMappingFailures.inc(); _messageBusSession->reply(std::move(reply)); diff --git a/storage/src/vespa/storage/visiting/stor-visitor.def b/storage/src/vespa/storage/visiting/stor-visitor.def index 6f16bcb60a2..72b3699fe2d 100644 --- a/storage/src/vespa/storage/visiting/stor-visitor.def +++ b/storage/src/vespa/storage/visiting/stor-visitor.def @@ -17,16 +17,6 @@ ignorenonexistingvisitortimelimit int default=300 restart ## 100 buckets, 8 of them will be visited in parallel. defaultparalleliterators int default=8 -## The number of iterators we send for each bucket being visited from visitor -## thread. For streaming search we would likely want two or three. Since -## supporting more than one is a new feature, default is still one. -## (If you visit 8 buckets in parallel and have 2 iterators per bucket, this -## will be 16 requests to persistence layer, but only 8 will be able to execute -## at the same time, since only one operation can be executed at the same time -## for one bucket) -## DEPRECATED: ignored by backend, 1 is always used. -iterators_per_bucket int default=1 - ## Default number of maximum client replies pending. defaultpendingmessages int default=32 diff --git a/storageapi/src/vespa/storageapi/message/persistence.h b/storageapi/src/vespa/storageapi/message/persistence.h index 6e93c0e7c05..1fdd5c369bf 100644 --- a/storageapi/src/vespa/storageapi/message/persistence.h +++ b/storageapi/src/vespa/storageapi/message/persistence.h @@ -94,7 +94,6 @@ public: Timestamp getTimestamp() const { return _timestamp; }; Timestamp getUpdateTimestamp() const { return _updateTimestamp; } - bool isHeadersOnlyPut() const { return (_updateTimestamp != 0); } bool wasFound() const { return _wasFound; } void print(std::ostream& out, bool verbose, const std::string& indent) const override; diff --git a/tenant-base/pom.xml b/tenant-base/pom.xml index a29302f9ab4..4326323e63f 100644 --- a/tenant-base/pom.xml +++ b/tenant-base/pom.xml @@ -7,7 +7,7 @@ <groupId>com.yahoo.vespa</groupId> <artifactId>tenant-base</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>Hosted Vespa tenant base</name> <description>Parent POM for all hosted Vespa applications.</description> <url>https://github.com/vespa-engine</url> @@ -33,8 +33,8 @@ <properties> <vespaversion>${project.version}</vespaversion> - <test_framework_version>${project.version}</test_framework_version> - <target_jdk_version>1.8</target_jdk_version> + <test_framework_version>${vespaversion}</test_framework_version> + <target_jdk_version>11</target_jdk_version> <compiler_plugin_version>3.8.0</compiler_plugin_version> <surefire_version>2.22.0</surefire_version> <!-- NOTE bjorncs 15.06.2017: Version 2.20 has OoM issues --> </properties> @@ -316,7 +316,7 @@ <configuration> <rules> <requireJavaVersion> - <version>[1.8, 9)</version> + <version>[9, )</version> </requireJavaVersion> <requireMavenVersion> <version>[3.5, )</version> diff --git a/tenant-cd/pom.xml b/tenant-cd/pom.xml index faff289ece1..e9853dfa368 100644 --- a/tenant-cd/pom.xml +++ b/tenant-cd/pom.xml @@ -14,7 +14,7 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent</relativePath> </parent> diff --git a/testutil/pom.xml b/testutil/pom.xml index 491c144fdb0..503754e9aa2 100644 --- a/testutil/pom.xml +++ b/testutil/pom.xml @@ -6,12 +6,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>testutil</artifactId> <packaging>jar</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>${project.artifactId}</name> <description>Library of useful Hamcrest matchers.</description> <dependencies> diff --git a/travis/travis.sh b/travis/travis.sh index 77bfebf2505..4143a9802aa 100755 --- a/travis/travis.sh +++ b/travis/travis.sh @@ -10,7 +10,7 @@ function bell() { done } -DOCKER_IMAGE=vespaengine/vespa-dev:latest +DOCKER_IMAGE=vespaengine/vespa-dev:vespa7-java11 bell & docker run --rm -v ${HOME}/.m2:/root/.m2 -v ${HOME}/.ccache:/root/.ccache -v $(pwd):/source \ diff --git a/vdslib/pom.xml b/vdslib/pom.xml index d2fedcb1143..8907b0958f3 100644 --- a/vdslib/pom.xml +++ b/vdslib/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vdslib</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>vdslib</name> <description> </description> <dependencies> diff --git a/vdslib/src/main/java/com/yahoo/vdslib/BinaryDocumentList.java b/vdslib/src/main/java/com/yahoo/vdslib/BinaryDocumentList.java deleted file mode 100644 index 25b1a6acff4..00000000000 --- a/vdslib/src/main/java/com/yahoo/vdslib/BinaryDocumentList.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vdslib; - -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.vespa.objects.Serializer; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - */ -class BinaryDocumentList extends DocumentList { - - private DocumentTypeManager docMan; - private byte[] buffer; - private int docCount; - - /** - * Create a new documentlist, using the given buffer. - * - * @param buffer buffer containing documents - */ - BinaryDocumentList(DocumentTypeManager docMan, byte[] buffer) { - this.docMan = docMan; - ByteBuffer buf = ByteBuffer.wrap(buffer); - buf.order(ByteOrder.LITTLE_ENDIAN); - docCount = buf.getInt(); - this.buffer = buffer; - - } - - @Override - public Entry get(int index) throws ArrayIndexOutOfBoundsException { - if (index < docCount) { - return Entry.create(docMan, buffer, index); - } else { - throw new ArrayIndexOutOfBoundsException(index + " >= " + docCount); - } - } - - @Override - public int size() { return docCount; } - - @Override - public int getApproxByteSize() { - return buffer.length; - } - - @Override - public void serialize(Serializer buf) { - buf.put(null, buffer); - } - -} diff --git a/vdslib/src/main/java/com/yahoo/vdslib/BinaryEntry.java b/vdslib/src/main/java/com/yahoo/vdslib/BinaryEntry.java deleted file mode 100644 index c11bb009a73..00000000000 --- a/vdslib/src/main/java/com/yahoo/vdslib/BinaryEntry.java +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vdslib; - -import com.yahoo.document.*; -import com.yahoo.document.serialization.DocumentDeserializer; -import com.yahoo.document.serialization.DocumentDeserializerFactory; -import com.yahoo.io.GrowableByteBuffer; - -/** - * An entry in serialized form. - * - * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>, <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - */ -class BinaryEntry extends Entry { - private MetaEntry metaEntry; - private byte[] buffer; - private DocumentTypeManager docMan; - - /** - * Creates an entry from serialized form. - * @param docMan The documentmanager to use when deserializing. - * @param buffer the buffer to read the entry from - * @param entryIndex the index of the entry in the buffer - */ - BinaryEntry(DocumentTypeManager docMan, byte[] buffer, int entryIndex) { - this.buffer = buffer; - metaEntry = new MetaEntry(buffer, 4 + entryIndex * MetaEntry.SIZE); - this.docMan = docMan; - } - - @Override - public boolean valid() { return buffer != null; } - - @Override - public boolean isRemoveEntry() { return (metaEntry.flags & MetaEntry.REMOVE_ENTRY) != 0; } - - @Override - public boolean isBodyStripped() { return (metaEntry.flags & MetaEntry.BODY_STRIPPED) != 0; } - - @Override - public boolean isUpdateEntry() { return (metaEntry.flags & MetaEntry.UPDATE_ENTRY) != 0; } - - @Override - public long getTimestamp() { return metaEntry.timestamp; } - - @Override - public DocumentOperation getDocumentOperation() { - DocumentDeserializer buf = DocumentDeserializerFactory.create42( - docMan, - GrowableByteBuffer.wrap(buffer, metaEntry.headerPos, metaEntry.headerLen), - (metaEntry.bodyLen > 0) ? GrowableByteBuffer.wrap(buffer, metaEntry.bodyPos, metaEntry.bodyLen) : null - ); - - DocumentOperation op; - - if ((metaEntry.flags & MetaEntry.UPDATE_ENTRY) != 0) { - op = new DocumentUpdate(buf); - } else if ((metaEntry.flags & MetaEntry.REMOVE_ENTRY) != 0) { - op = new DocumentRemove(new Document(buf).getId()); - } else { - op = new DocumentPut(new Document(buf)); - ((DocumentPut) op).getDocument().setLastModified(getTimestamp()); - - } - return op; - } - - @Override - public DocumentOperation getHeader() { - DocumentDeserializer buf = DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(buffer, metaEntry.headerPos, metaEntry.headerLen)); - if ((metaEntry.flags & MetaEntry.UPDATE_ENTRY) != 0) { - return new DocumentUpdate(buf); - } else if ((metaEntry.flags & MetaEntry.REMOVE_ENTRY) != 0) { - return new DocumentRemove(new Document(buf).getId()); - } else { - DocumentPut op = new DocumentPut(new Document(buf)); - op.getDocument().setLastModified(getTimestamp()); - return op; - } - } - -} diff --git a/vdslib/src/main/java/com/yahoo/vdslib/DocumentList.java b/vdslib/src/main/java/com/yahoo/vdslib/DocumentList.java deleted file mode 100644 index 76d320102e3..00000000000 --- a/vdslib/src/main/java/com/yahoo/vdslib/DocumentList.java +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vdslib; - -import com.yahoo.document.DocumentId; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.vespa.objects.Serializer; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public abstract class DocumentList { - - protected DocumentList() { } - - /** - * Creates a DocumentList from serialized form. - * - * @param docMan Documentmanager to use when deserializing - * @param buffer the buffer to read from - * @return a DocumentList instance - */ - public static DocumentList create(DocumentTypeManager docMan, byte[] buffer) { - return new BinaryDocumentList(docMan, buffer); - } - - /** - * Creates a DocumentList from a list of entries. - * @param entries the entries to create a DocumentList from - * @return a DocumentList instance - * @see com.yahoo.vdslib.Entry - */ - public static DocumentList create(List<Entry> entries) { - return new DynamicDocumentList(entries); - } - - /** - * Creates a DocumentList containing a single entry. - * - * @param entry the entry to create a DocumentList from - * @return a DocumentList instance - * @see com.yahoo.vdslib.Entry - */ - public static DocumentList create(Entry entry) { - return new DynamicDocumentList(entry); - } - - /** - * Retrieves the specified Entry from the list. - * - * @param index the index of the Entry to return (0-based) - * @return the entry at the specified position - * @throws ArrayIndexOutOfBoundsException if index is < 0 or > size() - * @throws com.yahoo.document.serialization.DeserializationException if the DocumentList is stored in binary form internally and deserialization fails - */ - public abstract Entry get(int index) throws ArrayIndexOutOfBoundsException; - - /** - * Returns the size of the list. - * - * @return the size of the list - */ - public abstract int size(); - - /** - * Returns the byte size of the list. The value returned is exact if the list is stored in - * binary form internally, otherwise it is approximate. - * - * @return the byte size of the list - */ - public abstract int getApproxByteSize(); - - /** - * Serialize the list into the given buffer. - * - * @param buf the buffer to serialize into - */ - public abstract void serialize(Serializer buf); - - /** - * Test if a contains b - * - * @param list DocumentList contained - * @return true if a contains b - */ - public boolean containsAll(DocumentList list) { - if( this.size() < list.size()) { - return false; - } - - Map<DocumentId, Integer> indexes = new HashMap<DocumentId, Integer>(); - for (int i=0; i<this.size(); ++i) { - indexes.put(this.get(i).getDocumentOperation().getId(), i); - } - for (int i=0; i<list.size(); ++i) { - Integer index = indexes.get(list.get(i).getDocumentOperation().getId()); - if (index == null || - list.get(i).getTimestamp() != this.get(index).getTimestamp() || - list.get(i).kind() != this.get(index).kind()) - { - return false; - } - } - return true; - } - -} - diff --git a/vdslib/src/main/java/com/yahoo/vdslib/DynamicDocumentList.java b/vdslib/src/main/java/com/yahoo/vdslib/DynamicDocumentList.java deleted file mode 100644 index a3cb6376b58..00000000000 --- a/vdslib/src/main/java/com/yahoo/vdslib/DynamicDocumentList.java +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vdslib; - -import com.yahoo.compress.CompressionType; -import com.yahoo.document.BucketIdFactory; -import com.yahoo.document.DataType; -import com.yahoo.document.Document; -import com.yahoo.document.DocumentPut; -import com.yahoo.document.DocumentRemove; -import com.yahoo.document.DocumentUpdate; -import com.yahoo.document.serialization.DocumentSerializer; -import com.yahoo.document.serialization.DocumentSerializerFactory; -import com.yahoo.vespa.objects.Serializer; - -import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * A list of document operations. - * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - */ -public class DynamicDocumentList extends DocumentList { - private List<Entry> entries; - - DynamicDocumentList(List<Entry> entries) { - //the entries themselves are of course still modifiable, this is just an internal safeguard: - this.entries = Collections.unmodifiableList(entries); - } - - DynamicDocumentList(Entry entry) { - List<Entry> list = new ArrayList<>(1); - list.add(entry); - BucketIdFactory factory = new BucketIdFactory(); - //the entry itself is of course still modifiable, this is just an internal safeguard: - this.entries = Collections.unmodifiableList(list); - } - - @Override - public Entry get(int index) throws ArrayIndexOutOfBoundsException { - return entries.get(index); - } - - @Override - public int size() { - return entries.size(); - } - - @Override - public int getApproxByteSize() { - int size = 4; - for (Entry entry : entries) { - if (entry.getDocumentOperation() instanceof DocumentPut) { - Document doc = ((DocumentPut)entry.getDocumentOperation()).getDocument(); - size += MetaEntry.SIZE + doc.getSerializedSize(); - } else if (entry.getDocumentOperation() instanceof DocumentUpdate) { - //TODO: Implement getSerializedSize() for DocumentUpdate!!! - size += MetaEntry.SIZE + 1024; - } else if (entry.getDocumentOperation() instanceof DocumentRemove) { - //TODO: Implement getSerializedSize() for DocumentRemove!!! - size += MetaEntry.SIZE + 64; - } - } - return size; - } - - @Override - public void serialize(Serializer buf) { - if (buf instanceof DocumentSerializer) { - serializeInternal((DocumentSerializer) buf); - } else { - DocumentSerializer serializer = DocumentSerializerFactory.create42(); - serializeInternal(serializer); - serializer.getBuf().getByteBuffer().flip(); - buf.put(null, serializer.getBuf().getByteBuffer()); - } - } - @SuppressWarnings("deprecation") - private void serializeInternal(DocumentSerializer buf) { - ByteOrder originalOrder = buf.getBuf().order(); - buf.getBuf().order(ByteOrder.LITTLE_ENDIAN); - //save the position before the size - int posAtBeginning = buf.getBuf().position(); - - //write the number of entries - buf.putInt(null, entries.size()); - - //create a list of metaentries, one for each entry - List<MetaEntry> metaEntries = new ArrayList<MetaEntry>(entries.size()); - - //jump past the meta block, we will serialize this afterwards when we know sizes and positions - byte[] bogusEntry = new byte[entries.size() * MetaEntry.SIZE]; - buf.put(null, bogusEntry); - - for (Entry entry : entries) { - MetaEntry metaEntry = new MetaEntry(); - metaEntries.add(metaEntry); - - // is this a remove? in that case, set this flag - if (entry.isRemoveEntry()) metaEntry.flags |= MetaEntry.REMOVE_ENTRY; - // is the body stripped? in that case, set this flag - if (entry.isBodyStripped()) metaEntry.flags |= MetaEntry.BODY_STRIPPED; - // is this an update? in that case, set this flag - if (entry.getDocumentOperation() instanceof DocumentUpdate) metaEntry.flags |= MetaEntry.UPDATE_ENTRY; - // is this a document? in that case, try to set the timestamp - if (entry.getDocumentOperation() instanceof DocumentPut) { - Document doc = ((DocumentPut)entry.getDocumentOperation()).getDocument(); - Long lastModified = doc.getLastModified(); - if (lastModified != null) { - metaEntry.timestamp = lastModified; - } - - if (doc.getDataType().contentStruct().getCompressionConfig() != null - && doc.getDataType().contentStruct().getCompressionConfig().type != CompressionType.NONE) { - metaEntry.flags |= MetaEntry.COMPRESSED; - } - if (doc.getDataType().getBodyType().getCompressionConfig() != null - && doc.getDataType().getBodyType().getCompressionConfig().type != CompressionType.NONE) { - metaEntry.flags |= MetaEntry.COMPRESSED; - } - } - - metaEntry.headerPos = buf.getBuf().position() - posAtBeginning; - - buf.getBuf().order(ByteOrder.BIG_ENDIAN); - if (entry.getDocumentOperation() instanceof DocumentPut) { - Document doc = ((DocumentPut)entry.getDocumentOperation()).getDocument(); - //serialize document and save length: - doc.serializeHeader(buf); - } else if (entry.getDocumentOperation() instanceof DocumentUpdate) { - DocumentUpdate docUp = (DocumentUpdate) entry.getDocumentOperation(); - docUp.serialize(buf); - } else if (entry.getDocumentOperation() instanceof DocumentRemove) { - new Document(DataType.DOCUMENT, entry.getDocumentOperation().getId()).serialize(buf); - } else { - throw new IllegalArgumentException("Can not handle class " + entry.getDocumentOperation().getClass().getName()); - } - - metaEntry.headerLen = buf.getBuf().position() - metaEntry.headerPos - posAtBeginning; - - if (entry.getDocumentOperation() instanceof DocumentPut) { - metaEntry.bodyPos = buf.getBuf().position() - posAtBeginning; - Document doc = ((DocumentPut)entry.getDocumentOperation()).getDocument(); - doc.serializeBody(buf); - metaEntry.bodyLen = buf.getBuf().position() - metaEntry.bodyPos - posAtBeginning; - } else { - metaEntry.bodyPos = 0; - metaEntry.bodyLen = 0; - } - buf.getBuf().order(ByteOrder.LITTLE_ENDIAN); - - } - //save position after payload: - int posAfterEntries = buf.getBuf().position(); - //go to beginning (after length) to serialize metaentries: - buf.getBuf().position(posAtBeginning + 4); - //serialize metaentries - for (MetaEntry metaEntry : metaEntries) { - metaEntry.serialize(buf.getBuf()); - } - //set position to after payload: - buf.getBuf().position(posAfterEntries); - buf.getBuf().order(originalOrder); - } -} diff --git a/vdslib/src/main/java/com/yahoo/vdslib/DynamicEntry.java b/vdslib/src/main/java/com/yahoo/vdslib/DynamicEntry.java deleted file mode 100644 index 81226c64604..00000000000 --- a/vdslib/src/main/java/com/yahoo/vdslib/DynamicEntry.java +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vdslib; - -import com.yahoo.document.DocumentOperation; -import com.yahoo.document.DocumentPut; -import com.yahoo.document.DocumentRemove; -import com.yahoo.document.DocumentUpdate; - -/** - * Represents an in-memory entry. - * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - */ -class DynamicEntry extends Entry { - private DocumentOperation op; - private boolean bodyStripped; - - DynamicEntry(DocumentOperation op, boolean bodyStripped) { - this.op = op; - this.bodyStripped = bodyStripped; - } - - DynamicEntry(DocumentUpdate op) { - this.op = op; - this.bodyStripped = false; - } - - DynamicEntry(DocumentRemove op) { - this.op = op; - this.bodyStripped = false; - } - - @Override - public boolean valid() { - return true; - } - - @Override - public boolean isRemoveEntry() { - return op instanceof DocumentRemove; - } - - @Override - public boolean isBodyStripped() { - return bodyStripped; - } - - @Override - public boolean isUpdateEntry() { - return op instanceof DocumentUpdate; - } - - @Override - public long getTimestamp() { - if (op instanceof DocumentPut) { - DocumentPut put = (DocumentPut) op; - final Long lastModified = put.getDocument().getLastModified(); - if (lastModified != null) { - return lastModified; - } - } - return 0L; - } - - @Override - public DocumentOperation getDocumentOperation() { - return op; - } - - @Override - public DocumentOperation getHeader() { - return op; - //TODO: Only return header fields of Document here...? - } -} diff --git a/vdslib/src/main/java/com/yahoo/vdslib/Entry.java b/vdslib/src/main/java/com/yahoo/vdslib/Entry.java deleted file mode 100644 index 2ce3822f5a9..00000000000 --- a/vdslib/src/main/java/com/yahoo/vdslib/Entry.java +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vdslib; - -import com.yahoo.document.DocumentOperation; -import com.yahoo.document.DocumentRemove; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.DocumentUpdate; - -/** - * Represents a document operation in a DocumentList, which can currently be - * PUT, REMOVE and UPDATE. - * - * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>, <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - */ -public abstract class Entry { - - protected Entry() { } - - /** - * Creates a new entry from serialized form. - * - * @param docMan Documentmanager to use when deserializing - * @param buffer the buffer to read the entry from - * @param entryIndex the index of the entry in the buffer - * @return an Entry reading from the buffer - */ - public static Entry create(DocumentTypeManager docMan, byte[] buffer, int entryIndex) { - return new BinaryEntry(docMan, buffer, entryIndex); - } - - /** - * Creates a new entry from a document operation. - * - * @param op the document in the entry - * @param bodyStripped true if the document contains only header fields - * @return an Entry for this document - */ - public static Entry create(DocumentOperation op, boolean bodyStripped) { - return new DynamicEntry(op, bodyStripped); - } - - /** - * Creates a new entry from a document operation. - * - * @param op the document in the entry - * @return an Entry for this document - */ - public static Entry create(DocumentOperation op) { - return create(op, false); - } - /** - * Creates a new entry from a document remove operation. - * - * @param doc the document in the entry - * @return an Entry for this document - */ - public static Entry create(DocumentRemove doc) { - return new DynamicEntry(doc); - } - - /** - * Creates a new entry from a document update operation. - * - * @param doc the document update in the entry - * @return an Entry for this document update - */ - public static Entry create(DocumentUpdate doc) { - return new DynamicEntry(doc); - } - - /** - * Entries in iterators gotten from DocumentList::end() are invalid. - * @return true if valid - */ - public abstract boolean valid(); - - /** - * Returns true if the Document represented by this entry has been removed from persistent storage. - * - * @return true if the Document has been removed - */ - public abstract boolean isRemoveEntry(); - - /** - * Returns true if the Document represented by this entry has gotten its body fields stripped - * away (note: the body fields might still be stored in persistent storage). - * - * @return true if the Document only has header fields - */ - public abstract boolean isBodyStripped(); - - /** - * Returns true if this entry represents a document update operation. - * - * @return true if this is a document update operation - */ - public abstract boolean isUpdateEntry(); - - - public int kind(){ - if (isRemoveEntry()) { - return 0; //REMOVE - } - if (isUpdateEntry()) { - return 2; //UPDATE - } - return 1; // PUT - } - - /** - * Returns the timestamp (last modified) of this entry, from persistent storage. - * @return the last modified timestamp of this entry - */ - public abstract long getTimestamp(); - - /** - * Returns the DocumentPut or DocumentUpdate operation in this entry. - * - * @return the DocumentOperation in this entry. - */ - public abstract DocumentOperation getDocumentOperation(); - - /** - * Returns the Document header (if this is a DocumentPut or a DocumentRemove operation), otherwise - * a DocumentUpdate operation. - * - * @return a DocumentPut operation containing a Document with only the header fields present - * @throws RuntimeException if deserialization fails, or if this is a DocumentUpdate operation - */ - public abstract DocumentOperation getHeader(); - - @Override - public boolean equals(Object obj) { - if ( this == obj ) { - return true; - } - if (!(obj instanceof Entry)) { - return false; - } - Entry entry = (Entry) obj; - return this.getDocumentOperation().getId().equals(entry.getDocumentOperation().getId()) && - this.getTimestamp() == entry.getTimestamp() && - this.kind() == entry.kind() && - this.isBodyStripped() == entry.isBodyStripped() && - this.valid() == entry.valid(); - } - - @Override - public int hashCode() { - int res = 31; - res = 31 * res + getDocumentOperation().getId().hashCode(); - res = (int) (31 * res + getTimestamp()); - res = 31 * res + kind()*31; - res = 31 * res + (isBodyStripped() ? 17 : 249); - res = 31 * res + (valid() ? 333 : 31); - - return res; - } -} diff --git a/vdslib/src/test/java/com/yahoo/vdslib/DocumentListTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/DocumentListTestCase.java deleted file mode 100644 index cae51440400..00000000000 --- a/vdslib/src/test/java/com/yahoo/vdslib/DocumentListTestCase.java +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vdslib; - -import com.yahoo.document.DocumentId; -import com.yahoo.document.DocumentPut; -import com.yahoo.document.DocumentRemove; -import com.yahoo.document.DocumentType; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.DocumentTypeManagerConfigurer; -import com.yahoo.document.DocumentUpdate; -import com.yahoo.document.Field; -import com.yahoo.document.datatypes.FloatFieldValue; -import com.yahoo.document.datatypes.StringFieldValue; -import com.yahoo.document.serialization.DocumentSerializer; -import com.yahoo.document.serialization.DocumentSerializerFactory; -import com.yahoo.document.update.FieldUpdate; -import org.junit.Test; - -import java.io.FileOutputStream; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -public class DocumentListTestCase { - - @Test - @SuppressWarnings("deprecation") - public void testSelfSerializationAndWriteJavaFile() throws Exception { - DocumentTypeManager docMan = new DocumentTypeManager(); - DocumentTypeManagerConfigurer.configure(docMan, "file:src/test/files/documentmanager.cfg"); - - DocumentType bmType = docMan.getDocumentType("benchmark"); - DocumentPut put1 = new DocumentPut(bmType, "userdoc:foo:99999999:1"); - put1.getDocument().setFieldValue("headerstring", "foo"); - DocumentRemove doc2 = new DocumentRemove(new DocumentId("userdoc:foo:99999999:2")); - DocumentPut put3 = new DocumentPut(bmType, "userdoc:foo:99999999:3"); - put3.getDocument().setFieldValue("bodyfloat", new FloatFieldValue(5.5f)); - - - Field bodystringField = bmType.getField("bodystring"); - DocumentUpdate docUp = new DocumentUpdate(docMan.getDocumentType("benchmark"), new DocumentId("userdoc:foo:99999999:4")); - docUp.addFieldUpdate(FieldUpdate.createAssign(bodystringField, new StringFieldValue("ballooooo"))); - - List<Entry> entries = new ArrayList<>(); - entries.add(Entry.create(put1)); - entries.add(Entry.create(doc2)); - entries.add(Entry.create(put3)); - entries.add(Entry.create(docUp)); - - DocumentList documentList = DocumentList.create(entries); - - DocumentSerializer gbuf = DocumentSerializerFactory.create42(); - gbuf.putInt(null, 1234); // Add some data to avoid special case where position() is 0 for buffer used. - int startPos = gbuf.getBuf().position(); - documentList.serialize(gbuf); - - int size = gbuf.getBuf().position() - startPos; - byte[] data = new byte[size]; - gbuf.getBuf().position(startPos); - gbuf.getBuf().get(data); - FileOutputStream stream = new FileOutputStream("./src/test/files/documentlist-java.dat"); - stream.write(data); - stream.close(); - - gbuf.getBuf().position(0); - - DocumentList documentList2 = DocumentList.create(docMan, data); - - assertEquals(4, documentList2.size()); - Entry entry1 = documentList2.get(0); - assertEquals(0L, entry1.getTimestamp()); - assertFalse(entry1.isBodyStripped()); - assertFalse(entry1.isRemoveEntry()); - assertFalse(entry1.isUpdateEntry()); - assertTrue(entry1.getDocumentOperation() instanceof DocumentPut); - assertEquals(new StringFieldValue("foo"), ((DocumentPut) entry1.getDocumentOperation()).getDocument().getFieldValue("headerstring")); - - Entry entry2 = documentList2.get(1); - assertEquals(0L, entry2.getTimestamp()); - assertFalse(entry2.isBodyStripped()); - assertTrue(entry2.isRemoveEntry()); - assertFalse(entry2.isUpdateEntry()); - assertTrue(entry2.getDocumentOperation() instanceof DocumentRemove); - - Entry entry3 = documentList2.get(2); - assertEquals(0L, entry3.getTimestamp()); - assertFalse(entry3.isBodyStripped()); - assertFalse(entry3.isRemoveEntry()); - assertFalse(entry3.isUpdateEntry()); - assertTrue(entry3.getDocumentOperation() instanceof DocumentPut); - assertEquals(new FloatFieldValue(5.5f), ((DocumentPut) entry3.getDocumentOperation()).getDocument().getFieldValue("bodyfloat")); - - Entry entry4 = documentList2.get(3); - assertEquals(0L, entry4.getTimestamp()); - assertFalse(entry4.isBodyStripped()); - assertFalse(entry4.isRemoveEntry()); - assertTrue(entry4.isUpdateEntry()); - assertTrue(entry4.getDocumentOperation() instanceof DocumentUpdate); - assertEquals(new StringFieldValue("ballooooo"),((DocumentUpdate) entry4.getDocumentOperation()).getFieldUpdate(bodystringField).getValueUpdate(0).getValue()); - } - - @Test - public void testContains() { - DocumentTypeManager manager = new DocumentTypeManager(); - DocumentTypeManagerConfigurer.configure(manager, "file:src/test/files/documentmanager.cfg"); - - DocumentType bmType = manager.getDocumentType("benchmark"); - DocumentPut put1 = new DocumentPut(bmType, "userdoc:foo:99999999:1"); - DocumentRemove remove1 = new DocumentRemove(new DocumentId("userdoc:foo:99999999:2")); - DocumentPut put2 = new DocumentPut(bmType, "userdoc:foo:99999999:3"); - - List<Entry> entries = new ArrayList<Entry>(); - entries.add(Entry.create(put1)); - entries.add(Entry.create(remove1)); - entries.add(Entry.create(put2)); - - DocumentList documentList = DocumentList.create(entries); - - DocumentPut put3 = new DocumentPut(bmType, "userdoc:foo:99999999:1"); - DocumentRemove remove2 = new DocumentRemove(new DocumentId("userdoc:foo:99999999:2")); - DocumentPut put4 = new DocumentPut(bmType, "userdoc:foo:99999999:3"); - - List<Entry> entries2 = new ArrayList<Entry>(); - entries2.add(Entry.create(put3)); - entries2.add(Entry.create(remove2)); - entries2.add(Entry.create(put4)); - - DocumentList documentList2 = DocumentList.create(entries2); - - assertTrue(documentList.containsAll(documentList2)); - - Long t = put4.getDocument().getLastModified(); - put4.getDocument().setLastModified(13L); - assertTrue(!documentList.containsAll(documentList2)); - put4.getDocument().setLastModified(t); - - assert(documentList.containsAll(documentList2)); - - entries.remove(2); - assertTrue(!documentList.containsAll(documentList2)); - } - -} diff --git a/vdslib/src/test/java/com/yahoo/vdslib/EntryTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/EntryTestCase.java deleted file mode 100644 index ccce2ffcbb7..00000000000 --- a/vdslib/src/test/java/com/yahoo/vdslib/EntryTestCase.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vdslib; - -import com.yahoo.document.DocumentPut; -import com.yahoo.document.DocumentType; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.DocumentTypeManagerConfigurer; -import org.junit.Test; - -import static org.junit.Assert.assertTrue; - -/** - * @author banino - */ -public class EntryTestCase { - - @Test - public void testEquals() { - DocumentTypeManager manager = new DocumentTypeManager(); - DocumentTypeManagerConfigurer.configure(manager, "file:src/test/files/documentmanager.cfg"); - - DocumentType bmType = manager.getDocumentType("benchmark"); - DocumentPut put1 = new DocumentPut(bmType, "userdoc:foo:99999999:1"); - DocumentPut put2 = new DocumentPut(bmType, "userdoc:foo:99999999:2"); - - Entry entry1 = Entry.create(put1); - Entry entry2 = Entry.create(put1); - assertTrue(entry1.equals(entry2)); - - Entry entry3 = Entry.create(put2); - assertTrue(!entry1.equals(entry3)); - } - - @Test - public void testHashCode() { - DocumentTypeManager manager = new DocumentTypeManager(); - DocumentTypeManagerConfigurer.configure(manager, "file:src/test/files/documentmanager.cfg"); - - DocumentType bmType = manager.getDocumentType("benchmark"); - DocumentPut put1 = new DocumentPut(bmType, "userdoc:foo:99999999:1"); - DocumentPut put2 = new DocumentPut(bmType, "userdoc:foo:99999999:2"); - - Entry entry1 = Entry.create(put1); - Entry entry2 = Entry.create(put1); - assertTrue(entry1.hashCode() == entry2.hashCode()); - - Entry entry3 = Entry.create(put2); - assertTrue(entry1.hashCode() != entry3.hashCode()); - } - -} diff --git a/vespa-application-maven-plugin/pom.xml b/vespa-application-maven-plugin/pom.xml index 6468a24e5a9..aaa30ed28af 100644 --- a/vespa-application-maven-plugin/pom.xml +++ b/vespa-application-maven-plugin/pom.xml @@ -6,11 +6,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa-application-maven-plugin</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>maven-plugin</packaging> <description>Maven Plugin for assembling a vespa application package</description> <prerequisites> diff --git a/vespa-athenz/pom.xml b/vespa-athenz/pom.xml index ea3c28bbcc3..27b68fbf360 100644 --- a/vespa-athenz/pom.xml +++ b/vespa-athenz/pom.xml @@ -7,12 +7,12 @@ <modelVersion>4.0.0</modelVersion> <artifactId>vespa-athenz</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <dependencies> @@ -109,12 +109,10 @@ <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpcore</artifactId> - <scope>compile</scope> </dependency> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> - <scope>compile</scope> </dependency> </dependencies> diff --git a/vespa-documentgen-plugin/etc/music/music.sd b/vespa-documentgen-plugin/etc/music/music.sd index 2e40523a6d9..b5083d4e490 100644 --- a/vespa-documentgen-plugin/etc/music/music.sd +++ b/vespa-documentgen-plugin/etc/music/music.sd @@ -7,33 +7,25 @@ search music { field title type string { indexing: summary | index # How this field should be indexed - # index-to: title, default # Create two indexes weight: 75 # Ranking importancy of this field, used by the built in nativeRank feature - header } field artist type string { indexing: summary | attribute | index - # index-to: artist, default - weight: 25 - header } field year type int { indexing: summary | attribute - header } # Increase query field popularity type int { indexing: summary | attribute - body } field url type uri { indexing: summary | index - header } } diff --git a/vespa-documentgen-plugin/pom.xml b/vespa-documentgen-plugin/pom.xml index 3aa66856cb4..1aad4857f49 100644 --- a/vespa-documentgen-plugin/pom.xml +++ b/vespa-documentgen-plugin/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa-documentgen-plugin</artifactId> <packaging>maven-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>Vespa DocumentGen Plugin</name> <dependencies> <dependency> diff --git a/vespa-hadoop/abi-spec.json b/vespa-hadoop/abi-spec.json index 40c624d2577..c6174aca970 100644 --- a/vespa-hadoop/abi-spec.json +++ b/vespa-hadoop/abi-spec.json @@ -36,8 +36,8 @@ "public" ], "methods": [ - "protected void <init>(com.yahoo.component.ComponentId)", "protected void <init>()", + "protected void <init>(com.yahoo.component.ComponentId)", "public final void initId(com.yahoo.component.ComponentId)", "public final com.yahoo.component.ComponentId getId()", "public final boolean hasInitializedId()", @@ -229,35 +229,6 @@ "public static final com.yahoo.component.Version currentVersion" ] }, - "com.yahoo.component.provider.ComponentClass$ComponentConstructor": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.reflect.Constructor)", - "public java.lang.reflect.Constructor getConstructor()", - "public boolean preferredTo(com.yahoo.component.provider.ComponentClass$ComponentConstructor)" - ], - "fields": [ - "public final boolean isLegal", - "public final boolean hasComponentId" - ] - }, - "com.yahoo.component.provider.ComponentClass": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.Class)", - "public com.yahoo.component.AbstractComponent createComponent(com.yahoo.component.ComponentId, java.util.Map, java.lang.String)", - "public com.yahoo.component.provider.ComponentClass$ComponentConstructor getPreferredConstructor()" - ], - "fields": [] - }, "com.yahoo.component.provider.ComponentRegistry": { "superClass": "java.lang.Object", "interfaces": [], @@ -711,6 +682,7 @@ "public java.lang.String getName()", "public java.io.Reader getReader()", "public java.lang.String toString()", + "public static java.io.Reader nullReader()", "public int read(java.nio.CharBuffer)", "public int read()", "public int read(char[])", @@ -721,6 +693,7 @@ "public void mark(int)", "public void reset()", "public void close()", + "public long transferTo(java.io.Writer)", "public static void closeAll(java.util.List)" ], "fields": [] @@ -2880,35 +2853,6 @@ ], "fields": [] }, - "com.yahoo.text.DoubleFormatter": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>()", - "public static java.lang.StringBuilder fmt(java.lang.StringBuilder, double)", - "public static java.lang.String stringValue(double)", - "public static void append(java.lang.StringBuilder, double)", - "public static void append(java.lang.StringBuilder, int)" - ], - "fields": [] - }, - "com.yahoo.text.DoubleParser": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>()", - "public static double parse(java.lang.String)" - ], - "fields": [] - }, "com.yahoo.text.ForwardWriter": { "superClass": "com.yahoo.text.GenericWriter", "interfaces": [], @@ -3034,19 +2978,6 @@ ], "fields": [] }, - "com.yahoo.text.LanguageHacks": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public static boolean isCJK(java.lang.String)", - "public static boolean yellDesegments(java.lang.String)" - ], - "fields": [] - }, "com.yahoo.text.Lowercase": { "superClass": "java.lang.Object", "interfaces": [], @@ -3589,7 +3520,6 @@ "public void <init>(com.yahoo.vespa.http.client.config.Endpoint)", "public com.yahoo.vespa.http.client.config.Endpoint getEndpoint()", "public boolean isSuccess()", - "public boolean isTransient()", "public com.yahoo.vespa.http.client.Result$ResultType getResultType()", "public java.lang.Exception getException()", "public java.lang.String getTraceMessage()", @@ -3628,7 +3558,6 @@ "public java.lang.CharSequence getDocumentDataAsCharSequence()", "public java.lang.Object getContext()", "public boolean isSuccess()", - "public boolean isTransient()", "public java.util.List getDetails()", "public boolean hasLocalTrace()", "public java.lang.String toString()" @@ -3773,7 +3702,6 @@ "public com.yahoo.vespa.http.client.config.ConnectionParams$Builder setUseCompression(boolean)", "public com.yahoo.vespa.http.client.config.ConnectionParams$Builder setMaxRetries(int)", "public com.yahoo.vespa.http.client.config.ConnectionParams$Builder setDryRun(boolean)", - "public com.yahoo.vespa.http.client.config.ConnectionParams$Builder setEnableV3Protocol(boolean)", "public com.yahoo.vespa.http.client.config.ConnectionParams$Builder setMinTimeBetweenRetries(long, java.util.concurrent.TimeUnit)", "public long getMinTimeBetweenRetriesMs()", "public com.yahoo.vespa.http.client.config.ConnectionParams$Builder setTraceLevel(int)", @@ -3825,7 +3753,6 @@ "public int getMaxRetries()", "public long getMinTimeBetweenRetriesMs()", "public boolean isDryRun()", - "public boolean isEnableV3Protocol()", "public int getTraceLevel()", "public int getTraceEveryXOperation()", "public boolean getPrintTraceToStdErr()" diff --git a/vespa-hadoop/pom.xml b/vespa-hadoop/pom.xml index 8fe5d8b7ff5..2f8dbb46a78 100644 --- a/vespa-hadoop/pom.xml +++ b/vespa-hadoop/pom.xml @@ -7,11 +7,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa-hadoop</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>${project.artifactId}</name> <description>Integration tools between Vespa and Hadoop</description> diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java index 600658067cb..e5810c4e7b2 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/VespaRecordWriter.java @@ -100,7 +100,6 @@ public class VespaRecordWriter extends RecordWriter { ConnectionParams.Builder connParamsBuilder = new ConnectionParams.Builder(); connParamsBuilder.setDryRun(configuration.dryrun()); connParamsBuilder.setUseCompression(configuration.useCompression()); - connParamsBuilder.setEnableV3Protocol(configuration.useV3Protocol()); connParamsBuilder.setNumPersistentConnectionsPerEndpoint(configuration.numConnections()); connParamsBuilder.setMaxRetries(configuration.numRetries()); if (configuration.proxyHost() != null) { diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java index 99928fd80fb..2a1179dbec6 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/util/VespaConfiguration.java @@ -19,7 +19,6 @@ public class VespaConfiguration { public static final String USE_COMPRESSION = "vespa.feed.usecompression"; public static final String DATA_FORMAT = "vespa.feed.data.format"; public static final String PROGRESS_REPORT = "vespa.feed.progress.interval"; - public static final String V3_PROTOCOL = "vespa.feed.v3.protocol"; public static final String CONNECTIONS = "vespa.feed.connections"; public static final String THROTTLER_MIN_SIZE = "vespa.feed.throttler.min.size"; public static final String QUERY_CONNECTION_TIMEOUT = "vespa.query.connection.timeout"; @@ -78,11 +77,6 @@ public class VespaConfiguration { } - public boolean useV3Protocol() { - return getBoolean(V3_PROTOCOL, true); - } - - public int numConnections() { return getInt(CONNECTIONS, 1); } @@ -187,7 +181,6 @@ public class VespaConfiguration { sb.append(USE_COMPRESSION + ": " + useCompression() +"\n"); sb.append(DATA_FORMAT + ": " + dataFormat() +"\n"); sb.append(PROGRESS_REPORT + ": " + progressInterval() +"\n"); - sb.append(V3_PROTOCOL + ": " + useV3Protocol() +"\n"); sb.append(CONNECTIONS + ": " + numConnections() +"\n"); sb.append(THROTTLER_MIN_SIZE + ": " + throttlerMinSize() +"\n"); sb.append(QUERY_CONNECTION_TIMEOUT + ": " + queryConnectionTimeout() +"\n"); diff --git a/vespa-http-client/abi-spec.json b/vespa-http-client/abi-spec.json index 72ae6588d8f..22c8ce9b7fc 100644 --- a/vespa-http-client/abi-spec.json +++ b/vespa-http-client/abi-spec.json @@ -94,7 +94,6 @@ "public void <init>(com.yahoo.vespa.http.client.config.Endpoint)", "public com.yahoo.vespa.http.client.config.Endpoint getEndpoint()", "public boolean isSuccess()", - "public boolean isTransient()", "public com.yahoo.vespa.http.client.Result$ResultType getResultType()", "public java.lang.Exception getException()", "public java.lang.String getTraceMessage()", @@ -133,7 +132,6 @@ "public java.lang.CharSequence getDocumentDataAsCharSequence()", "public java.lang.Object getContext()", "public boolean isSuccess()", - "public boolean isTransient()", "public java.util.List getDetails()", "public boolean hasLocalTrace()", "public java.lang.String toString()" @@ -278,7 +276,6 @@ "public com.yahoo.vespa.http.client.config.ConnectionParams$Builder setUseCompression(boolean)", "public com.yahoo.vespa.http.client.config.ConnectionParams$Builder setMaxRetries(int)", "public com.yahoo.vespa.http.client.config.ConnectionParams$Builder setDryRun(boolean)", - "public com.yahoo.vespa.http.client.config.ConnectionParams$Builder setEnableV3Protocol(boolean)", "public com.yahoo.vespa.http.client.config.ConnectionParams$Builder setMinTimeBetweenRetries(long, java.util.concurrent.TimeUnit)", "public long getMinTimeBetweenRetriesMs()", "public com.yahoo.vespa.http.client.config.ConnectionParams$Builder setTraceLevel(int)", @@ -330,7 +327,6 @@ "public int getMaxRetries()", "public long getMinTimeBetweenRetriesMs()", "public boolean isDryRun()", - "public boolean isEnableV3Protocol()", "public int getTraceLevel()", "public int getTraceEveryXOperation()", "public boolean getPrintTraceToStdErr()" diff --git a/vespa-http-client/pom.xml b/vespa-http-client/pom.xml index 1b70ea7cba5..f98b39c70b1 100644 --- a/vespa-http-client/pom.xml +++ b/vespa-http-client/pom.xml @@ -6,11 +6,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa-http-client</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>${project.artifactId}</name> <description>Independent external feeding API towards Vespa.</description> diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java index e35c0316433..bde637a3e39 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java @@ -62,7 +62,6 @@ public interface FeedClient extends AutoCloseable { * @see FeedEndpointException * @param exception An exception specifying endpoint and cause. See {@link FeedEndpointException} for details. */ - // TODO Vespa 7: Remove empty default implementation default void onEndpointException(FeedEndpointException exception) {} } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Result.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Result.java index efa609a2d59..3abae261f5e 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Result.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Result.java @@ -30,7 +30,6 @@ public class Result { private final Document document; private final boolean success; - private final boolean _transient; private final List<Detail> details; private final String localTrace; @@ -38,13 +37,10 @@ public class Result { this.document = document; this.details = Collections.unmodifiableList(new ArrayList<>(values)); boolean totalSuccess = true; - boolean totalTransient = true; for (Detail d : details) { if (d.getResultType() != ResultType.OPERATION_EXECUTED) {totalSuccess = false; } - if (d.getResultType() != ResultType.TRANSITIVE_ERROR) {totalTransient = false; } } this.success = totalSuccess; - this._transient = totalTransient; this.localTrace = localTrace == null ? null : localTrace.toString(); } @@ -83,17 +79,6 @@ public class Result { public boolean isSuccess() { return success; } - /** - * @deprecated use resultType on items getDetails() to check operations. - * Returns true if an error is transient, false if it is permanent. Irrelevant - * if {@link #isSuccess()} is true (returns true in those cases). - * - * @return true if an error is transient (or there is no error), false otherwise. - */ - @Deprecated - public boolean isTransient() { - return _transient; - } public List<Detail> getDetails() { return details; } @@ -150,18 +135,6 @@ public class Result { } /** - * @deprecated use getResultType. - * Returns true if an error is transient, false if it is permanent. Irrelevant - * if {@link #isSuccess()} is true (returns true in those cases). - * - * @return true if an error is transient (or there is no error), false otherwise. - */ - @Deprecated - public boolean isTransient() { - return resultType == ResultType.TRANSITIVE_ERROR || resultType == ResultType.OPERATION_EXECUTED; - } - - /** * Returns the result of the operation. */ public ResultType getResultType() { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java index d5f362ce1d1..9e34d4d1747 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java @@ -46,7 +46,6 @@ public final class ConnectionParams { private int maxRetries = 100; private long minTimeBetweenRetriesMs = 700; private boolean dryRun = false; - private boolean enableV3Protocol = true; private int traceLevel = 0; private int traceEveryXOperation = 0; private boolean printTraceToStdErr = true; @@ -175,12 +174,6 @@ public final class ConnectionParams { return this; } - @Beta - public Builder setEnableV3Protocol(boolean enableV3Protocol) { - this.enableV3Protocol = enableV3Protocol; - return this; - } - /** * Set the min time between retries when temporarily failing against a gateway. * @@ -245,7 +238,6 @@ public final class ConnectionParams { maxRetries, minTimeBetweenRetriesMs, dryRun, - enableV3Protocol, traceLevel, traceEveryXOperation, printTraceToStdErr); @@ -301,7 +293,6 @@ public final class ConnectionParams { private final int maxRetries; private final long minTimeBetweenRetriesMs; private final boolean dryRun; - private final boolean enableV3Protocol; private final int traceLevel; private final int traceEveryXOperation; private final boolean printTraceToStdErr; @@ -319,7 +310,6 @@ public final class ConnectionParams { int maxRetries, long minTimeBetweenRetriesMs, boolean dryRun, - boolean enableV3Protocol, int traceLevel, int traceEveryXOperation, boolean printTraceToStdErr) { @@ -335,7 +325,6 @@ public final class ConnectionParams { this.maxRetries = maxRetries; this.minTimeBetweenRetriesMs = minTimeBetweenRetriesMs; this.dryRun = dryRun; - this.enableV3Protocol = enableV3Protocol; this.traceLevel = traceLevel; this.traceEveryXOperation = traceEveryXOperation; this.printTraceToStdErr = printTraceToStdErr; @@ -387,8 +376,6 @@ public final class ConnectionParams { return dryRun; } - public boolean isEnableV3Protocol() { return enableV3Protocol; } - public int getTraceLevel() { return traceLevel; } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/FeedParams.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/FeedParams.java index 06edd222be2..4adf3912dbd 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/FeedParams.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/FeedParams.java @@ -39,13 +39,11 @@ public final class FeedParams { * Mutable class used to instantiate a {@link FeedParams}. */ public static final class Builder { - private DataFormat dataFormat = DataFormat.XML_UTF8; + private DataFormat dataFormat = DataFormat.JSON_UTF8; private long serverTimeout = TimeUnit.SECONDS.toMillis(180); private long clientTimeout = TimeUnit.SECONDS.toMillis(20); private String route = null; private int maxChunkSizeBytes = 50 * 1024; - // This value is deprecated, and the default is no longer used. - private int clientQueueSize = -1; private int maxInFlightRequests = 5000; private long localQueueTimeOut = 180000; private String priority = null; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java index e9852de215a..57007743b1b 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java @@ -78,7 +78,7 @@ class ApacheGatewayConnection implements GatewayConnection { ConnectionParams connectionParams, HttpClientFactory httpClientFactory, String clientId) { - SUPPORTED_VERSIONS.add(2); + SUPPORTED_VERSIONS.add(3); this.endpoint = endpoint; this.feedParams = feedParams; this.clusterSpecificRoute = clusterSpecificRoute; @@ -94,11 +94,8 @@ class ApacheGatewayConnection implements GatewayConnection { endOfFeed = END_OF_FEED_XML; } this.clientId = clientId; - if (connectionParams.isEnableV3Protocol()) { - if (this.clientId == null) { - throw new RuntimeException("Set to support version 3, but got no client Id."); - } - SUPPORTED_VERSIONS.add(3); + if (this.clientId == null) { + throw new RuntimeException("Got no client Id."); } } @@ -161,7 +158,7 @@ class ApacheGatewayConnection implements GatewayConnection { private ByteBuffer[] getDataWithStartAndEndOfFeed(List<Document> docs, int version) { List<ByteBuffer> data = new ArrayList<>(); - if (version == 2 || version == 3) { // TODO: Vespa 7: Remove support for version 2 + if (version == 3) { for (Document doc : docs) { int operationSize = doc.size() + startOfFeed.length + endOfFeed.length; StringBuilder envelope = new StringBuilder(); @@ -344,10 +341,6 @@ class ApacheGatewayConnection implements GatewayConnection { log.log(Level.FINE, "Server decided upon protocol version " + serverVersion + "."); } } - if (this.connectionParams.isEnableV3Protocol() && serverVersion != 3) { - throw new ServerResponseException("Client was set up to use v3 of protocol, however, gateway wants to " + - "use version " + serverVersion + ". Already set up structures for v3 so can not do v2 now."); - } this.negotiatedVersion = serverVersion; } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java index d124475e3a5..da45acc5687 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java @@ -79,12 +79,8 @@ public class ClusterConnection implements AutoCloseable { operationProcessor.getClientId() ); } - if (connectionParams.isEnableV3Protocol()) { - if (documentQueue == null) { - documentQueue = new DocumentQueue(clientQueueSizePerCluster); - } - } else { - documentQueue = new DocumentQueue(clientQueueSizePerCluster / cluster.getEndpoints().size()); + if (documentQueue == null) { + documentQueue = new DocumentQueue(clientQueueSizePerCluster); } final IOThread ioThread = new IOThread( operationProcessor.getIoThreadGroup(), @@ -95,7 +91,7 @@ public class ClusterConnection implements AutoCloseable { maxInFlightPerSession, feedParams.getLocalQueueTimeOut(), documentQueue, - connectionParams.isEnableV3Protocol() ? feedParams.getMaxSleepTimeMs() : 0); + feedParams.getMaxSleepTimeMs()); ioThreads.add(ioThread); } } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java index 7ead0c4a37f..45133901567 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java @@ -51,7 +51,6 @@ public class OperationProcessor { private final long minTimeBetweenRetriesMs; private final Random random = new SecureRandom(); private final int traceEveryXOperation; - private final boolean blockOperationsToSameDocument; private int traceCounter = 0; private final boolean traceToStderr; private final ThreadGroup ioThreadGroup; @@ -66,7 +65,6 @@ public class OperationProcessor { this.resultCallback = resultCallback; this.incompleteResultsThrottler = incompleteResultsThrottler; this.timeoutExecutor = timeoutExecutor; - this.blockOperationsToSameDocument = sessionParams.getConnectionParams().isEnableV3Protocol(); this.ioThreadGroup = new ThreadGroup("operationprocessor"); if (sessionParams.getClusters().isEmpty()) { @@ -243,7 +241,7 @@ public class OperationProcessor { incompleteResultsThrottler.operationStart(); synchronized (monitor) { - if (blockOperationsToSameDocument && inflightDocumentIds.contains(document.getDocumentId())) { + if (inflightDocumentIds.contains(document.getDocumentId())) { blockedDocumentsByDocumentId.put(document.getDocumentId(), document); return; } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java index 837df3c98ef..98cd13a226d 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java @@ -66,16 +66,6 @@ public class CommandLineArguments { return null; } } - if (cmdArgs.enableV2Protocol) { - if (cmdArgs.enableV3Protocol) { - System.err.println("both --useV2Protocol and --useV3Protocol options specified, ignoring deprecated --useV2Protocol option"); - cmdArgs.enableV2Protocol = false; - } else { - System.err.println("--useV2Protocol option is deprecated"); - } - } else { - cmdArgs.enableV3Protocol = true; - } return cmdArgs; } @@ -110,10 +100,7 @@ public class CommandLineArguments { private HelpOption helpOption; @Option(name = {"--useV3Protocol"}, description = "Use V3 protocol to gateway. This is the default protocol.") - private boolean enableV3Protocol = false; - - @Option(name = {"--useV2Protocol"}, description = "Use old V2 protocol to gateway. This option is deprecated.") - private boolean enableV2Protocol = false; + private boolean enableV3Protocol = true; @Option(name = {"--file"}, description = "The name of the input file to read.") @@ -152,10 +139,6 @@ public class CommandLineArguments { "to be pending at any given time.") private int maxPendingOperationCountArg = 10000; - @Option(name = {"--debugport"}, - description = "Deprecated, not used.") - private int debugportArg = 9988; - @Option(name = {"-v", "--verbose"}, description = "Enable verbose output of progress.") private boolean verboaseArg = false; @@ -244,7 +227,6 @@ public class CommandLineArguments { .setHostnameVerifier(insecure ? NoopHostnameVerifier.INSTANCE : SSLConnectionSocketFactory.getDefaultHostnameVerifier()) .setNumPersistentConnectionsPerEndpoint(16) - .setEnableV3Protocol(! enableV2Protocol) .setUseCompression(useCompressionArg) .setMaxRetries(noRetryArg ? 0 : 100) .setMinTimeBetweenRetries(retrydelayArg, TimeUnit.SECONDS) diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/FeedParamsTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/FeedParamsTest.java index f2dd9a44878..3af73dbf47a 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/FeedParamsTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/FeedParamsTest.java @@ -20,7 +20,7 @@ public class FeedParamsTest { public void testDefaults() { FeedParams params = new FeedParams.Builder().build(); - assertThat(params.getDataFormat(), equalTo(FeedParams.DataFormat.XML_UTF8)); + assertThat(params.getDataFormat(), equalTo(FeedParams.DataFormat.JSON_UTF8)); assertThat(params.getMaxChunkSizeBytes(), is(50 * 1024)); assertThat(params.getRoute(), nullValue()); assertThat(params.getServerTimeout(TimeUnit.SECONDS), is(180L)); diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionTest.java index 1e2d37884ef..90d2fa28833 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionTest.java @@ -55,7 +55,6 @@ public class ApacheGatewayConnectionTest { final FeedParams feedParams = new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.JSON_UTF8).build(); final String clusterSpecificRoute = ""; final ConnectionParams connectionParams = new ConnectionParams.Builder() - .setEnableV3Protocol(true) .build(); final List<Document> documents = new ArrayList<>(); @@ -90,7 +89,6 @@ public class ApacheGatewayConnectionTest { final FeedParams feedParams = new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.JSON_UTF8).build(); final String clusterSpecificRoute = ""; final ConnectionParams connectionParams = new ConnectionParams.Builder() - .setEnableV3Protocol(true) .build(); // This is the fake server, returns wrong session Id. @@ -115,7 +113,6 @@ public class ApacheGatewayConnectionTest { final FeedParams feedParams = new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.JSON_UTF8).build(); final String clusterSpecificRoute = ""; final ConnectionParams connectionParams = new ConnectionParams.Builder() - .setEnableV3Protocol(true) .build(); final ApacheGatewayConnection.HttpClientFactory mockFactory = @@ -192,7 +189,7 @@ public class ApacheGatewayConnectionTest { @Test public void testCompressedWriteOperations() throws Exception { final Endpoint endpoint = Endpoint.create("hostname", 666, false); - final FeedParams feedParams = new FeedParams.Builder().build(); + final FeedParams feedParams = new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.XML_UTF8).build(); final String clusterSpecificRoute = ""; final ConnectionParams connectionParams = new ConnectionParams.Builder() .setUseCompression(true) diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessorTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessorTest.java index 3a335aa5da0..5c58aa347e8 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessorTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessorTest.java @@ -124,7 +124,7 @@ public class OperationProcessorTest { SessionParams sessionParams = new SessionParams.Builder() .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("host")).build()) .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("host")).build()) - .setConnectionParams(new ConnectionParams.Builder().setEnableV3Protocol(true).build()) + .setConnectionParams(new ConnectionParams.Builder().build()) .build(); OperationProcessor operationProcessor = new OperationProcessor( new IncompleteResultsThrottler(1000, 1000, null, null), @@ -161,7 +161,7 @@ public class OperationProcessorTest { public void testBlockingOfOperationsToSameDocIdWithTwoOperations() { SessionParams sessionParams = new SessionParams.Builder() .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("host")).build()) - .setConnectionParams(new ConnectionParams.Builder().setEnableV3Protocol(true).build()) + .setConnectionParams(new ConnectionParams.Builder().build()) .build(); OperationProcessor operationProcessor = new OperationProcessor( @@ -194,7 +194,7 @@ public class OperationProcessorTest { public void testBlockingOfOperationsToSameDocIdMany() { SessionParams sessionParams = new SessionParams.Builder() .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("host")).build()) - .setConnectionParams(new ConnectionParams.Builder().setEnableV3Protocol(true).build()) + .setConnectionParams(new ConnectionParams.Builder().build()) .build(); OperationProcessor operationProcessor = new OperationProcessor( @@ -229,7 +229,7 @@ public class OperationProcessorTest { Endpoint endpoint = Endpoint.create("host"); SessionParams sessionParams = new SessionParams.Builder() .addCluster(new Cluster.Builder().addEndpoint(endpoint).build()) - .setConnectionParams(new ConnectionParams.Builder().setEnableV3Protocol(true).build()) + .setConnectionParams(new ConnectionParams.Builder().build()) .build(); OperationProcessor operationProcessor = new OperationProcessor( diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java index fd10aeb6100..ec80dd53a4b 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java @@ -95,7 +95,6 @@ public class CommandLineArgumentsTest { assertThat(params.getClusters().get(0).getEndpoints().get(0).isUseSsl(), is(false)); assertThat(params.getConnectionParams().getUseCompression(), is(false)); assertThat(params.getConnectionParams().getNumPersistentConnectionsPerEndpoint(), is(16)); - assertThat(params.getConnectionParams().isEnableV3Protocol(), is(true)); assertThat(params.getFeedParams().getRoute(), is("default")); assertThat(params.getFeedParams().getDataFormat(), is(FeedParams.DataFormat.XML_UTF8)); assertThat(params.getFeedParams().getLocalQueueTimeOut(), is(180000L)); @@ -113,7 +112,6 @@ public class CommandLineArgumentsTest { args.add("--useCompression"); args.add("--useDynamicThrottling"); add("maxpending", "3456"); - add("debugport", "7890"); args.add("--verbose"); args.add("--useTls"); add("header", "Header-Name: Header-Value"); @@ -177,30 +175,10 @@ public class CommandLineArgumentsTest { } @Test - public void testDeprecatedUseV2Protocol() { - addMinimum(); - args.add("--useV2Protocol"); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(true /* use json */); - assertThat(params.getConnectionParams().isEnableV3Protocol(), is(false)); - } - - @Test public void testUseV3Protocol() { addMinimum(); args.add("--useV3Protocol"); CommandLineArguments arguments = CommandLineArguments.build(asArray()); SessionParams params = arguments.createSessionParams(true /* use json */); - assertThat(params.getConnectionParams().isEnableV3Protocol(), is(true)); - } - - @Test - public void testDeprecatedUseV2ProtocolAndUseV3Protocol() { - addMinimum(); - args.add("--useV2Protocol"); - args.add("--useV3Protocol"); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(true /* use json */); - assertThat(params.getConnectionParams().isEnableV3Protocol(), is(true)); } } diff --git a/vespa_feed_perf/pom.xml b/vespa_feed_perf/pom.xml index dbb0e4cd437..c6cf0cb9edf 100644 --- a/vespa_feed_perf/pom.xml +++ b/vespa_feed_perf/pom.xml @@ -12,11 +12,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa_feed_perf</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/vespa_jersey2/pom.xml b/vespa_jersey2/pom.xml index 781075fd041..e19b6bde538 100644 --- a/vespa_jersey2/pom.xml +++ b/vespa_jersey2/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa_jersey2</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>pom</packaging> <dependencies> <dependency> diff --git a/vespaclient-container-plugin/pom.xml b/vespaclient-container-plugin/pom.xml index 150a3241153..3567e6ed1e6 100644 --- a/vespaclient-container-plugin/pom.xml +++ b/vespaclient-container-plugin/pom.xml @@ -1,18 +1,18 @@ <?xml version="1.0"?> <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <!-- Implementation of document-api in the container. --> -<!-- TODO: Rename to container-documentapi on Vespa 7 --> +<!-- TODO: Rename to container-documentapi --> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> <modelVersion>4.0.0</modelVersion> <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespaclient-container-plugin</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <dependencies> <dependency> diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java index f32b3594f28..cfa77455f41 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java @@ -31,18 +31,6 @@ public interface OperationHandler { public final Optional<Integer> concurrency; public final Optional<String> bucketSpace; - /** @deprecated Use a VisitOptions.Builder instead */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public VisitOptions(Optional<String> cluster, Optional<String> continuation, Optional<Integer> wantedDocumentCount) { - this.cluster = cluster; - this.continuation = continuation; - this.wantedDocumentCount = wantedDocumentCount; - this.fieldSet = Optional.empty(); - this.concurrency = Optional.empty(); - this.bucketSpace = Optional.empty(); - } - private VisitOptions(Builder builder) { this.cluster = Optional.ofNullable(builder.cluster); this.continuation = Optional.ofNullable(builder.continuation); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/StatusResponse.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/StatusResponse.java deleted file mode 100755 index 365f605688c..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/StatusResponse.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedhandler; - -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.metrics.MetricManager; -import com.yahoo.metrics.MetricSnapshot; -import com.yahoo.text.Utf8String; -import com.yahoo.text.XMLWriter; - -import java.io.IOException; -import java.io.OutputStream; -import java.io.OutputStreamWriter; - -/** - * @deprecated Legacy API. Will be removed in Vespa 7 - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class StatusResponse extends HttpResponse { - - MetricManager manager; - int verbosity; - int snapshotTime; - - StatusResponse(MetricManager manager, int verbosity, int snapshotTime) { - super(com.yahoo.jdisc.http.HttpResponse.Status.OK); - this.manager = manager; - this.snapshotTime = snapshotTime; - this.verbosity = verbosity; - } - - @Override - public void render(OutputStream stream) throws IOException { - XMLWriter writer = new XMLWriter(new OutputStreamWriter(stream)); - writer.openTag("status"); - if (verbosity >= 2) { - writer.attribute(new Utf8String("description"), "Metrics since start"); - } - - if (snapshotTime == 0) { - MetricSnapshot snapshot = (new MetricSnapshot( - "Total metrics from start until current time", 0, - manager.getActiveMetrics().getMetrics(), false)); - manager.getTotalMetricSnapshot().addToSnapshot(snapshot, (int)(System.currentTimeMillis() / 1000), false); - snapshot.printXml(manager, "", verbosity, writer); - } else { - try { - manager.getMetricSnapshotSet(snapshotTime).getSnapshot().printXml(manager, "", verbosity, writer); - } catch (Exception e) { - writer.openTag("error"); - writer.attribute(new Utf8String("details"), "No metric snapshot with period " + snapshotTime + - " was found. Legal snapshot periods are: " + manager.getSnapshotPeriods()); - writer.closeTag(); - } - } - writer.closeTag(); - writer.flush(); - } - - @Override - public java.lang.String getContentType() { - return "application/xml"; - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java deleted file mode 100755 index 8fe721899f9..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerCompatibility.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedhandler; - -import java.util.concurrent.Executor; -import javax.inject.Inject; - -import com.yahoo.jdisc.Metric; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; - -/** - * @deprecated Legacy API. Will be removed in Vespa 7 - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class VespaFeedHandlerCompatibility extends ThreadedHttpRequestHandler { - - private final VespaFeedHandlerGet getHandler; - private final VespaFeedHandler feedHandler; - - @Inject - public VespaFeedHandlerCompatibility(Executor executor, Metric metric, VespaFeedHandlerGet getHandler, - VespaFeedHandler feedHandler) { - super(executor, metric); - this.getHandler = getHandler; - this.feedHandler = feedHandler; - } - - @Override - public HttpResponse handle(HttpRequest request) { - boolean hasType = request.hasProperty("type"); - // If we have an ID and no document type, redirect to Get - if (request.hasProperty("id") && !hasType) { - return getHandler.handle(request); - } else { - return feedHandler.handle(request); - } - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java deleted file mode 100755 index ab8728e017b..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerGet.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedhandler; - -import java.util.Collections; -import java.util.concurrent.Executor; -import javax.inject.Inject; - -import com.yahoo.jdisc.Metric; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.search.handler.SearchHandler; - -/** - * @deprecated Legacy API. Will be removed in Vespa 7 - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class VespaFeedHandlerGet extends ThreadedHttpRequestHandler { - - private final SearchHandler searchHandler; - - @Inject - public VespaFeedHandlerGet(SearchHandler searchHandler, Executor executor, Metric metric) { - super(executor, metric, true); - this.searchHandler = searchHandler; - } - - @Override - public HttpResponse handle(HttpRequest request) { - return searchHandler.handle(new HttpRequest(request.getJDiscRequest(), request.getData(), Collections.singletonMap("searchChain", "vespaget"))); - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java deleted file mode 100755 index 9baad1a605d..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedhandler; - -import com.google.inject.Inject; -import com.yahoo.clientmetrics.RouteMetricSet; -import com.yahoo.cloud.config.ClusterListConfig; -import com.yahoo.cloud.config.SlobroksConfig; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.protect.Error; -import com.yahoo.document.DocumentId; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.feedapi.FeedContext; -import com.yahoo.feedapi.MessagePropertyProcessor; -import com.yahoo.feedapi.SingleSender; -import com.yahoo.jdisc.Metric; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.vespaclient.config.FeederConfig; - -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.util.concurrent.Executor; - -/** - * @deprecated Legacy API. Will be removed in Vespa 7 - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class VespaFeedHandlerRemove extends VespaFeedHandlerBase { - - @Inject - public VespaFeedHandlerRemove(FeederConfig feederConfig, - LoadTypeConfig loadTypeConfig, - DocumentmanagerConfig documentmanagerConfig, - SlobroksConfig slobroksConfig, - ClusterListConfig clusterListConfig, - Executor executor, - Metric metric) throws Exception { - super(feederConfig, loadTypeConfig, documentmanagerConfig, slobroksConfig, clusterListConfig, executor, metric); - } - - VespaFeedHandlerRemove(FeedContext context, Executor executor) throws Exception { - super(context, executor); - } - - @Override - public HttpResponse handle(HttpRequest request) { - if (request.getProperty("status") != null) { - return new MetricResponse(context.getMetrics().getMetricSet()); - } - - MessagePropertyProcessor.PropertySetter properties = getPropertyProcessor().buildPropertySetter(request); - String route = properties.getRoute().toString(); - FeedResponse response = new FeedResponse(new RouteMetricSet(route, null)); - SingleSender sender = new SingleSender(response, getSharedSender(route)); - sender.addMessageProcessor(properties); - - response.setAbortOnFeedError(properties.getAbortOnFeedError()); - - if (request.hasProperty("id")) { - sender.remove(new DocumentId(request.getProperty("id"))); - } else if (request.hasProperty("id[0]")) { - int index = 0; - while (request.hasProperty("id[" + index + "]")) { - sender.remove(new DocumentId(request.getProperty("id[" + index + "]"))); - ++index; - } - } - - if (request.getData() != null) { - try { - String line; - BufferedReader reader = new BufferedReader( - new InputStreamReader(getRequestInputStream(request), "UTF-8")); - while ((line = reader.readLine()) != null) { - sender.remove(new DocumentId(line)); - } - } catch (Exception e) { - response.addError(e.getClass() + ": " + e.getCause()); - } - } - - sender.done(); - long millis = getTimeoutMillis(request); - boolean completed = sender.waitForPending(millis); - if ( ! completed) - response.addError(Error.TIMEOUT, "Timed out after "+millis+" ms waiting for responses"); - return response; - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java deleted file mode 100644 index 0ca77decf22..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedhandler; - -import com.google.inject.Inject; -import com.yahoo.clientmetrics.RouteMetricSet; -import com.yahoo.cloud.config.ClusterListConfig; -import com.yahoo.cloud.config.SlobroksConfig; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.protect.Error; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.documentapi.messagebus.protocol.RemoveLocationMessage; -import com.yahoo.feedapi.FeedContext; -import com.yahoo.feedapi.MessagePropertyProcessor; -import com.yahoo.feedapi.SingleSender; -import com.yahoo.jdisc.Metric; -import com.yahoo.messagebus.routing.Route; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.vespaclient.config.FeederConfig; - -import java.util.concurrent.Executor; - -/** - * @deprecated Legacy API. Will be removed in Vespa 7 - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class VespaFeedHandlerRemoveLocation extends VespaFeedHandlerBase { - - @Inject - public VespaFeedHandlerRemoveLocation(FeederConfig feederConfig, - LoadTypeConfig loadTypeConfig, - DocumentmanagerConfig documentmanagerConfig, - SlobroksConfig slobroksConfig, - ClusterListConfig clusterListConfig, - Executor executor, Metric metric) throws Exception { - super(feederConfig, loadTypeConfig, documentmanagerConfig, slobroksConfig, clusterListConfig, executor, metric); - } - - VespaFeedHandlerRemoveLocation(FeedContext context, Executor executor) throws Exception { - super(context, executor); - } - - @Override - public HttpResponse handle(HttpRequest request) { - MessagePropertyProcessor.PropertySetter properties = getPropertyProcessor().buildPropertySetter(request); - FeedResponse response; - - if (request.getProperty("route") == null) { - if (context.getClusterList().getStorageClusters().size() == 0) { - return new FeedResponse(null).addError("No storage clusters configured and no alternate route specified."); - } else if (context.getClusterList().getStorageClusters().size() > 1) { - return new FeedResponse(null).addError("More than one storage cluster configured and no route specified."); - } else { - properties.setRoute(Route.parse(context.getClusterList().getStorageClusters().get(0).getName())); - } - } - - response = new FeedResponse(new RouteMetricSet(properties.getRoute().toString(), null)); - - SingleSender sender = new SingleSender(response, getSharedSender(properties.getRoute().toString())); - sender.addMessageProcessor(properties); - - String user = request.getProperty("user"); - String group = request.getProperty("group"); - String selection = request.getProperty("selection"); - - boolean oneFound = (user != null) ^ (group != null) ^ (selection != null); - - if (!oneFound) { - response.addError("Exactly one of \"user\", \"group\" or \"selection\" must be specified for removelocation"); - return response; - } - - if (user != null) { - selection = "id.user=" + user; - } - if (group != null) { - selection = "id.group=\"" + group + "\""; - } - - sender.send(new RemoveLocationMessage(selection)); - sender.done(); - long millis = getTimeoutMillis(request); - boolean completed = sender.waitForPending(millis); - if ( ! completed) - response.addError(Error.TIMEOUT, "Timed out after "+millis+" ms waiting for responses"); - return response; - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java deleted file mode 100755 index ac16159a9bb..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedhandler; - -import java.util.concurrent.Executor; - -import com.yahoo.cloud.config.ClusterListConfig; -import com.yahoo.cloud.config.SlobroksConfig; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.jdisc.Metric; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.feedapi.FeedContext; -import com.yahoo.metrics.MetricManager; -import com.yahoo.metrics.MetricSet; -import com.yahoo.vespaclient.config.FeederConfig; - -/** - * @deprecated Legacy API. Will be removed in Vespa 7 - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class VespaFeedHandlerStatus extends ThreadedHttpRequestHandler { - - private MetricManager manager; - - public VespaFeedHandlerStatus(FeederConfig feederConfig, - LoadTypeConfig loadTypeConfig, - DocumentmanagerConfig documentmanagerConfig, - SlobroksConfig slobroksConfig, - ClusterListConfig clusterListConfig, - Executor executor, - Metric metric) { - this(FeedContext.getInstance(feederConfig, loadTypeConfig, - documentmanagerConfig, slobroksConfig, - clusterListConfig, metric), - true, true, executor); - } - - VespaFeedHandlerStatus(FeedContext context, boolean doLog, boolean makeSnapshots, Executor executor) { - super(executor); - manager = new MetricManager(); - final MetricSet metricSet = context.getMetrics().getMetricSet(); - metricSet.unregister(); - manager.registerMetric(metricSet); - if (doLog) { - manager.addMetricToConsumer("log", "routes.total.putdocument.count"); - manager.addMetricToConsumer("log", "routes.total.removedocument.count"); - manager.addMetricToConsumer("log", "routes.total.updatedocument.count"); - manager.addMetricToConsumer("log", "routes.total.getdocument.count"); - - manager.addMetricToConsumer("log", "routes.total.putdocument.errors.total"); - manager.addMetricToConsumer("log", "routes.total.removedocument.errors.total"); - manager.addMetricToConsumer("log", "routes.total.updatedocument.errors.total"); - manager.addMetricToConsumer("log", "routes.total.getdocument.errors.total"); - - manager.addMetricToConsumer("log", "routes.total.putdocument.latency"); - manager.addMetricToConsumer("log", "routes.total.removedocument.latency"); - manager.addMetricToConsumer("log", "routes.total.updatedocument.latency"); - manager.addMetricToConsumer("log", "routes.total.getdocument.latency"); - } - - if (doLog || makeSnapshots) { - new Thread(manager).start(); - } - } - - @Override - public HttpResponse handle(HttpRequest request) { - try { - return new StatusResponse(manager, asInt(request.getProperty("verbosity"), 0), asInt(request.getProperty("snapshotperiod"), 0)); - } catch (Exception e) { - e.printStackTrace(); - return null; - } - } - - private int asInt(String value, int defaultValue) { - if (value == null) return defaultValue; - return Integer.parseInt(value); - } - - @Override - public void destroy() { - manager.stop(); - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerVisit.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerVisit.java deleted file mode 100644 index b73ea455c32..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerVisit.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedhandler; - -import java.util.Collections; -import java.util.concurrent.Executor; -import javax.inject.Inject; - -import com.yahoo.jdisc.Metric; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.search.handler.SearchHandler; - -/** - * @author thomasg - * - * @deprecated Legacy API. Will be removed in Vespa 7 - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class VespaFeedHandlerVisit extends ThreadedHttpRequestHandler { - - private final SearchHandler searchHandler; - - @Inject - public VespaFeedHandlerVisit(SearchHandler searchHandler, Executor executor, Metric metric) { - super(executor, metric, true); - this.searchHandler = searchHandler; - } - - @Override - public HttpResponse handle(HttpRequest request) { - return searchHandler.handle(new HttpRequest(request.getJDiscRequest(), request.getData(), Collections.singletonMap("searchChain", "vespavisit"))); - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/package-info.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/package-info.java deleted file mode 100644 index 857e7127b6f..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/package-info.java +++ /dev/null @@ -1,3 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// TODO: This implements the old, deprecated document http API. Remove this package on Vespa 7. -package com.yahoo.feedhandler; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/ContinuationHit.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/ContinuationHit.java deleted file mode 100755 index c66af4f02ba..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/ContinuationHit.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.documentapi.ProgressToken; -import com.yahoo.search.result.Hit; -import java.io.IOException; -import java.util.Base64; - -/** - * @deprecated - */ -@Deprecated // OK -// TODO: Remove on Vespa 7 -public class ContinuationHit extends Hit { - - private final String value; - - public ContinuationHit(ProgressToken token) { - super("continuation"); - value = token.serializeToString(); - } - - public static ProgressToken getToken(String continuation) { - return ProgressToken.fromSerializedString(continuation); - } - - public String getValue() { - return value; - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java deleted file mode 100755 index cf35c789f65..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentFieldTemplate.java +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.document.DataType; -import com.yahoo.document.Document; -import com.yahoo.document.Field; -import com.yahoo.document.datatypes.FieldValue; -import com.yahoo.document.datatypes.Raw; -import com.yahoo.io.ByteWriter; -import com.yahoo.text.XML; - -import java.io.IOException; -import java.io.Writer; - -/** - * Template used to render a single field for a single Document. Fields - * that are either of type CONTENT or RAW are written directly, while - * all other fields are wrapped in Vespa XML and escaped. - * - * @deprecated use a renderer instead - */ -@Deprecated // OK -// TODO: Remove on Vespa 7 -@SuppressWarnings("deprecation") -public class DocumentFieldTemplate extends com.yahoo.prelude.templates.UserTemplate<Writer> { - - Field field; - String contentType; - String encoding; - boolean wrapXml; - - public DocumentFieldTemplate(Field field, String contentType, String encoding, boolean wrapXml) { - super("documentfield", contentType, encoding); - this.field = field; - this.contentType = contentType; - this.encoding = encoding; - this.wrapXml = wrapXml; - } - - @Override - public void error(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - // Error shouldn't be handled by this template, but rather - // delegated to the searcher - } - - @Override - public Writer wrapWriter(Writer writer) { - /* TODO: uncomment - if (!(writer instanceof ByteWriter)) { - throw new IllegalArgumentException("ByteWriter required, but got " + writer.getClass().getName()); - } - */ - - return writer; - } - - @Override - public void header(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - if (wrapXml) { - // XML wrapping should only be used for default field rendering - writer.write("<?xml version=\"1.0\" encoding=\"" + encoding + "\"?>\n"); - writer.write("<result>"); - } - } - - @Override - public void footer(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - if (wrapXml) { - writer.write("</result>\n"); - } - } - - @Override - public void hit(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - DocumentHit hit = (DocumentHit)context.get("hit"); - Document doc = hit.getDocument(); - // Assume field existence has been checked before we ever get here. - // Also assume that relevant encoding/content type is set - // appropriately according to the request and the field's content - // type, as this is immutable in the template set. - FieldValue value = doc.getFieldValue(field); - if (field.getDataType() == DataType.RAW) { - ByteWriter bw = (ByteWriter)writer; - bw.append(((Raw) value).getByteBuffer().array()); - } else { - writer.write(XML.xmlEscape(value.toString(), false)); - } - } - - @Override - public void hitFooter(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - } - - @Override - public void noHits(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentHit.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentHit.java deleted file mode 100755 index 514cf055e84..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentHit.java +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.document.Document; -import com.yahoo.document.Field; -import com.yahoo.document.datatypes.FieldValue; -import com.yahoo.search.result.Hit; - -import java.util.Iterator; -import java.util.Map; - -@Deprecated // OK -// TODO: Remove on Vespa 7 -public class DocumentHit extends Hit { - - private Document document; - private int index; - - public DocumentHit(Document document, int index) { - super(document.getId().toString()); - this.document = document; - this.index = index; - } - - public void populateHitFields() { - // Create hit fields for all document fields - Iterator<Map.Entry<Field, FieldValue>> fieldIter = document.iterator(); - while (fieldIter.hasNext()) { - Map.Entry<Field, FieldValue> field = fieldIter.next(); - setField(field.getKey().getName(), field.getValue()); - } - - // Assign an explicit document id field - setField("documentid", document.getId().toString()); - } - - public Document getDocument() { - return document; - } - - public int getIndex() { - return index; - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentRemoveHit.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentRemoveHit.java deleted file mode 100644 index 73363e68e2b..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentRemoveHit.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.document.DocumentId; -import com.yahoo.search.result.Hit; - -@Deprecated // OK -// TODO: Remove on Vespa 7 -public class DocumentRemoveHit extends Hit { - - private final DocumentId idOfRemovedDoc; - - public DocumentRemoveHit(DocumentId idOfRemovedDoc) { - super(idOfRemovedDoc.toString()); - this.idOfRemovedDoc = idOfRemovedDoc; - setField("documentid", idOfRemovedDoc.toString()); - } - - public DocumentId getIdOfRemovedDoc() { - return idOfRemovedDoc; - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java deleted file mode 100755 index 2366e7059bd..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/DocumentXMLTemplate.java +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.log.LogLevel; -import com.yahoo.search.Result; -import com.yahoo.search.result.ErrorHit; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.HitGroup; -import com.yahoo.search.result.Hit; -import com.yahoo.text.XML; - -import java.io.IOException; -import java.io.Writer; -import java.util.logging.Logger; - -/** - * @deprecated use a renderer instead - */ -@Deprecated // OK -// TODO: Remove on Vespa 7 -@SuppressWarnings("deprecation") -public class DocumentXMLTemplate extends com.yahoo.prelude.templates.UserTemplate<Writer> { - - private static final Logger log = Logger.getLogger(DocumentXMLTemplate.class.getName()); - - public DocumentXMLTemplate() { - super("vespa_xml"); - } - - public DocumentXMLTemplate(String mimeType, String encoding) { - super("vespa_xml", mimeType, encoding); - } - - private void writeErrorMessage(Writer writer, String type, int code, - String message, String detailedMessage) throws IOException { - writer.write("<error type=\"" + type + "\" code=\"" + code + "\" message=\""); - writer.write(XML.xmlEscape(message, true)); - if (detailedMessage != null) { - writer.write(": "); - writer.write(XML.xmlEscape(detailedMessage, true)); - } - writer.write("\"/>\n"); - } - - private void writeGenericErrorMessage(Writer writer, ErrorMessage message) throws IOException { - // A bit dirty, but we don't have to support many different types - if (message instanceof MessageBusErrorMessage) { - writeErrorMessage(writer, "messagebus", - ((MessageBusErrorMessage)message).getMessageBusCode(), - message.getMessage(), message.getDetailedMessage()); - } else { - writeErrorMessage(writer, "searcher", message.getCode(), - message.getMessage(), message.getDetailedMessage()); - } - } - - @Override - public void error(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - writer.write("<errors>\n"); - // If the error contains no error hits, use a single error with the main - // code and description. Otherwise, use the error hits explicitly - ErrorHit errorHit = ((Result)context.get("result")).hits().getErrorHit(); - if (errorHit == null || errorHit.errors().isEmpty()) { - ErrorMessage message = ((Result)context.get("result")).hits().getError(); - writeGenericErrorMessage(writer, message); - } else { - for (ErrorMessage message : errorHit.errors()) { - writeGenericErrorMessage(writer, message); - } - } - writer.write("</errors>\n"); - } - - @Override - public void header(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - writer.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"); - writer.write("<result>\n"); - HitGroup rootGroup = ((Result) context.get("result")).hits(); - if (rootGroup.getField(VisitSearcher.VISITOR_CONTINUATION_TOKEN_FIELDNAME) != null) { - writer.write("<continuation>" + rootGroup.getField(VisitSearcher.VISITOR_CONTINUATION_TOKEN_FIELDNAME) + "</continuation>"); - } - } - - @Override - public void footer(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - writer.write("</result>\n"); - } - - @Override - public void hit(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - Hit hit = (Hit)context.get("hit"); - if (hit instanceof DocumentHit) { - DocumentHit docHit = (DocumentHit) hit; - if (docHit.getDocument() != null) { - writer.write(docHit.getDocument().toXML(" ")); - } - } else if (hit instanceof DocumentRemoveHit) { - writeDocumentRemoveHit(writer, (DocumentRemoveHit) hit); - } else { - log.log(LogLevel.WARNING, "Cannot render document XML; expected hit of type " + - "com.yahoo.storage.searcher.Document[Remove]Hit, got " + hit.getClass().getName() + - ". Is there another backend searcher present?"); - } - } - - private void writeDocumentRemoveHit(Writer writer, DocumentRemoveHit remove) throws IOException { - writer.write("<remove documentid=\""); - writer.write(XML.xmlEscape(remove.getIdOfRemovedDoc().toString())); - writer.write("\"/>\n"); - } - - @Override - public void hitFooter(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - } - - @Override - public void noHits(com.yahoo.prelude.templates.Context context, Writer writer) throws IOException { - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/GetSearcher.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/GetSearcher.java deleted file mode 100755 index 869a2f9317a..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/GetSearcher.java +++ /dev/null @@ -1,508 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.google.inject.Inject; -import com.yahoo.cloud.config.ClusterListConfig; -import com.yahoo.cloud.config.SlobroksConfig; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.jdisc.Metric; -import com.yahoo.processing.request.CompoundName; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.document.DataType; -import com.yahoo.document.Document; -import com.yahoo.document.DocumentId; -import com.yahoo.document.Field; -import com.yahoo.document.datatypes.FieldValue; -import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; -import com.yahoo.feedapi.*; -import com.yahoo.log.LogLevel; -import com.yahoo.messagebus.Reply; -import com.yahoo.search.Query; -import com.yahoo.search.query.Properties; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.result.DefaultErrorHit; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.vespaclient.config.FeederConfig; - -import java.io.*; -import java.util.*; -import java.util.logging.Logger; -import java.util.zip.GZIPInputStream; - -/** - * Searcher component to make GET requests to a content cluster. - * <p> - * Document ID must be given either as 1 "id=docid" query parameter - * for single-document GETs and 1-n "id[0]=docid_1&id[1]=...&id[n-1]=docid_n" - * parameters for multi-document GETs. - * - * <p> - * Standard gateway query parameters are implicitly supported: - * priority, timeout, route - * - * <p> - * The searcher also accepts the following (optional) query parameters: - * headersonly=true|false (default: false) - * For specifying whether or not to return only header fields. - * - * <p> - * field=string (default: no parameter specified) - * For getting a single document field. - * - * <p> - * contenttype=string (default: no content type specified) - * For specifiying the returned HTTP header content type for a returned - * document field's content. field must also be specified. - * @deprecated do not use - */ -@SuppressWarnings("deprecation") -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class GetSearcher extends Searcher { - - private static final Logger log = Logger.getLogger(GetSearcher.class.getName()); - - private static final CompoundName ID = new CompoundName("id"); - private static final CompoundName HEADERS_ONLY = new CompoundName("headersonly"); - private static final CompoundName POPULATE_HIT_FIELDS = new CompoundName("populatehitfields"); - private static final CompoundName FIELDSET = new CompoundName("fieldset"); - private static final CompoundName FIELD = new CompoundName("field"); - private static final CompoundName CONTENT_TYPE = new CompoundName("contenttype"); - private static final CompoundName TIEMOUT = new CompoundName("timeout"); - - FeedContext context; - - private final long defaultTimeoutMillis; - - private class GetResponse implements SharedSender.ResultCallback { - - /** - * We have to maintain the same ordering of results as that - * given in the request. Do this by remembering the index of - * each requested document ID. - */ - private Map<String, Integer> ordering; - private List<DocumentHit> documentHits = new ArrayList<>(); - private List<DefaultErrorHit> errorHits = new ArrayList<>(); - private List<Reply> replies = new ArrayList<>(); - private final SharedSender.Pending pendingNumber = new SharedSender.Pending(); - - public GetResponse(List<String> documentIds) { - ordering = new HashMap<>(documentIds.size()); - for (int i = 0; i < documentIds.size(); ++i) { - ordering.put(documentIds.get(i), i); - } - } - - public boolean isAborted() { - return false; - } - - private String stackTraceFromException(Exception e) { - StringWriter sw = new StringWriter(); - PrintWriter ps = new PrintWriter(sw); - e.printStackTrace(ps); - ps.flush(); - return sw.toString(); - } - - public boolean handleReply(Reply reply) { - if ((reply.getTrace().getLevel() > 0) && log.isLoggable(LogLevel.DEBUG)) { - String str = reply.getTrace().toString(); - log.log(LogLevel.DEBUG, str); - } - replies.add(reply); - return true; - } - public SharedSender.Pending getPending() { return pendingNumber; } - - private void processReplies() { - for (Reply reply : replies) { - processReply(reply); - } - } - private void processReply(Reply reply) { - if (!reply.hasErrors()) { - try { - addDocumentHit(reply); - } catch (Exception e) { - String msg = "Got exception of type " + e.getClass().getName() - + " during document deserialization: " + e.getMessage(); - errorHits.add(new DefaultErrorHit("GetSearcher", ErrorMessage.createInternalServerError(msg))); - log.log(LogLevel.DEBUG, "Got exception during document deserialization: " + stackTraceFromException(e)); - } - } else { - errorHits.add(new DefaultErrorHit("GetSearcher", new MessageBusErrorMessage( - reply.getError(0).getCode(), 0, reply.getError(0).getMessage()))); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Received error reply with message " + reply.getError(0).getMessage()); - } - } - } - - private void addDocumentHit(Reply reply) { - Document doc = ((GetDocumentReply)reply).getDocument(); - GetDocumentMessage msg = (GetDocumentMessage)reply.getMessage(); - Integer index = ordering.get(msg.getDocumentId().toString()); - if (index == null) { // Shouldn't happen - throw new IllegalStateException("Received GetDocumentReply for unknown document: " - + doc.getId().toString()); - } - if (doc != null) { - documentHits.add(new DocumentHit(doc, index)); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Received GetDocumentReply for " - + doc.getId().toString()); - } - } else { - // Don't add a hit for documents that can't be found - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Received empty (not found) GetDocumentReply for " - + msg.getDocumentId().toString()); - } - } - } - - private class IndexComparator implements Comparator<DocumentHit> { - public int compare(DocumentHit o1, DocumentHit o2) { - return o1.getIndex() - o2.getIndex(); - } - } - - public void addHitsToResult(Result result, boolean populateHitFields) { - for (DefaultErrorHit hit : errorHits) { - result.hits().add(hit); - } - // Sort document hits according to their request index - Collections.sort(documentHits, new IndexComparator()); - for (DocumentHit hit : documentHits) { - if (populateHitFields) { - hit.populateHitFields(); - } - result.hits().add(hit); - } - result.setTotalHitCount(documentHits.size()); - } - - public List<DocumentHit> getDocumentHits() { - return documentHits; - } - - public List<DefaultErrorHit> getErrorHits() { - return errorHits; - } - } - - @Inject - public GetSearcher(FeederConfig feederConfig, - LoadTypeConfig loadTypeConfig, - DocumentmanagerConfig documentmanagerConfig, - SlobroksConfig slobroksConfig, - ClusterListConfig clusterListConfig, - Metric metric) - throws Exception - { - this(FeedContext.getInstance(feederConfig, loadTypeConfig, - documentmanagerConfig, slobroksConfig, - clusterListConfig, metric), - (long)(feederConfig.timeout() * 1000)); - } - - GetSearcher(FeedContext context) throws Exception { - this.context = context; - this.defaultTimeoutMillis = context.getPropertyProcessor().getDefaultTimeoutMillis(); - } - - GetSearcher(FeedContext context, long defaultTimeoutMillis) throws Exception { - this.context = context; - this.defaultTimeoutMillis = defaultTimeoutMillis; - } - - private static void postValidateDocumentIdParameters(Properties properties, int arrayIdsFound) throws Exception { - for (Map.Entry<String, Object> kv : properties.listProperties().entrySet()) { - if (!kv.getKey().startsWith("id[")) { - continue; - } - if (!kv.getKey().endsWith("]")) { - throw new IllegalArgumentException("Malformed document ID array parameter"); - } - String indexStr = kv.getKey().substring(3, kv.getKey().length() - 1); - int idx = Integer.parseInt(indexStr); - if (idx >= arrayIdsFound) { - throw new IllegalArgumentException("query contains document ID array " + - "that is not zero-based and/or linearly increasing"); - } - } - } - - private List<String> getDocumentIds(Query query) throws Exception { - Properties properties = query.properties(); - List<String> docIds = new ArrayList<>(); - - // First check for regular "id=XX" syntax. If found, return vector with that - // document id only - String singleId = properties.getString(ID); - - int index = 0; - if (singleId != null) { - docIds.add(singleId); - } else { - // Check for id[0]=XX&id[1]=YY...id[n]=ZZ syntax. Indices always start - // at 0 and are always increased by 1. - while (true) { - String docId = properties.getString("id[" + index + "]"); - if (docId == null) { - break; - } - docIds.add(docId); - ++index; - } - postValidateDocumentIdParameters(properties, index); - } - - handleData(query.getHttpRequest(), docIds); - return docIds; - } - - private void handleData(HttpRequest request, List<String> docIds) throws IOException { - if (request.getData() != null) { - InputStream input; - if ("gzip".equals(request.getHeader("Content-Encoding"))) { - input = new GZIPInputStream(request.getData()); - } else { - input = request.getData(); - } - InputStreamReader reader = new InputStreamReader(input, "UTF-8"); - BufferedReader lineReader = new BufferedReader(reader); - String line; - while ((line = lineReader.readLine()) != null) { - docIds.add(line); - } - } - } - - private void handleFieldFiltering(GetResponse response, Result result, - String fieldName, String contentType, - boolean headersOnly) { - - if (response.getDocumentHits().isEmpty()) { - result.hits().addError(ErrorMessage.createNotFound( - "Document not found, could not return field '" + fieldName + "'")); - return; - } - - if (result.hits().getErrorHit() == null) { - Document doc = response.getDocumentHits().get(0).getDocument(); - Field field = doc.getDataType().getField(fieldName); - boolean wrapXml = false; - - if (field == null) { - result.hits().addError(ErrorMessage.createIllegalQuery( - "Field '" + fieldName + "' not found in document type")); - return; - } - FieldValue value = doc.getFieldValue(field); - // If the field exists but hasn't been set in this document, the - // content will be null. We treat this as an error. - if (value == null) { - if (!field.isHeader() && headersOnly) { - // TODO(vekterli): make this work with field sets as well. - result.hits().addError(ErrorMessage.createInvalidQueryParameter( - "Field '" + fieldName + "' is located in document body, but headersonly " - + "prevents it from being retrieved in " + doc.getId().toString())); - } else { - result.hits().addError(ErrorMessage.createNotFound( - "Field '" + fieldName + "' found in document type, but had " - + "no content in " + doc.getId().toString())); - } - return; - } - String encoding = null; - if (field.getDataType() == DataType.RAW) { - if (contentType == null) { - contentType = "application/octet-stream"; - } - encoding = "ISO-8859-1"; - } else { - // By default, return field wrapped in a blanket of vespa XML - contentType = "text/xml"; - wrapXml = true; - } - if (encoding == null) { - // Encoding doesn't matter for binary content, since we're always - // writing directly to the byte buffer and not through a charset - // encoder. Presumably, the client is intelligent enough to not - // attempt to UTF-8 decode binary data. - encoding = "UTF-8"; - } - // Add hit now that we know there aren't any field errors. Otherwise, - // there would be both an error hit and a document hit in the result - response.addHitsToResult(result, false); - // Override Vespa XML template - result.getTemplating().setTemplates(new DocumentFieldTemplate(field, contentType, encoding, wrapXml)); - } - // else: return with error hit, invoking regular Vespa XML error template - } - - private void validateParameters(String fieldName, String contentType, - List<String> documentIds) { - // Content-type only makes sense for single document queries with a field - // set - if (contentType != null) { - if (documentIds.size() > 1) { - throw new IllegalArgumentException( - "contenttype parameter only valid for single document id query"); - } - if (fieldName == null) { - throw new IllegalArgumentException( - "contenttype set without document field being specified"); - } - } - if (fieldName != null && documentIds.size() > 1) { - throw new IllegalArgumentException( - "Field only valid for single document id query"); - } - } - - // For unit testing - protected MessagePropertyProcessor getMessagePropertyProcessor() { - return context.getPropertyProcessor(); - } - - private void doGetDocuments(Query query, Result result, List<String> documentIds) { - GetResponse response = new GetResponse(documentIds); - Properties properties = query.properties(); - - boolean headersOnly = properties.getBoolean(HEADERS_ONLY, false); - boolean populateHitFields = properties.getBoolean(POPULATE_HIT_FIELDS, false); - String fieldSet = properties.getString(FIELDSET); - String fieldName = properties.getString(FIELD); - String contentType = properties.getString(CONTENT_TYPE); - long timeoutMillis = properties.getString(TIEMOUT) != null ? query.getTimeout() : defaultTimeoutMillis; - - if (fieldSet == null) { - fieldSet = headersOnly ? "[header]" : "[all]"; - } - - validateParameters(fieldName, contentType, documentIds); - - MessagePropertyProcessor.PropertySetter propertySetter; - propertySetter = context.getPropertyProcessor().buildPropertySetter(query.getHttpRequest()); - - SingleSender sender = new SingleSender(response, context.getSharedSender(propertySetter.getRoute().toString())); - sender.addMessageProcessor(propertySetter); - - sendDocumentGetMessages(documentIds, fieldSet, sender); - // Twiddle thumbs until we've received a reply for all documents - sender.done(); - boolean completed = sender.waitForPending(timeoutMillis); - if ( ! completed) { - result.hits().addError(ErrorMessage.createTimeout( - "Timed out after waiting "+timeoutMillis+" ms for responses")); - } - response.processReplies(); - if (fieldName != null) { - handleFieldFiltering(response, result, fieldName, contentType, headersOnly); - } else { - response.addHitsToResult(result, populateHitFields); - } - } - - private void sendDocumentGetMessages(List<String> documentIds, String fieldSet, SingleSender sender) { - for (String docIdStr : documentIds) { - DocumentId docId = new DocumentId(docIdStr); - GetDocumentMessage getMsg = new GetDocumentMessage(docId, fieldSet); - - sender.send(getMsg); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Sent GetDocumentMessage for " - + docId.toString()); - } - } - } - - boolean verifyBackendDocumentHitsOnly(Result result) { - if (result.hits().size() != 0) { - log.log(LogLevel.DEBUG, "Result had hits after being sent down"); - for (int i = 0; i < result.hits().size(); ++i) { - if (!(result.hits().get(i) instanceof DocumentHit)) { - log.log(LogLevel.WARNING, "Got hit from backend searcher which was " - + "not a com.yahoo.storage.searcher.DocumentHit instance: " - + result.hits().get(i).getClass().getName()); - return false; - } - } - } - return true; - } - - @Override - public Result search(Query query, Execution execution) { - // Pass through to next searcher - Result result = execution.search(query); - - List<String> documentIds; - try { - documentIds = getDocumentIds(query); - } catch (Exception e) { - setOutputFormat(query, result); - result.hits().addError(ErrorMessage.createIllegalQuery(e.getClass().getName() + ": " + e.getMessage())); - return result; - } - // Early-out for pass-through queries - if (documentIds.isEmpty()) { - return result; - } - // Make sure we don't try to combine non-document hits and document hits - // in the same result set. - if (!verifyBackendDocumentHitsOnly(result)) { - result = new Result(query); // Don't include unknown hits - setOutputFormat(query, result); - result.hits().addError(ErrorMessage.createInternalServerError( - "A backend searcher to com.yahoo.storage.searcher.GetSearcher " + - "returned a hit that was not an instance of com.yahoo.storage.searcher.DocumentHit. " + - "Only DocumentHit instances are supported in the backend hit result set when doing " + - "queries that contain document identifier sets recognised by the Get Searcher.")); - return result; - } - setOutputFormat(query, result); - // Do not propagate exceptions back up, as we want to have all errors - // be reported using the proper template - try { - doGetDocuments(query, result, documentIds); - query.setHits(result.hits().size()); - } catch (IllegalArgumentException e) { - result.hits().addError(ErrorMessage.createIllegalQuery(e.getClass().getName() + ": " + e.getMessage())); - } catch (Exception e) { - result.hits().addError(ErrorMessage.createUnspecifiedError(e.getClass().getName() + ": " + e.getMessage())); - } - - return result; - } - - private static final CompoundName formatShortcut = new CompoundName("format"); - private static final CompoundName format = new CompoundName("presentation.format"); - - /** - * Use custom XML output format unless the default JSON renderer is specified in the request. - */ - @SuppressWarnings("deprecation") - static void setOutputFormat(Query query, Result result) { - if (getRequestProperty(formatShortcut, "", query).equals("JsonRenderer")) return; - if (getRequestProperty(format, "", query).equals("JsonRenderer")) return; - if (getRequestProperty(formatShortcut, "", query).equals("json")) return; - if (getRequestProperty(format, "", query).equals("json")) return; - result.getTemplating().setTemplates(new DocumentXMLTemplate()); - } - - private static String getRequestProperty(CompoundName propertyName, String defaultValue, Query query) { - String propertyValue = query.getHttpRequest().getProperty(propertyName.toString()); - if (propertyValue == null) return defaultValue; - return propertyValue; - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/MessageBusErrorMessage.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/MessageBusErrorMessage.java deleted file mode 100755 index ad93645e55e..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/MessageBusErrorMessage.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.search.result.ErrorMessage; - -/** - * Simple ErrorMessage extension that includes a message bus error code, not - * just the searcher error code (which isn't very useful for a Vespa XML consumer) - */ -@Deprecated // OK -// TODO: Remove on Vespa 7 -public class MessageBusErrorMessage extends ErrorMessage { - - private int mbusCode; - - public MessageBusErrorMessage(int mbusCode, int qrsCode, String message) { - super(qrsCode, message); - this.mbusCode = mbusCode; - } - - public MessageBusErrorMessage(int mbusCode, int qrsCode, String message, String detailedMessage) { - super(qrsCode, message, detailedMessage); - this.mbusCode = mbusCode; - } - - public MessageBusErrorMessage(int mbusCode, int qrsCode, String message, String detailedMessage, Throwable cause) { - super(qrsCode, message, detailedMessage, cause); - this.mbusCode = mbusCode; - } - - public int getMessageBusCode() { - return mbusCode; - } - - public void setMessageBusCode(int code) { - this.mbusCode = code; - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/VisitSearcher.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/VisitSearcher.java deleted file mode 100644 index e330b76fa4b..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/VisitSearcher.java +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.cloud.config.ClusterListConfig; -import com.yahoo.cloud.config.SlobroksConfig; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.component.ComponentId; -import com.yahoo.component.ComponentSpecification; -import com.yahoo.document.Document; -import com.yahoo.document.DocumentId; -import com.yahoo.documentapi.*; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.feedapi.FeedContext; -import com.yahoo.feedapi.MessagePropertyProcessor; -import com.yahoo.jdisc.Metric; -import com.yahoo.messagebus.StaticThrottlePolicy; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.rendering.RendererRegistry; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.vdslib.VisitorOrdering; -import com.yahoo.vespaclient.ClusterDef; -import com.yahoo.vespaclient.config.FeederConfig; - -/** - * A searcher that allows you to iterate through a storage cluster using visiting. - * - * @deprecated do not use - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class VisitSearcher extends Searcher { - - public static final String VISITOR_CONTINUATION_TOKEN_FIELDNAME = "visitorContinuationToken"; - FeedContext context; - - public VisitSearcher(FeederConfig feederConfig, - LoadTypeConfig loadTypeConfig, - DocumentmanagerConfig documentmanagerConfig, - SlobroksConfig slobroksConfig, - ClusterListConfig clusterListConfig, - Metric metric) - throws Exception - { - this(FeedContext.getInstance(feederConfig, loadTypeConfig, - documentmanagerConfig, slobroksConfig, - clusterListConfig, metric)); - } - - VisitSearcher(FeedContext context) throws Exception { - this.context = context; - } - - class HitDataHandler extends DumpVisitorDataHandler { - private final Result result; - private final boolean populateHits; - private final Object monitor = new Object(); - - HitDataHandler(Result result, boolean populateHits) { - this.result = result; - this.populateHits = populateHits; - } - - @Override - public void onDocument(Document document, long l) { - final DocumentHit hit = new DocumentHit(document, 0); - if (populateHits) { - hit.populateHitFields(); - } - synchronized (monitor) { - result.hits().add(hit); - } - } - - @Override - public void onRemove(DocumentId documentId) { - final DocumentRemoveHit hit = new DocumentRemoveHit(documentId); - synchronized (monitor) { - result.hits().add(hit); - } - } - } - - public VisitorParameters getVisitorParameters(Query query, Result result) throws Exception { - String documentSelection = query.properties().getString("visit.selection"); - if (documentSelection == null) { - documentSelection = ""; - } - - VisitorParameters params = new VisitorParameters(documentSelection); - params.setMaxBucketsPerVisitor(query.properties().getInteger("visit.maxBucketsPerVisitor", 1)); - params.setMaxPending(query.properties().getInteger("visit.maxPendingMessagesPerVisitor", 32)); - params.setMaxFirstPassHits(query.properties().getInteger("visit.approxMaxDocs", 1)); - params.setMaxTotalHits(query.properties().getInteger("visit.approxMaxDocs", 1)); - params.setThrottlePolicy(new StaticThrottlePolicy().setMaxPendingCount( - query.properties().getInteger("visit.maxPendingVisitors", 1))); - params.setToTimestamp(query.properties().getLong("visit.toTimestamp", 0L)); - params.setFromTimestamp(query.properties().getLong("visit.fromTimestamp", 0L)); - - String pri = query.properties().getString("visit.priority"); - if (pri != null) { - params.setPriority(DocumentProtocol.Priority.valueOf(pri)); - } - - if (query.properties().getBoolean("visit.visitInconsistentBuckets")) { - params.visitInconsistentBuckets(true); - } - - String ordering = query.properties().getString("visit.order"); - if (!"ascending".equalsIgnoreCase(ordering)) { - params.setVisitorOrdering(VisitorOrdering.ASCENDING); - } else { - params.setVisitorOrdering(VisitorOrdering.DESCENDING); - } - - String remoteCluster = query.properties().getString("visit.dataHandler"); - if (remoteCluster != null) { - params.setRemoteDataHandler(remoteCluster); - } else { - params.setLocalDataHandler(new HitDataHandler( - result, query.properties().getBoolean("populatehitfields", false))); - } - - String fieldSet = query.properties().getString("visit.fieldSet"); - if (fieldSet != null) { - params.fieldSet(fieldSet); - } - - String continuation = query.properties().getString("visit.continuation"); - if (continuation != null) { - params.setResumeToken(ContinuationHit.getToken(continuation)); - } - - params.setVisitRemoves(query.properties().getBoolean("visit.visitRemoves")); - - MessagePropertyProcessor.PropertySetter propertySetter; - propertySetter = context.getPropertyProcessor().buildPropertySetter(query.getHttpRequest()); - - propertySetter.process(params); - - if (context.getClusterList().getStorageClusters().size() == 0) { - throw new IllegalArgumentException("No content clusters have been defined"); - } - - String route = query.properties().getString("visit.cluster"); - ClusterDef found = null; - if (route != null) { - String names = ""; - for (ClusterDef c : context.getClusterList().getStorageClusters()) { - if (c.getName().equals(route)) { - found = c; - } - if (!names.isEmpty()) { - names += ", "; - } - names += c.getName(); - } - if (found == null) { - throw new IllegalArgumentException("Your vespa cluster contains the storage clusters " + names + ", not " + route + ". Please select a valid vespa cluster."); - } - } else if (context.getClusterList().getStorageClusters().size() == 1) { - found = context.getClusterList().getStorageClusters().get(0); - } else { - throw new IllegalArgumentException("Multiple content clusters are defined, select one using the \"visit.cluster\" option"); - } - - params.setRoute("[Storage:cluster=" + found.getName() + ";clusterconfigid=" + found.getConfigId() + "]"); - return params; - } - - @Override - public Result search(Query query, Execution execution) { - Result result = execution.search(query); - - VisitorParameters parameters; - - try { - parameters = getVisitorParameters(query, result); - } catch (Exception e) { - return new Result(query, ErrorMessage.createBadRequest("Illegal parameters: " + e.toString())); - } - - if (parameters != null) { - VisitorSession session = context.getSessionFactory().createVisitorSession(parameters); - - try { - if (!session.waitUntilDone(query.getTimeout())) { - return new Result(query, ErrorMessage.createTimeout("Visitor timed out")); - } - - ProgressToken token = session.getProgress(); - if (!token.isFinished()) { - final ContinuationHit continuation = new ContinuationHit(token); - result.hits().setField(VISITOR_CONTINUATION_TOKEN_FIELDNAME, continuation.getValue()); - } - } catch (InterruptedException e) { - } finally { - session.destroy(); - } - } - - GetSearcher.setOutputFormat(query, result); - query.setHits(result.hits().size()); - return result; - } - -} diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/package-info.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/package-info.java deleted file mode 100644 index 581afb0b1c8..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/package-info.java +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// TODO: This implements the old, deprecated document http API. Remove this package on Vespa 7. -@ExportPackage -package com.yahoo.storage.searcher; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java index 5294545ad50..bd7d195b48b 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandler.java @@ -51,16 +51,8 @@ import java.util.zip.GZIPInputStream; */ public class FeedHandler extends LoggingRequestHandler { - private final ExecutorService workers = Executors.newCachedThreadPool(ThreadFactoryFactory.getThreadFactory("feedhandler")); - private final DocumentTypeManager docTypeManager; - private final Map<String, ClientState> clients; - private final ScheduledThreadPoolExecutor cron; - private final SessionCache sessionCache; protected final ReplyHandler feedReplyHandler; - private final AtomicLong sessionId; - private final Metric metric; - private static final List<Integer> serverSupportedVersions = Collections.unmodifiableList(Arrays.asList(2, 3)); - private final String localHostname; + private static final List<Integer> serverSupportedVersions = Collections.unmodifiableList(Arrays.asList(3)); private final FeedHandlerV3 feedHandlerV3; @Inject @@ -73,46 +65,7 @@ public class FeedHandler extends LoggingRequestHandler { super(parentCtx); DocumentApiMetrics metricsHelper = new DocumentApiMetrics(metricReceiver, "vespa.http.server"); feedHandlerV3 = new FeedHandlerV3(parentCtx, documentManagerConfig, sessionCache, threadpoolConfig, metricsHelper); - docTypeManager = createDocumentManager(documentManagerConfig); - clients = new HashMap<>(); - this.sessionCache = sessionCache; - sessionId = new AtomicLong(ThreadLocalRandom.current().nextLong()); feedReplyHandler = new FeedReplyReader(parentCtx.getMetric(), metricsHelper); - cron = new ScheduledThreadPoolExecutor(1, ThreadFactoryFactory.getThreadFactory("feedhandler.cron")); - cron.scheduleWithFixedDelay(new CleanClients(), 16, 11, TimeUnit.MINUTES); - this.metric = parentCtx.getMetric(); - this.localHostname = resolveLocalHostname(); - } - - /** - * Exposed for creating mocks. - */ - protected DocumentTypeManager createDocumentManager(DocumentmanagerConfig documentManagerConfig) { - return new DocumentTypeManager(documentManagerConfig); - } - - private class CleanClients implements Runnable { - - @Override - public void run() { - List<ClientState> clientsToShutdown = new ArrayList<>(); - long now = System.currentTimeMillis(); - - synchronized (clients) { - for (Iterator<Map.Entry<String, ClientState>> i = clients - .entrySet().iterator(); i.hasNext();) { - ClientState client = i.next().getValue(); - - if (now - client.creationTime > 10 * 60 * 1000) { - clientsToShutdown.add(client); - i.remove(); - } - } - } - for (ClientState client : clientsToShutdown) { - client.sourceSession.getReference().close(); - } - } } private Tuple2<HttpResponse, Integer> checkProtocolVersion(HttpRequest request) { @@ -135,8 +88,6 @@ public class FeedHandler extends LoggingRequestHandler { int version; if (washedClientVersions.contains("3")) { version = 3; - } else if (washedClientVersions.contains("2")) { // TODO: Vespa 7: Remove support for Version 2 - version = 2; } else { return new Tuple2<>(new ErrorHttpResponse( Headers.HTTP_NOT_ACCEPTABLE, @@ -175,43 +126,7 @@ public class FeedHandler extends LoggingRequestHandler { if (protocolVersion.first != null) { return protocolVersion.first; } - if (3 == protocolVersion.second) { - return feedHandlerV3.handle(request); - } - final BlockingQueue<OperationStatus> operations = new LinkedBlockingQueue<>(); - Tuple2<String, Boolean> clientId; - clientId = sessionId(request); - - if (clientId.second != null && clientId.second) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Received initial request from client with session ID " + - clientId.first + ", protocol version " + protocolVersion.second); - } - } - - Feeder feeder; - try { - feeder = createFeeder(request, request.getData(), operations, clientId.first, - clientId.second, protocolVersion.second); - // the synchronous FeedResponse blocks draining the InputStream, letting the Feeder read it - workers.submit(feeder); - } catch (UnknownClientException uce) { - String msg = Exceptions.toMessageString(uce); - log.log(LogLevel.WARNING, msg); - return new ErrorHttpResponse(Status.BAD_REQUEST, msg); - } catch (Exception e) { - String msg = "Could not initialize document parsing"; - log.log(LogLevel.WARNING, "Could not initialize document parsing", e); - return new ErrorHttpResponse(Status.INTERNAL_SERVER_ERROR, msg + ": " + Exceptions.toMessageString(e)); - } - - try { - feeder.waitForRequestReceived(); - } catch (InterruptedException e) { - return new ErrorHttpResponse(Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } - - return new FeedResponse(200, operations, protocolVersion.second, clientId.first); + return feedHandlerV3.handle(request); } // Protected for testing @@ -225,82 +140,6 @@ public class FeedHandler extends LoggingRequestHandler { } } - /** - * Exposed for creating mocks. - */ - protected Feeder createFeeder( - HttpRequest request, - InputStream requestInputStream, - BlockingQueue<OperationStatus> operations, - String clientId, - boolean sessionIdWasGeneratedJustNow, - int protocolVersion) throws Exception { - if (protocolVersion != 2) - throw new IllegalStateException("Protocol version " + protocolVersion + " not supported."); - - return new Feeder( - unzipStreamIfNeeded(requestInputStream, request), - new FeedReaderFactory(), - docTypeManager, - operations, - popClient(clientId), - new FeederSettings(request), - clientId, - sessionIdWasGeneratedJustNow, - sourceSessionParams(request), - sessionCache, - this, - metric, - feedReplyHandler, - localHostname); - } - - private Tuple2<String, Boolean> sessionId(HttpRequest request) { - boolean sessionIdWasGeneratedJustNow = false; - String sessionId = request.getHeader(Headers.SESSION_ID); - if (sessionId == null) { - sessionId = Long.toString(this.sessionId.incrementAndGet()) + "-" + - remoteHostAddressAndPort(request.getJDiscRequest()) + "#" + - localHostname; - sessionIdWasGeneratedJustNow = true; - } - return new Tuple2<>(sessionId, sessionIdWasGeneratedJustNow); - } - - private static String remoteHostAddressAndPort(com.yahoo.jdisc.http.HttpRequest httpRequest) { - SocketAddress remoteAddress = httpRequest.getRemoteAddress(); - if (remoteAddress instanceof InetSocketAddress) { - InetSocketAddress isa = (InetSocketAddress) remoteAddress; - return isa.getAddress().getHostAddress() + "-" + isa.getPort(); - } - return ""; - } - - private static String resolveLocalHostname() { - String hostname = HostName.getLocalhost(); - if (hostname.equals("localhost")) { - return ""; - } - return hostname; - } - - /** - * Exposed for use when creating mocks. - */ - protected SourceSessionParams sourceSessionParams(HttpRequest request) { - SourceSessionParams params = new SourceSessionParams(); - String timeout = request.getHeader(Headers.TIMEOUT); - - if (timeout != null) { - try { - params.setTimeout(Double.parseDouble(timeout)); - } catch (NumberFormatException e) { - // NOP - } - } - return params; - } - @Override protected void destroy() { feedHandlerV3.destroy(); @@ -316,33 +155,5 @@ public class FeedHandler extends LoggingRequestHandler { private void internalDestroy() { super.destroy(); - workers.shutdown(); - cron.shutdown(); - synchronized (clients) { - for (ClientState client : clients.values()) { - client.sourceSession.getReference().close(); - } - clients.clear(); - } - } - - void putClient(final String sessionId, final ClientState value) { - synchronized (clients) { - clients.put(sessionId, value); - } } - - ClientState popClient(String sessionId) { - synchronized (clients) { - return clients.remove(sessionId); - } - } - - /** - * Guess what, testing only. - */ - void forceRunCleanClients() { - new CleanClients().run(); - } - } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Feeder.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Feeder.java deleted file mode 100644 index f7890db3b35..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/Feeder.java +++ /dev/null @@ -1,537 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.server; - -import com.yahoo.collections.Tuple2; -import com.yahoo.container.jdisc.messagebus.SessionCache; -import com.yahoo.document.DocumentId; -import com.yahoo.document.DocumentUpdate; -import com.yahoo.document.DocumentRemove; -import com.yahoo.document.DocumentPut; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; -import com.yahoo.documentapi.metrics.DocumentOperationType; -import com.yahoo.jdisc.Metric; -import com.yahoo.jdisc.ReferencedResource; -import com.yahoo.log.LogLevel; -import com.yahoo.messagebus.Message; -import com.yahoo.messagebus.ReplyHandler; -import com.yahoo.messagebus.Result; -import com.yahoo.messagebus.SourceSessionParams; -import com.yahoo.messagebus.routing.ErrorDirective; -import com.yahoo.messagebus.routing.Hop; -import com.yahoo.messagebus.routing.Route; -import com.yahoo.messagebus.shared.SharedSourceSession; -import com.yahoo.yolean.Exceptions; -import com.yahoo.text.Utf8String; -import com.yahoo.vespa.http.client.core.Encoder; -import com.yahoo.vespa.http.client.core.ErrorCode; -import com.yahoo.vespa.http.client.core.OperationStatus; -import com.yahoo.vespa.http.server.util.ByteLimitedInputStream; -import com.yahoo.vespaxmlparser.FeedReader; -import com.yahoo.vespaxmlparser.VespaXMLFeedReader; -import com.yahoo.vespaxmlparser.VespaXMLFeedReader.Operation; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; - -import static com.yahoo.messagebus.ErrorCode.SEND_QUEUE_FULL; - -/** - * Read documents from client, and send them through message bus. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class Feeder implements Runnable { - - protected static final Logger log = Logger.getLogger(Feeder.class.getName()); - - final InputStream requestInputStream; - final DocumentTypeManager docTypeManager; - final BlockingQueue<OperationStatus> operations; - final BlockingQueue<OperationStatus> feedReplies; - int numPending; - final FeederSettings settings; - final String clientId; - final ReferencedResource<SharedSourceSession> sourceSession; - final FeedHandler handler; - final Metric metric; - final Metric.Context metricContext; - private long prevOpsPerSecTime; // previous measurement time of OPS - private double operationsForOpsPerSec; - private final ReplyHandler feedReplyHandler; - protected final static String EOF = "End of stream"; - protected final boolean sessionIdWasGeneratedJustNow; - private final CountDownLatch requestReceived = new CountDownLatch(1); - private final FeedReaderFactory feedReaderFactory; - - public Feeder(InputStream requestInputStream, - FeedReaderFactory feedReaderFactory, - DocumentTypeManager docTypeManager, - BlockingQueue<OperationStatus> operations, - ClientState storedState, - FeederSettings settings, - String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams, - SessionCache sessionCache, - FeedHandler handler, Metric metric, ReplyHandler feedReplyHandler, - String localHostname) throws Exception { - super(); - this.feedReaderFactory = feedReaderFactory; - if (storedState == null) { - if (!sessionIdWasGeneratedJustNow) { - // We do not have a stored state, BUT the session ID came in with the request. - // Possible session timeout, server restart, server reconfig, or VIP usage. Examine. - examineClientId(clientId, localHostname); - } - numPending = 0; - feedReplies = new LinkedBlockingQueue<>(); - sourceSession = retainSession(sessionParams, sessionCache); - metricContext = createClientMetricContext(metric, clientId); - prevOpsPerSecTime = System.currentTimeMillis(); - operationsForOpsPerSec = 0.0; - } else { - //we have a stored state, and the session ID was obviously not generated now. All OK. - numPending = storedState.pending; - feedReplies = storedState.feedReplies; - sourceSession = storedState.sourceSession; - metricContext = storedState.metricContext; - prevOpsPerSecTime = storedState.prevOpsPerSecTime; - operationsForOpsPerSec = storedState.operationsForOpsPerSec; - } - this.clientId = clientId; - this.sessionIdWasGeneratedJustNow = sessionIdWasGeneratedJustNow; - this.requestInputStream = requestInputStream; - this.docTypeManager = docTypeManager; - this.operations = operations; - this.settings = settings; - this.handler = handler; - this.metric = metric; - this.feedReplyHandler = feedReplyHandler; - } - protected void examineClientId(String clientId, String localHostname) { - if (!clientId.contains("#")) { - throw new UnknownClientException("Got request from client with id '" + clientId + - "', but found no session for this client. " + - "This is expected during upgrades of gateways and infrastructure nodes."); - } - int hashPos = clientId.indexOf("#"); - String supposedHostname = clientId.substring(hashPos + 1, clientId.length()); - if (supposedHostname.isEmpty()) { - throw new UnknownClientException("Got request from client with id '" + clientId + - "', but found no session for this client. Possible session " + - "timeout due to inactivity, server restart or reconfig, " + - "or bad VIP usage. " + - "This is expected during upgrades of gateways and infrastructure nodes."); - } - - if (!supposedHostname.equals(localHostname)) { - throw new UnknownClientException("Got request from client with id '" + clientId + - "', but found no session for this client. " + - "Session was originally established towards host " + - supposedHostname + ", but our hostname is " + - localHostname + ". " + - "If using VIP rotation, this could be due to a session was rotated from one server to another. " + - "Configure VIP with persistence=enabled. " + - "This is expected during upgrades of gateways and infrastructure nodes."); - } - log.log(LogLevel.DEBUG, "Client '" + clientId + "' reconnected after session inactivity, or server restart " + - "or reconfig. Re-establishing session."); - } - - - - private static Metric.Context createClientMetricContext(Metric metric, String clientId) { - // No real value in separate metric dimensions per client. - return null; - } - - /** - * Exposed for creating mocks. - */ - protected ReferencedResource<SharedSourceSession> retainSession( - SourceSessionParams sessionParams, SessionCache sessionCache) { - return sessionCache.retainSource(sessionParams); - } - - @Override - public void run() { - try { - if (handshake()) { - return; //will putClient in finally block below - } - flushResponseQueue(); - feed(); - requestReceived.countDown(); - drain(); - } catch (InterruptedException e) { - // NOP, just terminate - } catch (Exception e) { - log.log(LogLevel.WARNING, "Unhandled exception while feeding: " - + Exceptions.toMessageString(e), e); - } catch (Throwable e) { - log.log(LogLevel.WARNING, "Unhandled error while feeding: " - + Exceptions.toMessageString(e), e); - throw e; - } finally { - requestReceived.countDown(); - putClient(); - try { - enqueue("-", "-", ErrorCode.END_OF_FEED, false, null); - } catch (InterruptedException e) { - // NOP, we are already exiting the thread - } - } - } - - protected boolean handshake() throws IOException { - if (sessionIdWasGeneratedJustNow) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Handshake completed for client " + clientId + "."); - } - requestInputStream.close(); - return true; - } - return false; - } - - void feed() throws InterruptedException { - while (true) { - Result result; - String operationId; - try { - operationId = getNextOperationId(); - } catch (IOException ioe) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, Exceptions.toMessageString(ioe), ioe); - } - break; - } - - //noinspection StringEquality - if (operationId == EOF) { - break; - } - - Tuple2<String, Message> msg; - try { - msg = getNextMessage(operationId); - setRoute(msg); - } catch (Exception e) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, Exceptions.toMessageString(e), e); - } - //noinspection StringEquality - if (operationId != null) { //v1 always returns null, all others return something useful, or throw an exception above - msg = newErrorMessage(operationId, e); - } else { - break; - } - } - - if (msg == null) { - break; - } - - setMessageParameters(msg); - - while (true) { - try { - msg.second.pushHandler(feedReplyHandler); - if (settings.denyIfBusy) { - result = sourceSession.getResource().sendMessage(msg.second); - } else { - result = sourceSession.getResource().sendMessageBlocking(msg.second); - } - } catch (RuntimeException e) { - enqueue(msg.first, Exceptions.toMessageString(e), - ErrorCode.ERROR, false, msg.second); - return; - } - if (result.isAccepted() || result.getError().getCode() != SEND_QUEUE_FULL) { - break; - } - if (settings.denyIfBusy) { - break; - } else { - //This will never happen - Thread.sleep(100); - } - } - - if (result.isAccepted()) { - ++numPending; - updateMetrics(msg.second); - updateOpsPerSec(); - log(LogLevel.DEBUG, "Sent message successfully, document id: ", msg.first); - } else if (!result.getError().isFatal()) { - enqueue(msg.first, result.getError().getMessage(), ErrorCode.TRANSIENT_ERROR, false, msg.second); - break; - } else { - // should probably not happen, but everybody knows stuff that - // shouldn't happen, happens all the time - boolean isConditionNotMet = result.getError().getCode() == DocumentProtocol.ERROR_TEST_AND_SET_CONDITION_FAILED; - enqueue(msg.first, result.getError().getMessage(), ErrorCode.ERROR, isConditionNotMet, msg.second); - break; - } - } - } - - private Tuple2<String, Message> newErrorMessage(String operationId, Exception e) { - Message m = new FeedErrorMessage(operationId); - Tuple2<String, Message> msg = new Tuple2<>(operationId, m); - Hop hop = new Hop(); - hop.addDirective(new ErrorDirective(Exceptions.toMessageString(e))); - Route route = new Route(); - route.addHop(hop); - m.setRoute(route); - return msg; - } - - private void updateMetrics(Message m) { - metric.set( - MetricNames.PENDING, - Double.valueOf(sourceSession.getResource().session().getPendingCount()), - null); - - metric.add(MetricNames.NUM_OPERATIONS, 1, metricContext); - - if (m instanceof PutDocumentMessage) { - metric.add(MetricNames.NUM_PUTS, 1, metricContext); - } else if (m instanceof RemoveDocumentMessage) { - metric.add(MetricNames.NUM_REMOVES, 1, metricContext); - } else if (m instanceof UpdateDocumentMessage) { - metric.add(MetricNames.NUM_UPDATES, 1, metricContext); - } - } - - private void updateOpsPerSec() { - long now = System.currentTimeMillis(); - if ((now - prevOpsPerSecTime) >= 1000) { //every second - double ms = (double) (now - prevOpsPerSecTime); - final double opsPerSec = operationsForOpsPerSec / (ms / 1000); - metric.set(MetricNames.OPERATIONS_PER_SEC, opsPerSec, metricContext); - operationsForOpsPerSec = 1.0d; - prevOpsPerSecTime = now; - } else { - operationsForOpsPerSec += 1.0d; - } - } - - private Tuple2<String, Message> getNextMessage(String operationId) throws Exception { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - Tuple2<String, Message> msg; - - getNextOperation(op); - - switch (op.getType()) { - case DOCUMENT: - msg = newPutMessage(op, operationId); - break; - case REMOVE: - msg = newRemoveMessage(op, operationId); - break; - case UPDATE: - msg = newUpdateMessage(op, operationId); - break; - default: - // typical end of feed - return null; - } - log(LogLevel.DEBUG, "Successfully deserialized document id: ", msg.first); - return msg; - } - - private void setMessageParameters(Tuple2<String, Message> msg) { - msg.second.setContext(new ReplyContext(msg.first, feedReplies, DocumentOperationType.fromMessage(msg.second))); - if (settings.traceLevel != null) { - msg.second.getTrace().setLevel(settings.traceLevel); - } - if (settings.priority != null) { - try { - DocumentProtocol.Priority priority = DocumentProtocol.Priority.valueOf(settings.priority); - if (msg.second instanceof DocumentMessage) { - ((DocumentMessage) msg.second).setPriority(priority); - } - } - catch (IllegalArgumentException i) { - log.severe(i.getMessage()); - } - } - } - - private void setRoute(Tuple2<String, Message> msg) { - if (settings.route != null) { - msg.second.setRoute(settings.route); - } - } - - protected void getNextOperation(VespaXMLFeedReader.Operation op) throws Exception { - int length = readByteLength(); - - try (InputStream limitedInputStream = new ByteLimitedInputStream(requestInputStream, length)){ - FeedReader reader = feedReaderFactory.createReader(limitedInputStream, docTypeManager, settings.dataFormat); - reader.read(op); - } - } - - protected String getNextOperationId() throws IOException { - return readOperationId(); - } - - private String readOperationId() throws IOException { - StringBuilder idBuf = new StringBuilder(100); - int c; - while ((c = requestInputStream.read()) != -1) { - if (c == 32) { - break; - } - idBuf.append((char) c); //it's ASCII - } - if (idBuf.length() == 0) { - return EOF; - } - return Encoder.decode(idBuf.toString(), new StringBuilder(idBuf.length())).toString(); - } - - private int readByteLength() throws IOException { - StringBuilder lenBuf = new StringBuilder(8); - int c; - while ((c = requestInputStream.read()) != -1) { - if (c == 10) { - break; - } - lenBuf.append((char) c); //it's ASCII - } - if (lenBuf.length() == 0) { - throw new IllegalStateException("Operation length missing."); - } - return Integer.valueOf(lenBuf.toString(), 16); - } - - protected final void log(LogLevel level, Object... msgParts) { - StringBuilder s; - - if (!log.isLoggable(level)) { - return; - } - - s = new StringBuilder(); - for (Object part : msgParts) { - s.append(part.toString()); - } - - log.log(level, s.toString()); - } - - private Tuple2<String, Message> newUpdateMessage(Operation op, String operationId) { - DocumentUpdate update = op.getDocumentUpdate(); - update.setCondition(op.getCondition()); - Message msg = new UpdateDocumentMessage(update); - - String id = (operationId == null) ? update.getId().toString() : operationId; - return new Tuple2<>(id, msg); - } - - private Tuple2<String, Message> newRemoveMessage(Operation op, String operationId) { - DocumentRemove remove = new DocumentRemove(op.getRemove()); - remove.setCondition(op.getCondition()); - Message msg = new RemoveDocumentMessage(remove); - - String id = (operationId == null) ? remove.getId().toString() : operationId; - return new Tuple2<>(id, msg); - } - - private Tuple2<String, Message> newPutMessage(Operation op, String operationId) { - DocumentPut put = new DocumentPut(op.getDocument()); - put.setCondition(op.getCondition()); - Message msg = new PutDocumentMessage(put); - - String id = (operationId == null) ? put.getId().toString() : operationId; - return new Tuple2<>(id, msg); - } - - - void flushResponseQueue() throws InterruptedException { - OperationStatus status = feedReplies.poll(); - while (status != null) { - decreasePending(status); - status = feedReplies.poll(); - } - } - - void putClient() { - handler.putClient(clientId, - new ClientState(numPending, - feedReplies, sourceSession, metricContext, - prevOpsPerSecTime, operationsForOpsPerSec)); - } - - void drain() throws InterruptedException { - if (settings.drain) { - while (numPending > 0) { - OperationStatus o = feedReplies.take(); - decreasePending(o); - } - } - } - - private void decreasePending(OperationStatus o) throws InterruptedException { - operations.put(o); - --numPending; - } - - private void enqueue(String id, String message, ErrorCode code, boolean isConditionalNotMet, Message msg) - throws InterruptedException { - String traceMessage = msg != null && msg.getTrace() != null && msg.getTrace().getLevel() > 0 - ? msg.getTrace().toString() - : ""; - operations.put(new OperationStatus(message, id, code, isConditionalNotMet, traceMessage)); - } - - public void waitForRequestReceived() throws InterruptedException { - requestReceived.await(1, TimeUnit.HOURS); - } - - public class FeedErrorMessage extends Message { - private long sequenceId; - - private FeedErrorMessage(String operationId) { - try { - DocumentId id = new DocumentId(operationId); - sequenceId = Arrays.hashCode(id.getGlobalId()); - } catch (Exception e) { - sequenceId = 0; - } - } - - @Override - public Utf8String getProtocol() { - return new Utf8String("vespa-feed-handler-internal-bogus-protocol"); - } - - @Override - public int getType() { - return 1234; - } - - @Override - public boolean hasSequenceId() { - return true; - } - - @Override - public long getSequenceId() { - return sequenceId; - } - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationTest.java index b4067c3fe78..6bca3637297 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationTest.java @@ -32,4 +32,5 @@ public class DocumentApiApplicationTest { socket.close(); return port; } + } diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore b/vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore +++ /dev/null diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java deleted file mode 100644 index c0cc907c671..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedhandler; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.jdisc.HeaderFields; -import com.yahoo.jdisc.Metric; -import com.yahoo.container.logging.AccessLog; -import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.OperationStatus; -import com.yahoo.vespa.http.server.FeedHandler; -import com.yahoo.vespa.http.server.Feeder; -import org.junit.Test; - -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executors; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * Unit test for FeedHandler class. - * - * @author dybis - */ -public class FeedHandlerTest { - - /** - * This class extends FeedHandler and allows to create a custom Feeder. - */ - static class TestFeedHandler extends FeedHandler { - private final CountDownLatch countDownLatch = new CountDownLatch(1); - - public TestFeedHandler() throws Exception { - super(new FeedHandler.Context(Executors.newCachedThreadPool(), - mock(AccessLog.class), - mock(Metric.class)), - null, null, null, MetricReceiver.nullImplementation); - } - - /** - * Builds a feeder that blocks until countDownLatch is stepped down. - */ - @Override - protected Feeder createFeeder( - com.yahoo.container.jdisc.HttpRequest request, - InputStream requestInputStream, - final BlockingQueue<OperationStatus> operations, - String clientId, - boolean sessionIdWasGeneratedJustNow, - int protocolVersion) throws Exception { - Feeder feeder = mock(Feeder.class); - doAnswer(invocation -> { - try { - countDownLatch.await(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - return null; - }).when(feeder).waitForRequestReceived(); - return feeder; - } - } - - /** - * nginx require that a post is finished before the server ack with a response. This behaviour is verified - * in this test - */ - @Test - public void testResponseIsSentAfterWaitForRequestReceivedReturns() throws Exception { - HttpRequest request = mock(HttpRequest.class); - - // Create a request with valid version. - com.yahoo.jdisc.http.HttpRequest jdiscRequest = mock(com.yahoo.jdisc.http.HttpRequest.class); - HeaderFields headerFields = mock(HeaderFields.class); - List<String> version = new ArrayList<>(); - version.add("2"); - when(headerFields.get(Headers.VERSION)).thenReturn(version); - when(jdiscRequest.headers()).thenReturn(headerFields); - when(request.getJDiscRequest()).thenReturn(jdiscRequest); - - TestFeedHandler feedHandler = new TestFeedHandler(); - // After a short period, make the feed finish. - new Thread(() -> { - try { - Thread.sleep(50); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - feedHandler.countDownLatch.countDown(); - }).start(); - // This should not return before countdown latch is stepped down. - feedHandler.handle(request); - // This should always returns after the countDownLatch has become zero. This might cause false positive, - // but not false negatives. This is fine. - assertThat(feedHandler.countDownLatch.getCount(), is(0L)); - - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java deleted file mode 100755 index 2c99ec194c3..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java +++ /dev/null @@ -1,1064 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedhandler; - -import com.yahoo.component.ComponentId; -import com.yahoo.component.provider.ComponentRegistry; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.docproc.CallStack; -import com.yahoo.jdisc.HeaderFields; -import com.yahoo.messagebus.*; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.container.Container; -import com.yahoo.docproc.*; -import com.yahoo.docproc.jdisc.DocumentProcessingHandler; -import com.yahoo.docproc.jdisc.DocumentProcessingHandlerParameters; -import com.yahoo.document.*; -import com.yahoo.document.datatypes.IntegerFieldValue; -import com.yahoo.documentapi.messagebus.loadtypes.LoadType; -import com.yahoo.documentapi.messagebus.protocol.*; -import com.yahoo.feedapi.DummySessionFactory; -import com.yahoo.feedapi.FeedContext; -import com.yahoo.feedapi.MessagePropertyProcessor; -import com.yahoo.jdisc.handler.RequestHandler; -import com.yahoo.messagebus.routing.Route; -import com.yahoo.vespaclient.ClusterDef; -import com.yahoo.vespaclient.ClusterList; -import com.yahoo.vespaclient.config.FeederConfig; -import org.junit.After; -import org.junit.Test; - -import java.io.*; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Executor; -import java.util.concurrent.Executors; -import java.util.logging.Logger; -import java.util.zip.GZIPOutputStream; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -@SuppressWarnings("deprecation") // VespaFeedHandler classes are going away on Vespa 7 -public class VespaFeedHandlerTestCase { - - private VespaFeedHandler feedHandler; - private VespaFeedHandlerRemove removeHandler; - private VespaFeedHandlerStatus statusHandler; - private VespaFeedHandlerRemoveLocation removeLocationHandler; - private FeedContext context; - - private DummySessionFactory factory; - private final String xmlFilesPath = "src/test/files/feedhandler/"; - - public void setup(com.yahoo.messagebus.Error e, LoadTypeConfig loadTypeCfg, - boolean autoReply, - DummySessionFactory.ReplyFactory autoReplyFactory) throws Exception { - DocumentTypeManager docMan = new DocumentTypeManager(); - DocumentTypeManagerConfigurer.configure(docMan, "file:" + xmlFilesPath + "documentmanager.cfg"); - - if (autoReply) { - if (autoReplyFactory != null) { - factory = DummySessionFactory.createWithAutoReplyFactory(autoReplyFactory); - } else { - factory = DummySessionFactory.createWithErrorAutoReply(e); - } - } else { - factory = DummySessionFactory.createDefault(); - } - - context = new FeedContext(new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder()), loadTypeCfg), factory, docMan, new ClusterList(), new NullFeedMetric(true)); - - Executor threadPool = Executors.newCachedThreadPool(); - feedHandler = new VespaFeedHandler(context, threadPool); - removeHandler = new VespaFeedHandlerRemove(context, threadPool); - statusHandler = new VespaFeedHandlerStatus(context, false, false, threadPool); - removeLocationHandler = new VespaFeedHandlerRemoveLocation(context, threadPool); - - CallStack dpCallstack = new CallStack("bar"); - dpCallstack.addLast(new TestDocProc()); - dpCallstack.addLast(new TestLaterDocProc()); - - DocprocService myservice = new DocprocService("bar"); - myservice.setCallStack(dpCallstack); - myservice.setInService(true); - - ComponentRegistry<DocprocService> registry = new ComponentRegistry<DocprocService>(); - registry.register(new ComponentId(myservice.getName()), myservice); - - DocumentProcessingHandler handler = new DocumentProcessingHandler(registry, - new ComponentRegistry<>(), new ComponentRegistry<>(), - new DocumentProcessingHandlerParameters()); - - Container container = Container.get(); - ComponentRegistry<RequestHandler> requestHandlerComponentRegistry = new ComponentRegistry<>(); - requestHandlerComponentRegistry.register(new ComponentId(DocumentProcessingHandler.class.getName()), handler); - container.setRequestHandlerRegistry(requestHandlerComponentRegistry); - } - - public void setup(com.yahoo.messagebus.Error e) throws Exception { - setup(e, new LoadTypeConfig(new LoadTypeConfig.Builder()), true, null); - } - - public void setupWithReplyFactory(DummySessionFactory.ReplyFactory autoReplyFactory) throws Exception { - setup(null, new LoadTypeConfig(new LoadTypeConfig.Builder()), true, autoReplyFactory); - } - - public void setup() throws Exception { - setup(null, new LoadTypeConfig(new LoadTypeConfig.Builder()), false, null); - } - - @After - public void resetContainer() { - Container.resetInstance(); - } - - - @Test - public void testLoadTypes() throws Exception { - List<LoadTypeConfig.Type.Builder> typeBuilder = new ArrayList<>(); - typeBuilder.add(new LoadTypeConfig.Type.Builder().id(1234).name("foo").priority("VERY_LOW")); - typeBuilder.add(new LoadTypeConfig.Type.Builder().id(4567).name("bar").priority("NORMAL_3")); - - setup(null, new LoadTypeConfig(new LoadTypeConfig.Builder().type(typeBuilder)), true, null); - - { - Result res = testRequest(HttpRequest.createTestRequest("remove?id=doc:test:removeme&loadtype=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(1, res.messages.size()); - - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:removeme", d.toString()); - assertEquals(new LoadType(1234, "foo", DocumentProtocol.Priority.VERY_LOW), ((DocumentMessage)m).getLoadType()); - assertEquals(DocumentProtocol.Priority.VERY_LOW, ((DocumentMessage)m).getPriority()); - } - } - - @Test - public void testPostXML() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?"); - - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10b", d.toString()); - } - - assertTrue(res.output.contains("count=\"2\"")); - assertTrue(res.error == null); - } - - @Test - public void testPostXMLAsync() throws Exception { - setup(); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?asynchronous=true"); - - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10b", d.toString()); - } - - // Should not have metrics at this point. - assertTrue(!res.output.contains("count=\"2\"")); - assertTrue(res.error == null); - } - - - @Test - public void testPostGZIPedXML() throws Exception { - setup(null); - Result res = testFeedGZIP(xmlFilesPath + "test10b.xml", "feed?"); - - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10b", d.toString()); - } - - assertTrue(res.error == null); - } - - @Test - public void testDocProc() throws Exception { - setup(null); - - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?docprocchain=bar"); - - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - Document d = ((PutDocumentMessage)m).getDocumentPut().getDocument(); - - assertEquals("doc:news:http://news10a", d.getId().toString()); - assertEquals(new IntegerFieldValue(1234), d.getFieldValue("last_downloaded")); - } - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - Document d = ((PutDocumentMessage)m).getDocumentPut().getDocument(); - - assertEquals("doc:news:http://news10b", d.getId().toString()); - assertEquals(new IntegerFieldValue(1234), d.getFieldValue("last_downloaded")); - } - } - - @Test - public void testPostXMLVariousTypes() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test10.xml", "feed?"); - - assertEquals(5, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10b", d.toString()); - } - - { - Message m = res.messages.get(2); - assertEquals(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, m.getType()); - DocumentId d = ((UpdateDocumentMessage)m).getDocumentUpdate().getId(); - assertEquals("doc:news:http://news10c", d.toString()); - } - { - Message m = res.messages.get(3); - assertEquals(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, m.getType()); - DocumentId d = ((UpdateDocumentMessage)m).getDocumentUpdate().getId(); - assertEquals("doc:news:http://news10d", d.toString()); - } - { - Message m = res.messages.get(4); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:news:http://news10e", d.toString()); - } - - String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/"); - - assertEquals("<result>\n" + - "\n" + - " <route name=\"default\">\n" + - " <total>\n" + - " <latency count=\"5\"/>\n" + - " <count count=\"5\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " </putdocument>\n" + - " <updatedocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " </updatedocument>\n" + - " <removedocument>\n" + - " <latency count=\"1\"/>\n" + - " <count count=\"1\"/>\n" + - " </removedocument>\n" + - " </route>\n" + - "\n" + - "</result>\n", val); - } - - @Test - public void testStatusPage() throws Exception { - setup(null); - - testFeed(xmlFilesPath + "test10b.xml", "feed?docprocchain=bar"); - testFeed(xmlFilesPath + "test10.xml", "feed?"); - testFeed(xmlFilesPath + "test10.xml", "feed?route=storage"); - testFeed(xmlFilesPath + "test_removes", "remove?"); - - assertEquals(2, factory.sessionsCreated()); - Result res = testRequest(HttpRequest.createTestRequest("feedstatus?", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - - String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/"); - val = val.replaceAll("to=\"[0-9]*\"", "to=\"0\""); - - assertEquals("<status>\n" + - "\n" + - " <snapshot name=\"Total metrics from start until current time\" from=\"0\" to=\"0\" period=\"0\">\n" + - " <routes>\n" + - " <route name=\"total\">\n" + - " <total>\n" + - " <latency count=\"14\"/>\n" + - " <count count=\"14\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <latency count=\"6\"/>\n" + - " <count count=\"6\"/>\n" + - " </putdocument>\n" + - " <updatedocument>\n" + - " <latency count=\"4\"/>\n" + - " <count count=\"4\"/>\n" + - " </updatedocument>\n" + - " <removedocument>\n" + - " <latency count=\"4\"/>\n" + - " <count count=\"4\"/>\n" + - " </removedocument>\n" + - " </route>\n" + - " <route name=\"default\">\n" + - " <total>\n" + - " <latency count=\"9\"/>\n" + - " <count count=\"9\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <latency count=\"4\"/>\n" + - " <count count=\"4\"/>\n" + - " </putdocument>\n" + - " <updatedocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " </updatedocument>\n" + - " <removedocument>\n" + - " <latency count=\"3\"/>\n" + - " <count count=\"3\"/>\n" + - " </removedocument>\n" + - " </route>\n" + - " <route name=\"storage\">\n" + - " <total>\n" + - " <latency count=\"5\"/>\n" + - " <count count=\"5\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " </putdocument>\n" + - " <updatedocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " </updatedocument>\n" + - " <removedocument>\n" + - " <latency count=\"1\"/>\n" + - " <count count=\"1\"/>\n" + - " </removedocument>\n" + - " </route>\n" + - " </routes>\n" + - " </snapshot>\n" + - "\n" + - "</status>\n", val); - } - - @Test - public void testStatusPage2() throws Exception { - setup(null); - - testFeed(xmlFilesPath + "test10b.xml", "feed?docprocchain=bar"); - testFeed(xmlFilesPath + "test10.xml", "feed?"); - testFeed(xmlFilesPath + "test10.xml", "feed?route=storage"); - testFeed(xmlFilesPath + "test_removes", "remove?"); - - assertEquals(2, factory.sessionsCreated()); - Result res = testRequest(HttpRequest.createTestRequest("feed?status", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - - String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/"); - val = val.replaceAll("to=\"[0-9]*\"", "to=\"0\""); - - assertEquals("<status>\n" + - "\n" + - " <routes>\n" + - " <route name=\"total\" description=\"Messages sent to all routes\">\n" + - " <total description=\"All kinds of messages sent to the given route\">\n" + - " <latency count=\"14\"/>\n" + - " <count count=\"14\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <latency count=\"6\"/>\n" + - " <count count=\"6\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </putdocument>\n" + - " <updatedocument>\n" + - " <latency count=\"4\"/>\n" + - " <count count=\"4\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </updatedocument>\n" + - " <removedocument>\n" + - " <latency count=\"4\"/>\n" + - " <count count=\"4\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </removedocument>\n" + - " </route>\n" + - " <route name=\"default\" description=\"Messages sent to the named route\">\n" + - " <total description=\"All kinds of messages sent to the given route\">\n" + - " <latency count=\"9\"/>\n" + - " <count count=\"9\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <latency count=\"4\"/>\n" + - " <count count=\"4\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </putdocument>\n" + - " <updatedocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </updatedocument>\n" + - " <removedocument>\n" + - " <latency count=\"3\"/>\n" + - " <count count=\"3\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </removedocument>\n" + - " </route>\n" + - " <route name=\"storage\" description=\"Messages sent to the named route\">\n" + - " <total description=\"All kinds of messages sent to the given route\">\n" + - " <latency count=\"5\"/>\n" + - " <count count=\"5\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </putdocument>\n" + - " <updatedocument>\n" + - " <latency count=\"2\"/>\n" + - " <count count=\"2\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </updatedocument>\n" + - " <removedocument>\n" + - " <latency count=\"1\"/>\n" + - " <count count=\"1\"/>\n" + - " <ignored count=\"0\"/>\n" + - " </removedocument>\n" + - " </route>\n" + - " </routes>\n" + - "\n" + - "</status>\n", val); - } - - @Test - public void testMetricForIgnoredDocumentsIsIncreased() throws Exception { - DummySessionFactory.ReplyFactory replyFactory = new DummySessionFactory.ReplyFactory() { - @Override - public Reply createReply(Message m) { - return new DocumentIgnoredReply(); - } - }; - setupWithReplyFactory(replyFactory); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?"); - assertEquals(2, res.messages.size()); - - String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/"); - - assertEquals("<result>\n" + - "\n" + - " <route name=\"default\">\n" + - " <total>\n" + - " <ignored count=\"2\"/>\n" + - " </total>\n" + - " <putdocument>\n" + - " <ignored count=\"2\"/>\n" + - " </putdocument>\n" + - " </route>\n" + - "\n" + - "</result>\n", val); - } - - @Test - public void testPostXMLWithMBusFailureAllowed() throws Exception { - setup(new com.yahoo.messagebus.Error(DocumentProtocol.ERROR_BUCKET_DELETED, "Hello world in <document>")); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?abortonfeederror=false"); - - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10b", d.toString()); - } - - String val = res.output.replaceAll("average=\"[0-9]*\" last=\"[0-9]*\" min=\"[0-9]*\" max=\"[0-9]*\" ", ""); - System.out.println(val); - - assertEquals("<result>\n" + - "\n" + - " <route name=\"default\">\n" + - " <total>\n" + - " <errors>\n" + - " <error name=\"total\" count=\"2\"/>\n" + - " <error name=\"BUCKET_DELETED\" count=\"2\"/>\n" + - " </errors>\n" + - " </total>\n" + - " <putdocument>\n" + - " <errors>\n" + - " <error name=\"total\" count=\"2\"/>\n" + - " <error name=\"BUCKET_DELETED\" count=\"2\"/>\n" + - " </errors>\n" + - " </putdocument>\n" + - " </route>\n\n" + - " <errors count=\"2\">\n" + - " <error message=\"PUT[doc:news:http://news10a] [BUCKET_DELETED] Hello world in <document>\"/>\n" + - " <error message=\"PUT[doc:news:http://news10b] [BUCKET_DELETED] Hello world in <document>\"/>\n" + - " </errors>\n" + - "\n" + - "</result>\n", val); - - assertTrue(res.error != null); - assertTrue(res.errorCount > 0); - } - - @Test - public void testPostXMLWithMBusFailure() throws Exception { - setup(new com.yahoo.messagebus.Error(32, "Hello world")); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?"); - - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - - String val = res.output.replaceAll("average=\"[0-9]*\" last=\"[0-9]*\" min=\"[0-9]*\" max=\"[0-9]*\" ", ""); - assertEquals("<result>\n" + - "\n" + - " <route name=\"default\">\n" + - " <total>\n" + - " <errors>\n" + - " <error name=\"total\" count=\"1\"/>\n" + - " <error name=\"UNKNOWN(32)\" count=\"1\"/>\n" + - " </errors>\n" + - " </total>\n" + - " <putdocument>\n" + - " <errors>\n" + - " <error name=\"total\" count=\"1\"/>\n" + - " <error name=\"UNKNOWN(32)\" count=\"1\"/>\n" + - " </errors>\n" + - " </putdocument>\n" + - " </route>\n\n" + - " <errors count=\"1\">\n" + - " <error message=\"PUT[doc:news:http://news10a] [UNKNOWN(32)] Hello world\"/>\n" + - " </errors>\n" + - "\n" + - "</result>\n", val); - - assertTrue(res.error != null); - assertTrue(res.errorCount > 0); - } - - @Test - public void testPostXMLWithIllegalDocId() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test_bogus_docid.xml", "feed?"); - - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - } - - @Test - public void testPostXMLWithIllegalDocIdAllowFailure() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test_bogus_docid.xml", "feed?abortondocumenterror=false"); - - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:ok", d.toString()); - } - } - - @Test - public void testPostUnparseableXML() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test_bogus_xml.xml", "feed?"); - - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - } - - @Test - public void testOverrides() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?timeout=2.222&route=storage&priority=HIGH_2&totaltimeout=-1"); - - assertEquals(2, res.messages.size()); - - for (Message m : res.messages) { - assertEquals(2222, m.getTimeRemaining()); - assertEquals(Route.parse("storage"), m.getRoute()); - assertEquals(DocumentProtocol.Priority.HIGH_2, ((DocumentMessage)m).getPriority()); - } - } - - @Test - public void testTimeoutWithNoUpperBound() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?timeout=2.222&totaltimeout=-1"); - - assertEquals(2, res.messages.size()); - - for (Message m : res.messages) { - assertEquals(2222, m.getTimeRemaining()); - } - } - - @Test - public void testTimeout() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?timeout=2.222"); - - assertEquals(2, res.messages.size()); - - for (Message m : res.messages) { - assertTrue(2222 >= m.getTimeRemaining()); - } - } - - @Test - public void testTotalTimeout() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?totaltimeout=2.222"); - - assertEquals(2, res.messages.size()); - - for (Message m : res.messages) { - assertTrue(2222 >= m.getTimeRemaining()); - } - } - - @Test - public void testTotalTimeoutAndNormalTimeout() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?totaltimeout=1000&timeout=2.222"); - - assertEquals(2, res.messages.size()); - - for (Message m : res.messages) { - assertEquals(2222, m.getTimeRemaining()); - } - } - - @Test - public void testBogusPriority() throws Exception { - try { - setup(null); - Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?timeout=2222&route=storage&priority=HIPSTER_DOOFUS"); - assertTrue(false); - } catch (IllegalArgumentException e) { - } - } - - @Test - public void testPostXMLWithIllegalDocIdFirst() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test_bogus_docid_first.xml", "feed?"); - - assertEquals(0, res.messages.size()); - } - - @Test - public void testPostXMLWithIllegalDocIdFirstNoAbort() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test_bogus_docid_first.xml", "feed?abortondocumenterror=false"); - - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); - DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); - assertEquals("doc:news:http://news10a", d.toString()); - } - } - - @Test - public void testSimpleRemove() throws Exception { - setup(null); - Result res = testRequest(HttpRequest.createTestRequest("remove?id=doc:test:removeme", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:removeme", d.toString()); - } - } - - @Test - public void testRemoveUser() throws Exception { - setup(null); - - context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage")); - Result res = testRequest(HttpRequest.createTestRequest("removelocation?user=1234", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVELOCATION, m.getType()); - String selection = ((RemoveLocationMessage)m).getDocumentSelection(); - assertEquals("storage", m.getRoute().toString()); - assertEquals("id.user=1234", selection); - } - } - - @Test - public void testRemoveGroup() throws Exception { - setup(null); - context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage")); - Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVELOCATION, m.getType()); - String selection = ((RemoveLocationMessage)m).getDocumentSelection(); - assertEquals("storage", m.getRoute().toString()); - assertEquals("id.group=\"foo\"", selection); - } - } - - @Test - public void testRemoveBadSyntax() throws Exception { - setup(null); - context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage")); - Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo&user=12345", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(0, res.messages.size()); - assertTrue(res.error.toString().contains("Exactly one of")); - } - - @Test - public void testRemoveGroupMultipleClusters() throws Exception { - setup(null); - context.getClusterList().getStorageClusters().add(new ClusterDef("storage1", "storage/cluster.storage1")); - context.getClusterList().getStorageClusters().add(new ClusterDef("storage2", "storage/cluster.storage2")); - Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(0, res.messages.size()); - assertTrue(res.error.toString().contains("More than one")); - } - - @Test - public void testRemoveGroupNoClusters() throws Exception { - setup(null); - Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(0, res.messages.size()); - assertTrue(res.error.toString().contains("No storage clusters")); - } - - @Test - public void testRemoveSelection() throws Exception { - setup(null); - context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage")); - Result res = testRequest(HttpRequest.createTestRequest("removelocation?selection=id.user=1234", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVELOCATION, m.getType()); - String selection = ((RemoveLocationMessage)m).getDocumentSelection(); - assertEquals("id.user=1234", selection); - } - } - - @Test - public void testSimpleRemoveIndex() throws Exception { - setup(null); - Result res = testRequest(HttpRequest.createTestRequest("remove?id[0]=doc:test:removeme", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(1, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:removeme", d.toString()); - } - } - - @Test - public void testPostRemove() throws Exception { - setup(null); - Result res = testFeed(xmlFilesPath + "test_removes", "remove?"); - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:remove1", d.toString()); - } - - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:remove2", d.toString()); - } - } - - @Test - public void testRemoveBogusId() throws Exception { - try { - setup(null); - Result res = testRequest(HttpRequest.createTestRequest("remove?id=unknowndoc:test:removeme", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertTrue(false); - } catch (Exception e) { - } - } - - @Test - public void testMultiRemove() throws Exception { - setup(null); - Result res = testRequest(HttpRequest.createTestRequest("remove?id[0]=doc:test:removeme&id[1]=doc:test:remove2&id[2]=doc:test:remove3", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(3, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:removeme", d.toString()); - } - - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:remove2", d.toString()); - } - - { - Message m = res.messages.get(2); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); - assertEquals("doc:test:remove3", d.toString()); - } - } - - @Test - public void testMultiRemoveSameDoc() throws Exception { - setup(null); - Result res = testRequest(HttpRequest.createTestRequest("remove?id[0]=userdoc:footype:1234:foo&id[1]=userdoc:footype:1234:foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); - assertEquals(2, res.messages.size()); - - { - Message m = res.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - } - - { - Message m = res.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); - } - } - - @Test - public void testFeedHandlerStatusCreation() throws Exception { - VespaFeedHandlerStatus status = new VespaFeedHandlerStatus( - new FeedContext(new MessagePropertyProcessor( - new FeederConfig(new FeederConfig.Builder()), - new LoadTypeConfig(new LoadTypeConfig.Builder())), - factory, null, new ClusterList(), new NullFeedMetric(true)), - true, true, - Executors.newCachedThreadPool()); - } - - private class TestDocProc extends DocumentProcessor { - @Override - public Progress process(Processing processing) { - for (DocumentOperation op : processing.getDocumentOperations()) { - if (op instanceof DocumentPut) { - Document document = ((DocumentPut)op).getDocument(); - document.setFieldValue("last_downloaded", new IntegerFieldValue(1234)); - } - } - return Progress.DONE; - } - } - - private class TestLaterDocProc extends DocumentProcessor { - private final Logger log = Logger.getLogger(TestLaterDocProc.class.getName()); - - private int counter = 0; - @Override - public Progress process(Processing processing) { - synchronized (this) { - counter++; - if (counter % 2 == 1) { - log.info("Returning LATER."); - return Progress.LATER; - } - log.info("Returning DONE."); - return Progress.DONE; - } - } - } - - private Result testRequest(HttpRequest req) throws Exception { - HttpResponse response = null; - String feedPrefix = "feed"; - String removePrefix = "remove"; - String feedStatusPrefix = "feedstatus"; - String removeLocationPrefix = "removelocation"; - - if (req.getUri().getPath().startsWith(feedPrefix)) { - response = feedHandler.handle(req); - } - if (req.getUri().getPath().startsWith(removePrefix)) { - response = removeHandler.handle(req); - } - if (req.getUri().getPath().startsWith(feedStatusPrefix)) { - response = statusHandler.handle(req); - } - if (req.getUri().getPath().startsWith(removeLocationPrefix)) { - response = removeLocationHandler.handle(req); - } - - ByteArrayOutputStream output = new ByteArrayOutputStream(); - response.render(output); - - Result res = new Result(); - res.messages = factory.messages; - res.output = new String(output.toByteArray()); - - if (response instanceof FeedResponse) { - FeedResponse feedResponse = (FeedResponse)response; - res.error = feedResponse.getErrorMessageList().isEmpty() ? null : feedResponse.getErrorMessageList().get(0); - res.errorCount = feedResponse.getErrorMessageList().size(); - assertTrue(feedResponse.isSuccess() == (res.errorCount == 0)); - } - return res; - } - - private Result testFeed(String xmlFile, String request) throws Exception { - return testRequest(new FileRequest(new File(xmlFile), request).toRequest()); - } - - private Result testFeedGZIP(String xmlFile, String request) throws Exception { - return testRequest(new FileRequest(new File(xmlFile), request, true).toRequest()); - } - - private class FileRequest { - - private final String req; - private final File f; - private boolean gzip = false; - - FileRequest(File f, String req) { - this.req = req; - this.f = f; - } - - FileRequest(File f, String req, boolean gzip) { - this.f = f; - this.req = req; - this.gzip = gzip; - } - - public InputStream getData() { - try { - InputStream fileStream = new FileInputStream(f); - if (gzip) { - // Not exactly pretty, but in lack of an elegant way of transcoding - ByteArrayOutputStream rawOut = new ByteArrayOutputStream(); - GZIPOutputStream compressed = new GZIPOutputStream(rawOut); - byte[] buffer = new byte[1024]; - int read = -1; - while (true) { - read = fileStream.read(buffer); - if (read == -1) break; - compressed.write(buffer, 0, read); - } - compressed.finish(); - compressed.flush(); - rawOut.flush(); - return new ByteArrayInputStream(rawOut.toByteArray()); - } - return fileStream; - } catch (Exception e) { - return null; - } - } - - public void addHeaders(HeaderFields headers) { - headers.add("Content-Type", "image/jpeg"); - if (gzip) - headers.add("Content-Encoding", "gzip"); - } - - public HttpRequest toRequest() { - HttpRequest request = HttpRequest.createTestRequest(req, com.yahoo.jdisc.http.HttpRequest.Method.GET, getData()); - addHeaders(request.getJDiscRequest().headers()); - return request; - } - - } - - private class Result { - private List<Message> messages; - private String output; - private com.yahoo.processing.request.ErrorMessage error; - private int errorCount; - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java deleted file mode 100644 index b7a911109cc..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.document.BucketId; -import com.yahoo.documentapi.ProgressToken; -import com.yahoo.documentapi.VisitorIterator; -import org.junit.Test; - -import java.util.Set; -import java.util.TreeSet; - -import static org.junit.Assert.*; - -@SuppressWarnings("deprecation") -public class ContinuationHitTest { - - private static final String SINGLE_BUCKET_URL_SAFE_BASE64 - = "AAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAHqNFZ4mrz-_wAAAAAAAAAA"; - private static final String MULTI_BUCKET_URL_SAFE_BASE64 - = "AAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAPqNFZ4mrz--gAAAAAAAAAA6" + - "jRWeJq8_vsAAAAAAAAAAOo0VniavP7_AAAAAAAAAAA="; - - @Test - public void continuationTokensAreUrlSafeBase64Encoded() throws Exception { - ContinuationHit hit = new ContinuationHit(createSingleBucketProgress()); - // We want -_ instead of +/ - assertEquals(SINGLE_BUCKET_URL_SAFE_BASE64, hit.getValue()); - } - - @Test - public void continuationTokensAreNotBrokenIntoMultipleLines() throws Exception { - ContinuationHit hit = new ContinuationHit(createMultiBucketProgress()); - assertTrue(hit.getValue().length() > 76); // Ensure we exceed MIME line length limits. - assertFalse(hit.getValue().contains("\n")); - } - - @Test - public void decodingAcceptsUrlSafeTokens() throws Exception { - final ProgressToken token = ContinuationHit.getToken(SINGLE_BUCKET_URL_SAFE_BASE64); - // Roundtrip should yield identical results. - assertEquals(SINGLE_BUCKET_URL_SAFE_BASE64, - new ContinuationHit(token).getValue()); - } - - /** - * Legacy Base64 encoder emitted MIME Base64. Ensure we handle tokens from that era. - */ - @Test - public void decodingAcceptsLegacyNonUrlSafeTokens() throws Exception { - final String legacyBase64 = convertedToMimeBase64Chars(SINGLE_BUCKET_URL_SAFE_BASE64); - final ProgressToken legacyToken = ContinuationHit.getToken(legacyBase64); - - assertEquals(SINGLE_BUCKET_URL_SAFE_BASE64, - new ContinuationHit(legacyToken).getValue()); - } - - /** - * Legacy Base64 encoder would happily output line breaks after each MIME line - * boundary. Ensure we handle these gracefully. - */ - @Test - public void decodingAcceptsLegacyMimeLineBrokenTokens() throws Exception { - final String multiBucketLegacyToken = - "AAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAPqNFZ4mrz++gAAAAAAAAAA6jRWeJq8/vsA\r\n" + - "AAAAAAAAAOo0VniavP7/AAAAAAAAAAA="; - final ProgressToken legacyToken = ContinuationHit.getToken(multiBucketLegacyToken); - - assertEquals(MULTI_BUCKET_URL_SAFE_BASE64, - new ContinuationHit(legacyToken).getValue()); - } - - /** - * Returns a ProgressToken whose base 64 representation will be _less_ than 76 bytes (MIME line limit) - */ - private ProgressToken createSingleBucketProgress() { - ProgressToken token = new ProgressToken(16); - // Use explicit bucket set so we can better control the binary representation - // of the buckets, and thus the values written as base 64. - Set<BucketId> buckets = new TreeSet<>(); - // This particular bucket ID will contain +/ chars when output as non-URL safe base 64. - buckets.add(new BucketId(58, 0x123456789abcfeffL)); - VisitorIterator.createFromExplicitBucketSet(buckets, 16, token); // "Prime" the token. - return token; - } - - /** - * Returns a ProgressToken whose base 64 representation will be _more_ than 76 bytes (MIME line limit) - */ - private ProgressToken createMultiBucketProgress() { - ProgressToken token = new ProgressToken(16); - Set<BucketId> buckets = new TreeSet<>(); - buckets.add(new BucketId(58, 0x123456789abcfeffL)); - buckets.add(new BucketId(58, 0x123456789abcfefaL)); - buckets.add(new BucketId(58, 0x123456789abcfefbL)); - VisitorIterator.createFromExplicitBucketSet(buckets, 16, token); // "Prime" the token. - return token; - } - - private String convertedToMimeBase64Chars(String token) { - // Doesn't split on MIME line boundaries, so not fully MIME compliant. - return token.replace('-', '+').replace('_', '/'); - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java deleted file mode 100755 index 2015efa4d5d..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.document.Document; -import com.yahoo.document.DocumentId; -import com.yahoo.document.DocumentType; -import com.yahoo.documentapi.VisitorParameters; -import com.yahoo.documentapi.VisitorSession; -import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; -import com.yahoo.feedapi.DummySessionFactory; -import com.yahoo.feedapi.SendSession; -import com.yahoo.jdisc.Metric; -import com.yahoo.messagebus.*; -import com.yahoo.messagebus.Error; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.HashMap; - -/** - * Used to automatically reply with a GetDocumentReply for every received GetDocumentMessage - */ -public class DocumentSessionFactory extends DummySessionFactory { - - private DocumentType docType; - private Error error; - // Reply instances are shared between the two collections. - List<GetDocumentReply> autoReplies; - Map<DocumentId, GetDocumentReply> autoReplyLookup = new HashMap<>(); - boolean autoReply = true; - boolean nullReply = false; - private int sessionsCreated = 0; - - private class DocumentReplySession extends SendSession { - - ReplyHandler handler; - Error e; - DummySessionFactory owner; - - public DocumentReplySession(ReplyHandler handler, Error e, DummySessionFactory owner) { - this.handler = handler; - this.e = e; - this.owner = owner; - } - - protected Result onSend(Message m, boolean blockIfQueueFull) throws InterruptedException { - if (!(m instanceof GetDocumentMessage)) { - throw new IllegalArgumentException("Expected GetDocumentMessage"); - } - GetDocumentMessage gm = (GetDocumentMessage)m; - owner.messages.add(m); - if (autoReply) { - Document replyDoc; - if (!nullReply) { - replyDoc = new Document(docType, gm.getDocumentId()); - } else { - replyDoc = null; - } - Reply r = new GetDocumentReply(replyDoc); - r.setMessage(m); - r.setContext(m.getContext()); - if (e != null) { - r.addError(e); - } - handler.handleReply(r); - } else if (owner.messages.size() == autoReplies.size()) { - // Pair up all replies with their messages - for (Message msg : owner.messages) { - GetDocumentReply reply = autoReplyLookup.get(((GetDocumentMessage)msg).getDocumentId()); - reply.setMessage(msg); - reply.setContext(msg.getContext()); - if (e != null) { - reply.addError(e); - } - } - // Now send them in the correct order. Instances are shared, so source - // messages and contexts are properly set - for (Reply reply : autoReplies) { - handler.handleReply(reply); - } - } - - return Result.ACCEPTED; - } - - public void close() { - } - } - - public DocumentSessionFactory(DocumentType docType) { - this.docType = docType; - this.error = null; - } - - public DocumentSessionFactory(DocumentType docType, Error error, boolean autoReply, GetDocumentReply... autoReplies) { - this.docType = docType; - this.error = error; - this.autoReplies = Arrays.asList(autoReplies); - for (GetDocumentReply reply : autoReplies) { - this.autoReplyLookup.put(reply.getDocument().getId(), reply); - } - this.autoReply = autoReply; - } - - public boolean isNullReply() { - return nullReply; - } - - public void setNullReply(boolean nullReply) { - this.nullReply = nullReply; - } - - public int getSessionsCreated() { - return sessionsCreated; - } - - public SendSession createSendSession(ReplyHandler r, Metric metric) { - ++sessionsCreated; - return new DocumentReplySession(r, error, this); - } - - @Override - public VisitorSession createVisitorSession(VisitorParameters p) { - return new DummyVisitorSession(p, docType); - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java deleted file mode 100644 index 02d26f8bd93..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.document.Document; -import com.yahoo.document.DocumentId; -import com.yahoo.document.DocumentPut; -import com.yahoo.document.DocumentType; -import com.yahoo.documentapi.*; -import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; -import com.yahoo.messagebus.Message; -import com.yahoo.messagebus.Trace; - -import java.util.ArrayList; -import java.util.List; - -/** - * Stub to test visitors. - */ -public class DummyVisitorSession implements VisitorSession { - - final VisitorParameters parameters; - final DocumentType documentType; - final List<Message> autoReplyMessages = new ArrayList<>(); - - DummyVisitorSession(VisitorParameters p, DocumentType documentType) { - parameters = p; - this.documentType = documentType; - p.getLocalDataHandler().setSession(this); - addDefaultReplyMessages(); - } - - @Override - public boolean isDone() { - return true; - } - - @Override - public ProgressToken getProgress() { - return new ProgressToken(12); - } - - @Override - public Trace getTrace() { - return null; - } - - public void addDocumentReply(String docId) { - Document replyDoc = new Document(documentType, docId); - autoReplyMessages.add(new PutDocumentMessage(new DocumentPut(replyDoc))); - } - - public void addRemoveReply(String docId) { - autoReplyMessages.add(new RemoveDocumentMessage(new DocumentId(docId))); - } - - public void addDefaultReplyMessages() { - addDocumentReply("userdoc:foo:1234:bar"); - if (parameters.visitRemoves()) { - addRemoveReply("userdoc:foo:1234:removed"); - } - } - - public void clearAutoReplyMessages() { - autoReplyMessages.clear(); - } - - @Override - public boolean waitUntilDone(long l) throws InterruptedException { - for (Message msg : autoReplyMessages) { - parameters.getLocalDataHandler().onMessage(msg, new AckToken(this)); - } - return true; - } - - @Override - public void ack(AckToken ackToken) { - } - - @Override - public void abort() { - } - - @Override - public VisitorResponse getNext() { - return null; - } - - @Override - public VisitorResponse getNext(int i) throws InterruptedException { - return null; - } - - @Override - public void destroy() { - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java deleted file mode 100755 index 14fe0dc857d..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java +++ /dev/null @@ -1,1093 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.component.chain.Chain; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.feedhandler.NullFeedMetric; -import com.yahoo.jdisc.HeaderFields; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.document.*; -import com.yahoo.document.datatypes.IntegerFieldValue; -import com.yahoo.document.datatypes.Raw; -import com.yahoo.document.datatypes.StringFieldValue; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; -import com.yahoo.feedapi.FeedContext; -import com.yahoo.feedapi.MessagePropertyProcessor; -import com.yahoo.messagebus.Message; -import com.yahoo.messagebus.routing.Route; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.vespaclient.ClusterList; -import com.yahoo.vespaclient.config.FeederConfig; - -import org.junit.Test; - -import java.io.*; -import java.nio.ByteBuffer; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.zip.GZIPOutputStream; - -import static org.junit.Assert.*; - -@SuppressWarnings("deprecation") -public class GetSearcherTestCase { - - private DocumentTypeManager docMan = null; - private DocumentType docType; - private FeederConfig defFeedCfg = new FeederConfig(new FeederConfig.Builder()); - private LoadTypeConfig defLoadTypeCfg = new LoadTypeConfig(new LoadTypeConfig.Builder()); - - @org.junit.Before - public void setUp() { - docMan = new DocumentTypeManager(); - docType = new DocumentType("kittens"); - docType.addHeaderField("name", DataType.STRING); - docType.addField("description", DataType.STRING); - docType.addField("image", DataType.STRING); - docType.addField("fluffiness", DataType.INT); - docType.addField("foo", DataType.RAW); - docMan.registerDocumentType(docType); - } - - @org.junit.After - public void tearDown() { - docMan = null; - docType = null; - } - - private void assertHits(HitGroup hits, String... wantedHits) { - assertEquals(wantedHits.length, hits.size()); - for (int i = 0; i < wantedHits.length; ++i) { - assertTrue(hits.get(i) instanceof DocumentHit); - DocumentHit hit = (DocumentHit)hits.get(i); - assertEquals(wantedHits[i], hit.getDocument().getId().toString()); - } - } - - @Test - public void testGetSingleDocumentQuery() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); // Needs auto-reply - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=userdoc:kittens:1:2")); - System.out.println("HTTP request is " + result.getQuery().getHttpRequest()); - - assertEquals(1, factory.messages.size()); - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:1:2", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - } - assertEquals(1, result.hits().size()); - assertHits(result.hits(), "userdoc:kittens:1:2"); - // By default, document hit should not have its hit fields set - DocumentHit hit = (DocumentHit)result.hits().get(0); - assertEquals(0, hit.fieldKeys().size()); - } - - @Test - public void testGetMultipleDocumentsQuery() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Query query = newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4"); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); - - assertEquals(2, factory.messages.size()); - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:1:2", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - } - - { - Message m = factory.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:3:4", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - } - assertEquals(2, result.hits().size()); - assertNull(result.hits().getErrorHit()); - assertHits(result.hits(), "userdoc:kittens:1:2", "userdoc:kittens:3:4"); - assertEquals(2, query.getHits()); - } - - // Test that you can use both query string and POSTed IDs - @Test - public void testGetMultipleDocumentsQueryAndPOST() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - String data = "userdoc:kittens:5:6\nuserdoc:kittens:7:8\nuserdoc:kittens:9:10"; - MockHttpRequest request = new MockHttpRequest(data.getBytes("utf-8"), "/get/?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4"); - Query query = new Query(request.toRequest()); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); - - assertEquals(5, factory.messages.size()); - assertEquals(5, result.hits().size()); - assertNull(result.hits().getErrorHit()); - assertHits(result.hits(), "userdoc:kittens:1:2", "userdoc:kittens:3:4", - "userdoc:kittens:5:6", "userdoc:kittens:7:8", "userdoc:kittens:9:10"); - } - - @Test - public void testGetMultipleDocumentsQueryAndGZippedPOST() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - String data = "userdoc:kittens:5:6\nuserdoc:kittens:7:8\nuserdoc:kittens:9:10"; - - // Create with automatic GZIP encoding - MockHttpRequest request = new MockHttpRequest(data.getBytes("utf-8"), "/get/?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4", true); - Query query = new Query(request.toRequest()); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); - - assertEquals(5, factory.messages.size()); - assertEquals(5, result.hits().size()); - assertNull(result.hits().getErrorHit()); - assertHits(result.hits(), "userdoc:kittens:1:2", "userdoc:kittens:3:4", - "userdoc:kittens:5:6", "userdoc:kittens:7:8", "userdoc:kittens:9:10"); - } - - /* Test that a query without any ids is passed through to the next chain */ - @Test - public void testQueryPassThrough() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - HitGroup hits = new HitGroup("mock"); - hits.add(new Hit("blernsball")); - Chain<Searcher> searchChain = new Chain<>(searcher, new MockBackend(hits)); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?flarn=blern")); - - assertEquals(0, factory.messages.size()); - assertEquals(1, result.hits().size()); - assertNotNull(result.hits().get("blernsball")); - } - - /* Test that a query will contain both document hits and hits from a searcher - * further down the chain, iff the searcher returns a DocumentHit. - */ - @Test - public void testQueryPassThroughAndGet() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:1234:foo")); - doc1.setFieldValue("name", new StringFieldValue("megacat")); - doc1.setFieldValue("description", new StringFieldValue("supercat")); - doc1.setFieldValue("fluffiness", new IntegerFieldValue(10000)); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1) - }; - - DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, false, replies); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - DocumentHit backendHit = new DocumentHit(new Document(docType, new DocumentId("userdoc:kittens:5678:bar")), 5); - Chain<Searcher> searchChain = new Chain<>(searcher, new MockBackend(backendHit)); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?query=flarn&id=userdoc:kittens:1234:foo")); - - assertEquals(1, factory.messages.size()); - assertEquals(2, result.hits().size()); - assertNotNull(result.hits().get("userdoc:kittens:5678:bar")); - assertNotNull(result.hits().get("userdoc:kittens:1234:foo")); - } - - @Test - public void testQueryPassThroughAndGetUnknownBackendHit() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - HitGroup hits = new HitGroup("mock"); - hits.add(new Hit("blernsball")); - Chain<Searcher> searchChain = new Chain<>(searcher, new MockBackend(hits)); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?flarn=blern&id=userdoc:kittens:9:aaa")); - - assertEquals(0, factory.messages.size()); - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"18\" message=\"Internal server error.: " + - "A backend searcher to com.yahoo.storage.searcher.GetSearcher returned a " + - "hit that was not an instance of com.yahoo.storage.searcher.DocumentHit. " + - "Only DocumentHit instances are supported in the backend hit result set " + - "when doing queries that contain document identifier sets recognised by the " + - "Get Searcher.\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testConfig() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(58).route("route66").retryenabled(false)), defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=doc:batman:dahnahnahnah")); - - assertEquals(1, factory.messages.size()); - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("doc:batman:dahnahnahnah", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - assertEquals(Route.parse("route66"), gdm.getRoute()); - assertFalse(gdm.getRetryEnabled()); - assertTrue(58000 >= gdm.getTimeRemaining()); - } - } - - @Test - public void testConfigChanges() throws Exception { - String config = "raw:timeout 37\nroute \"riksveg18\"\nretryenabled true"; - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(58).route("riksveg18").retryenabled(true)), - defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=doc:batman:dahnahnahnah")); - - assertEquals(1, factory.messages.size()); - assertEquals(1, factory.getSessionsCreated()); - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("doc:batman:dahnahnahnah", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - assertEquals(Route.parse("riksveg18"), gdm.getRoute()); - assertTrue(gdm.getRetryEnabled()); - assertTrue(58000 >= gdm.getTimeRemaining()); - } - - factory.messages.clear(); - - FeederConfig newConfig = new FeederConfig(new FeederConfig.Builder() - .timeout(123) - .route("e6") - .retryenabled(false) - ); - searcher.getMessagePropertyProcessor().configure(newConfig, defLoadTypeCfg); - - new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=doc:spiderman:does_whatever_a_spider_can")); - - // riksveg18 is created again, and e6 is created as well. - assertEquals(3, factory.getSessionsCreated()); - - assertEquals(1, factory.messages.size()); - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("doc:spiderman:does_whatever_a_spider_can", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - assertEquals(Route.parse("e6"), gdm.getRoute()); - assertFalse(gdm.getRetryEnabled()); - assertTrue(123000 >= gdm.getTimeRemaining()); - } - } - - @Test - public void testQueryOverridesDefaults() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4&priority=LOW_2&route=highwaytohell&timeout=58")); - - assertEquals(2, factory.messages.size()); - long lastTimeout = 58000; - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:1:2", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority()); - assertEquals(Route.parse("highwaytohell"), gdm.getRoute()); - assertTrue(lastTimeout >= gdm.getTimeRemaining()); - lastTimeout = gdm.getTimeRemaining(); - } - - { - Message m = factory.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:3:4", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority()); - assertEquals(Route.parse("highwaytohell"), gdm.getRoute()); - assertTrue(lastTimeout >= gdm.getTimeRemaining()); - } - } - - @Test - public void testQueryOverridesConfig() throws Exception { - String config = "raw:timeout 458\nroute \"route66\""; - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4&priority=LOW_2&route=highwaytohell&timeout=123")); - - long lastTimeout = 123000; - assertEquals(2, factory.messages.size()); - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:1:2", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority()); - assertEquals(Route.parse("highwaytohell"), gdm.getRoute()); - assertTrue(lastTimeout >= gdm.getTimeRemaining()); - lastTimeout = gdm.getTimeRemaining(); - } - - { - Message m = factory.messages.get(1); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:3:4", d.toString()); - assertEquals("[all]", gdm.getFieldSet()); - assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority()); - assertEquals(Route.parse("highwaytohell"), gdm.getRoute()); - assertTrue(lastTimeout >= gdm.getTimeRemaining()); - } - } - - @Test - public void testBadPriorityValue() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:1:2&priority=onkel_jubalon")); - - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + - "java.lang.IllegalArgumentException: No enum const" + - "ant " + - "com.yahoo.documentapi.messagebus.protocol.DocumentProtocol" + - "." + - "Priority.onkel_jubalon\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testMultiIdBadArrayIndex() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - { - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[1]=userdoc:kittens:1:2")); - - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + - "java.lang.IllegalArgumentException: query contains document ID " + - "array that is not zero-based and/or linearly increasing\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - { - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:1:2&id[2]=userdoc:kittens:2:3")); - - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + - "java.lang.IllegalArgumentException: query contains document ID " + - "array that is not zero-based and/or linearly increasing\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - { - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[1]=userdoc:kittens:2:3")); - - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + - "java.lang.IllegalArgumentException: query contains document ID " + - "array that is not zero-based and/or linearly increasing\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - { - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0=userdoc:kittens:1:2")); - - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + - "java.lang.IllegalArgumentException: Malformed document ID array parameter\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - } - - @Test - public void testLegacyHeadersOnly() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); // Needs auto-reply - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:1:2&headersonly=true")); - - assertEquals(1, factory.messages.size()); - { - Message m = factory.messages.get(0); - assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); - GetDocumentMessage gdm = (GetDocumentMessage)m; - DocumentId d = gdm.getDocumentId(); - assertEquals("userdoc:kittens:1:2", d.toString()); - assertEquals("[header]", gdm.getFieldSet()); - } - assertEquals(1, result.hits().size()); - assertHits(result.hits(), "userdoc:kittens:1:2"); - } - - @Test - public void testFieldSet() throws Exception { - } - - @Test - public void testConsistentResultOrdering() throws Exception { - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(new Document(docType, new DocumentId("userdoc:kittens:1:2"))), - new GetDocumentReply(new Document(docType, new DocumentId("userdoc:kittens:7:8"))), - new GetDocumentReply(new Document(docType, new DocumentId("userdoc:kittens:555:123"))) - }; - - // Use a predefined reply list to ensure messages are answered out of order - DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, false, replies); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:555:123&id[1]=userdoc:kittens:1:2&id[2]=userdoc:kittens:7:8")); - - assertEquals(3, factory.messages.size()); - assertEquals(3, result.hits().size()); - // Hits must be in the same order as their document IDs in the query - assertHits(result.hits(), "userdoc:kittens:555:123", "userdoc:kittens:1:2", "userdoc:kittens:7:8"); - - assertEquals(0, ((DocumentHit)result.hits().get(0)).getIndex()); - assertEquals(1, ((DocumentHit)result.hits().get(1)).getIndex()); - assertEquals(2, ((DocumentHit)result.hits().get(2)).getIndex()); - } - - @Test - public void testResultWithSingleError() throws Exception { - com.yahoo.messagebus.Error err = new com.yahoo.messagebus.Error(32, "Alas, it went poorly"); - DocumentSessionFactory factory = new DocumentSessionFactory(docType, err, true); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4")); - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"messagebus\" code=\"32\" message=\"Alas, it went poorly\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testResultWithMultipleErrors() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:77:88")); - Document doc2 = new Document(docType, new DocumentId("userdoc:kittens:99:111")); - GetDocumentReply errorReply1 = new GetDocumentReply(doc1); - errorReply1.addError(new com.yahoo.messagebus.Error(123, "userdoc:kittens:77:88 had fleas.")); - GetDocumentReply errorReply2 = new GetDocumentReply(doc2); - errorReply2.addError(new com.yahoo.messagebus.Error(456, "userdoc:kittens:99:111 shredded the curtains.")); - GetDocumentReply[] replies = new GetDocumentReply[] { - errorReply1, - errorReply2 - }; - - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:77:88&id[1]=userdoc:kittens:99:111")); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"messagebus\" code=\"123\" message=\"userdoc:kittens:77:88 had fleas.\"/>\n" + - "<error type=\"messagebus\" code=\"456\" message=\"userdoc:kittens:99:111 shredded the curtains.\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testResultWithNullDocument() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, true); - factory.setNullReply(true); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:55:bad_document_id")); - // Document not found does not produce any hit at all, error or otherwise - assertNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "</result>\n", result); - } - - @Test - public void testDefaultDocumentHitRendering() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:3:4")); - doc1.setFieldValue("name", new StringFieldValue("mittens")); - doc1.setFieldValue("description", new StringFieldValue("it's a cat")); - doc1.setFieldValue("fluffiness", new IntegerFieldValue(8)); - Document doc2 = new Document(docType, new DocumentId("userdoc:kittens:1:2")); - doc2.setFieldValue("name", new StringFieldValue("garfield")); - doc2.setFieldValue("description", - new StringFieldValue("preliminary research indicates <em>hatred</em> of mondays. caution advised")); - doc2.setFieldValue("fluffiness", new IntegerFieldValue(2)); - Document doc3 = new Document(docType, new DocumentId("userdoc:kittens:77:88")); - GetDocumentReply errorReply = new GetDocumentReply(doc3); - errorReply.addError(new com.yahoo.messagebus.Error(123, "userdoc:kittens:77:88 had fleas.")); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1), - new GetDocumentReply(doc2), - errorReply - }; - - // Use a predefined reply list to ensure messages are answered out of order - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result xmlResult = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:77:88&id[1]=userdoc:kittens:1:2&id[2]=userdoc:kittens:3:4")); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"messagebus\" code=\"123\" message=\"userdoc:kittens:77:88 had fleas.\"/>\n" + - "</errors>\n" + - "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:1:2\">\n" + - " <name>garfield</name>\n" + - " <description>preliminary research indicates <em>hatred</em> of mondays. caution advised</description>\n" + - " <fluffiness>2</fluffiness>\n" + - "</document>\n" + - "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:3:4\">\n" + - " <name>mittens</name>\n" + - " <description>it's a cat</description>\n" + - " <fluffiness>8</fluffiness>\n" + - "</document>\n" + - "</result>\n", xmlResult); - } - - @Test - public void testDocumentFieldNoContentType() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); - doc1.setFieldValue("name", "derrick"); - doc1.setFieldValue("description", "kommisar katze"); - doc1.setFieldValue("fluffiness", 0); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1), - }; - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:5:1&field=description")); - - assertNull(result.hits().getErrorHit()); - assertEquals("text/xml", result.getTemplating().getTemplates().getMimeType()); - assertEquals("UTF-8", result.getTemplating().getTemplates().getEncoding()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>kommisar katze</result>\n", result); - } - - @Test - public void testDocumentFieldEscapeXML() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); - doc1.setFieldValue("name", "asfd"); - doc1.setFieldValue("description", "<script type=\"evil/madness\">horror & screams</script>"); - doc1.setFieldValue("fluffiness", 0); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1), - }; - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:5:1&field=description")); - - assertNull(result.hits().getErrorHit()); - assertEquals("text/xml", result.getTemplating().getTemplates().getMimeType()); - assertEquals("UTF-8", result.getTemplating().getTemplates().getEncoding()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result><script type=\"evil/madness\">horror & screams</script></result>\n", result); - } - - @Test - public void testDocumentFieldRawContent() throws Exception { - byte[] contentBytes = new byte[] { 0, -128, 127 }; - - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:123:456")); - doc1.setFieldValue("foo", new Raw(ByteBuffer.wrap(contentBytes))); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1) - }; - - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:123:456&field=foo")); - - assertNull(result.hits().getErrorHit()); - assertEquals("application/octet-stream", result.getTemplating().getTemplates().getMimeType()); - - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - com.yahoo.prelude.templates.SearchRendererAdaptor.callRender(stream, result); - stream.flush(); - - byte[] resultBytes = stream.toByteArray(); - assertEquals(contentBytes.length, resultBytes.length); - for (int i = 0; i < resultBytes.length; ++i) { - assertEquals(contentBytes[i], resultBytes[i]); - } - } - - @Test - public void testDocumentFieldRawWithContentOverride() throws Exception { - byte[] contentBytes = new byte[] { 0, -128, 127 }; - - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:123:456")); - doc1.setFieldValue("foo", new Raw(ByteBuffer.wrap(contentBytes))); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1) - }; - - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:123:456&field=foo&contenttype=text/fancy")); - - assertNull(result.hits().getErrorHit()); - assertEquals("text/fancy", result.getTemplating().getTemplates().getMimeType()); - - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - com.yahoo.prelude.templates.SearchRendererAdaptor.callRender(stream, result); - stream.flush(); - - byte[] resultBytes = stream.toByteArray(); - assertEquals(contentBytes.length, resultBytes.length); - for (int i = 0; i < resultBytes.length; ++i) { - assertEquals(contentBytes[i], resultBytes[i]); - } - } - - @Test - public void testDocumentFieldWithMultipleIDs() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4&field=name")); - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + - "java.lang.IllegalArgumentException: Field only valid for single document id query\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testDocumentFieldNotSet() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); - doc1.setFieldValue("name", "asdf"); - doc1.setFieldValue("description", "fdsafsdf"); - doc1.setFieldValue("fluffiness", 10); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1), - }; - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:5:1&field=image")); - - assertNotNull(result.hits().getErrorHit()); - assertEquals(1, result.hits().size()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"16\" message=\"Resource not found.: " + - "Field 'image' found in document type, but had no content in userdoc:kittens:5:1\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - - @Test - public void testDocumentFieldWithDocumentNotFound() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, true); - factory.setNullReply(true); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:1:2&field=name")); - assertNotNull(result.hits().getErrorHit()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"16\" message=\"Resource not found.: " + - "Document not found, could not return field 'name'\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testDocumentFieldNotReachableWithHeadersOnly() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); - doc1.setFieldValue("name", "asdf"); - // don't set body fields - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1), - }; - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:5:1&field=description&headersonly=true")); - - assertNotNull(result.hits().getErrorHit()); - assertEquals(1, result.hits().size()); - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"4\" message=\"Invalid query parameter: " + - "Field 'description' is located in document body, but headersonly " + - "prevents it from being retrieved in userdoc:kittens:5:1\"/>\n" + - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testVespaXMLTemplate() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:3:4")); - doc1.setFieldValue("name", "mittens"); - doc1.setFieldValue("description", "it's a cat"); - doc1.setFieldValue("fluffiness", 8); - Document doc2 = new Document(docType, new DocumentId("userdoc:kittens:1:2")); - doc2.setFieldValue("name", "garfield"); - doc2.setFieldValue("description", "preliminary research indicates <em>hatred</em> of mondays. caution advised"); - doc2.setFieldValue("fluffiness", 2); - Document doc3 = new Document(docType, new DocumentId("userdoc:kittens:77:88")); - GetDocumentReply errorReply = new GetDocumentReply(doc3); - errorReply.addError(new com.yahoo.messagebus.Error(123, "userdoc:kittens:77:88 lost in a <ni!>\"shrubbery\"</ni!>")); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1), - new GetDocumentReply(doc2), - errorReply - }; - - // Use a predefined reply list to ensure messages are answered out of order - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id[0]=userdoc:kittens:77:88&id[1]=userdoc:kittens:1:2&id[2]=userdoc:kittens:3:4")); // TODO! - - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"messagebus\" code=\"123\" message=\"userdoc:kittens:77:88 lost in a <ni!>"shrubbery"</ni!>\"/>\n"+ - "</errors>\n" + - "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:1:2\">\n" + - " <name>garfield</name>\n" + - " <description>preliminary research indicates <em>hatred</em> of mondays. caution advised</description>\n" + - " <fluffiness>2</fluffiness>\n" + - "</document>\n" + - "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:3:4\">\n" + - " <name>mittens</name>\n" + - " <description>it's a cat</description>\n" + - " <fluffiness>8</fluffiness>\n" + - "</document>\n" + - "</result>\n", result); - } - - @Test - public void testDocumentHitWithPopulatedHitFields() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:1234:foo")); - doc1.setFieldValue("name", new StringFieldValue("megacat")); - doc1.setFieldValue("description", new StringFieldValue("supercat")); - doc1.setFieldValue("fluffiness", new IntegerFieldValue(10000)); - GetDocumentReply[] replies = new GetDocumentReply[] { - new GetDocumentReply(doc1) - }; - - // Use a predefined reply list to ensure messages are answered out of order - Chain<Searcher> searchChain = createSearcherChain(replies); - - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("?id=userdoc:kittens:1234:foo&populatehitfields=true")); - assertEquals(1, result.hits().size()); - assertHits(result.hits(), "userdoc:kittens:1234:foo"); - - DocumentHit hit = (DocumentHit)result.hits().get(0); - Iterator<Map.Entry<String, Object>> iter = hit.fieldIterator(); - Set<String> fieldSet = new TreeSet<>(); - while (iter.hasNext()) { - Map.Entry<String, Object> kv = iter.next(); - StringBuilder field = new StringBuilder(); - field.append(kv.getKey()).append(" -> ").append(kv.getValue()); - fieldSet.add(field.toString()); - } - StringBuilder fields = new StringBuilder(); - for (String s : fieldSet) { - fields.append(s).append("\n"); - } - assertEquals( - "description -> supercat\n" + - "documentid -> userdoc:kittens:1234:foo\n" + - "fluffiness -> 10000\n" + - "name -> megacat\n", - fields.toString()); - } - - @Test - public void deserializationExceptionsAreHandledGracefully() throws Exception { - Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); - GetDocumentReply[] replies = new GetDocumentReply[] { - new MockFailingGetDocumentReply(doc1), - }; - Chain<Searcher> searchChain = createSearcherChain(replies); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=userdoc:kittens:5:1")); - assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<errors>\n" + - "<error type=\"searcher\" code=\"18\" message=\"Internal server error.: " + - "Got exception of type java.lang.RuntimeException during document " + - "deserialization: epic dragon attack\"/>\n"+ - "</errors>\n" + - "</result>\n", result); - } - - @Test - public void testJsonRendererSetting() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); // Needs auto-reply - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - Chain<Searcher> searchChain = new Chain<>(searcher); - - Query query = newQuery("?id=userdoc:kittens:1:2&format=json"); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); - assertFalse(result.getTemplating().getTemplates() instanceof DocumentXMLTemplate); - } - - private Query newQuery(String queryString) { - return new Query(HttpRequest.createTestRequest(queryString, com.yahoo.jdisc.http.HttpRequest.Method.GET)); - } - - private Chain<Searcher> createSearcherChain(GetDocumentReply[] replies) throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, false, replies); - GetSearcher searcher = new GetSearcher(new FeedContext( - new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), - factory, docMan, new ClusterList(), new NullFeedMetric(true))); - return new Chain<>(searcher); - } - - private static class MockFailingGetDocumentReply extends GetDocumentReply { - private int countdown = 2; - - private MockFailingGetDocumentReply(Document doc) { - super(doc); - } - - @Override - public Document getDocument() { - // Reason for countdown is that the test DocumentSessionFactory calls - // getDocument once internally before code can ever reach handleReply. - if (--countdown == 0) { - throw new RuntimeException("epic dragon attack"); - } - return super.getDocument(); - } - } - - private static class MockBackend extends Searcher { - private Hit hitToReturn; - - public MockBackend(Hit hitToReturn) { - this.hitToReturn = hitToReturn; - } - - @Override - public Result search(Query query, Execution execution) { - Result result = new Result(query); - result.hits().add(hitToReturn); - return result; - } - } - - private class MockHttpRequest { - - private final String req; - private byte[] data; - private boolean gzip = false; - - MockHttpRequest(byte[] data, String req) { - this.req = req; - this.data = data; - } - - MockHttpRequest(byte[] data, String req, boolean gzip) { - this.data = data; - this.req = req; - this.gzip = gzip; - } - - public InputStream getData() { - if (gzip) { - try { - ByteArrayOutputStream rawOut = new ByteArrayOutputStream(); - GZIPOutputStream compressed = new GZIPOutputStream(rawOut); - compressed.write(data, 0, data.length); - compressed.finish(); - compressed.flush(); - rawOut.flush(); - return new ByteArrayInputStream(rawOut.toByteArray()); - } catch (Exception e) { - return null; - } - } - return new ByteArrayInputStream(data); - } - - public void addHeaders(HeaderFields headers) { - headers.add("Content-Type", "text/plain;encoding=UTF-8"); - if (gzip) - headers.add("Content-Encoding", "gzip"); - } - - public com.yahoo.container.jdisc.HttpRequest toRequest() { - com.yahoo.container.jdisc.HttpRequest request = com.yahoo.container.jdisc.HttpRequest.createTestRequest(req, com.yahoo.jdisc.http.HttpRequest.Method.GET, getData()); - addHeaders(request.getJDiscRequest().headers()); - return request; - } - - } - - public static void assertRendered(String expected,Result result) throws Exception { - assertRendered(expected,result,true); - } - - public static void assertRendered(String expected,Result result,boolean checkFullEquality) throws Exception { - if (checkFullEquality) - assertEquals(expected, ResultRenderingUtil.getRendered(result)); - else - assertTrue(ResultRenderingUtil.getRendered(result).startsWith(expected)); - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java deleted file mode 100644 index 7f392b0565b..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.search.Result; - -import java.io.ByteArrayOutputStream; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.nio.charset.CharsetDecoder; - -@SuppressWarnings("deprecation") -public class ResultRenderingUtil { - - public static String getRendered(Result result) throws Exception { - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - Charset cs = Charset.forName("utf-8"); - CharsetDecoder decoder = cs.newDecoder(); - com.yahoo.prelude.templates.SearchRendererAdaptor.callRender(stream, result); - stream.flush(); - return decoder.decode(ByteBuffer.wrap(stream.toByteArray())).toString(); - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java deleted file mode 100644 index e49d2ca4db3..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.storage.searcher; - -import com.yahoo.component.chain.Chain; -import com.yahoo.cloud.config.ClusterListConfig; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.protect.Error; -import com.yahoo.documentapi.VisitorSession; -import com.yahoo.feedhandler.NullFeedMetric; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.document.DataType; -import com.yahoo.document.DocumentType; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.documentapi.VisitorParameters; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.feedapi.FeedContext; -import com.yahoo.feedapi.MessagePropertyProcessor; -import com.yahoo.messagebus.StaticThrottlePolicy; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.vdslib.VisitorOrdering; -import com.yahoo.vespaclient.ClusterList; -import com.yahoo.vespaclient.config.FeederConfig; - -import org.junit.Test; - -import java.util.Arrays; - -import static org.junit.Assert.*; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyLong; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -@SuppressWarnings("deprecation") -public class VisitorSearcherTestCase { - - private DocumentTypeManager docMan = null; - private DocumentType docType; - DocumentSessionFactory factory; - - @org.junit.Before - public void setUp() { - docMan = new DocumentTypeManager(); - docType = new DocumentType("kittens"); - docType.addHeaderField("name", DataType.STRING); - docType.addField("description", DataType.STRING); - docType.addField("image", DataType.RAW); - docType.addField("fluffiness", DataType.INT); - docType.addField("foo", DataType.RAW); - docMan.registerDocumentType(docType); - factory = new DocumentSessionFactory(docType); - } - - public VisitSearcher create() throws Exception { - ClusterListConfig.Storage.Builder storageCluster = new ClusterListConfig.Storage.Builder().configid("storage/cluster.foobar").name("foobar"); - ClusterListConfig clusterListCfg = new ClusterListConfig(new ClusterListConfig.Builder().storage(storageCluster)); - ClusterList clusterList = new ClusterList(clusterListCfg); - return new VisitSearcher(new FeedContext( - new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(458).route("riksveg18").retryenabled(true)), - new LoadTypeConfig(new LoadTypeConfig.Builder())), - factory, docMan, clusterList, new NullFeedMetric(true))); - } - - @Test - public void testQueryParameters() throws Exception { - VisitSearcher searcher = create(); - VisitorParameters params = searcher.getVisitorParameters( - newQuery("visit?visit.selection=id.user=1234&visit.cluster=foobar" + - "&visit.dataHandler=othercluster&visit.fieldSet=[header]&visit.fromTimestamp=112&visit.toTimestamp=224" + - "&visit.maxBucketsPerVisitor=2&visit.maxPendingMessagesPerVisitor=7&visit.maxPendingVisitors=14" + - "&visit.ordering=ASCENDING&priority=NORMAL_1&tracelevel=7&visit.visitInconsistentBuckets&visit.visitRemoves"), null); - - assertEquals("id.user=1234", params.getDocumentSelection()); - assertEquals(7, params.getMaxPending()); - assertEquals(2, params.getMaxBucketsPerVisitor()); - assertEquals(14, ((StaticThrottlePolicy)params.getThrottlePolicy()).getMaxPendingCount()); - assertEquals("[Storage:cluster=foobar;clusterconfigid=storage/cluster.foobar]", params.getRoute().toString()); - assertEquals("othercluster", params.getRemoteDataHandler()); - assertEquals("[header]", params.fieldSet()); - assertEquals(112, params.getFromTimestamp()); - assertEquals(224, params.getToTimestamp()); - assertEquals(VisitorOrdering.ASCENDING, params.getVisitorOrdering()); - assertEquals(DocumentProtocol.Priority.NORMAL_1, params.getPriority()); - assertEquals(7, params.getTraceLevel()); - assertEquals(true, params.visitInconsistentBuckets()); - assertEquals(true, params.visitRemoves()); - } - - @Test - public void timestampQueryParametersAreParsedAsLongs() throws Exception { - VisitorParameters params = create().getVisitorParameters( - newQuery("visit?visit.selection=id.user=1234&" + - "visit.fromTimestamp=1419021596000000&" + - "visit.toTimestamp=1419021597000000"), null); - assertEquals(1419021596000000L, params.getFromTimestamp()); - assertEquals(1419021597000000L, params.getToTimestamp()); - } - - @Test - public void testQueryParametersDefaults() throws Exception { - VisitSearcher searcher = create(); - VisitorParameters params = searcher.getVisitorParameters( - newQuery("visit?visit.selection=id.user=1234&hits=100"), null); - - assertEquals("id.user=1234", params.getDocumentSelection()); - assertEquals(1, params.getMaxBucketsPerVisitor()); - assertEquals(1, ((StaticThrottlePolicy)params.getThrottlePolicy()).getMaxPendingCount()); - assertEquals(1, params.getMaxFirstPassHits()); - assertEquals(1, params.getMaxTotalHits()); - assertEquals(32, params.getMaxPending()); - assertEquals(false, params.visitInconsistentBuckets()); - } - - @Test - public void testWrongCluster() throws Exception { - VisitSearcher searcher = create(); - - try { - searcher.getVisitorParameters( - newQuery("visit?visit.selection=id.user=1234&visit.cluster=unknown"), null); - - assertTrue(false); - } catch (Exception e) { - // e.printStackTrace(); - } - } - - - @Test(expected = IllegalArgumentException.class) - public void testNoClusterParamWhenSeveralClusters() throws Exception { - DocumentSessionFactory factory = new DocumentSessionFactory(docType); - ClusterListConfig.Storage.Builder storageCluster1 = new ClusterListConfig.Storage.Builder().configid("storage/cluster.foo").name("foo"); - ClusterListConfig.Storage.Builder storageCluster2 = new ClusterListConfig.Storage.Builder().configid("storage/cluster.bar").name("bar"); - ClusterListConfig clusterListCfg = new ClusterListConfig(new ClusterListConfig.Builder().storage(Arrays.asList(storageCluster1, storageCluster2))); - ClusterList clusterList = new ClusterList(clusterListCfg); - VisitSearcher searcher = new VisitSearcher(new FeedContext( - new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(100).route("whatever").retryenabled(true)), - new LoadTypeConfig(new LoadTypeConfig.Builder())), - factory, docMan, clusterList, new NullFeedMetric(true))); - - searcher.getVisitorParameters(newQuery("visit?visit.selection=id.user=1234"), null); - } - - @Test - public void testSimple() throws Exception { - Chain<Searcher> searchChain = new Chain<>(create()); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("visit?visit.selection=id.user=1234&hits=100")); - assertEquals(1, result.hits().size()); - assertRendered( - "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<document documenttype=\"kittens\" documentid=\"userdoc:foo:1234:bar\"/>\n" + - "</result>\n", result); - } - - private Result invokeVisitRemovesSearchChain() throws Exception { - Chain<Searcher> searchChain = new Chain<>(create()); - return new Execution(searchChain, Execution.Context.createContextStub()).search( - newQuery("visit?visit.selection=id.user=1234&hits=100&visit.visitRemoves=true")); - } - - @Test - public void visitRemovesIncludesRemoveEntriesInResultXml() throws Exception { - Result result = invokeVisitRemovesSearchChain(); - assertEquals(2, result.hits().size()); - assertRendered( - "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<document documenttype=\"kittens\" documentid=\"userdoc:foo:1234:bar\"/>\n" + - "<remove documentid=\"userdoc:foo:1234:removed\"/>\n" + - "</result>\n", result); - } - - @Test - public void removedDocumentIdsAreXmlEscaped() throws Exception { - factory = mock(DocumentSessionFactory.class); - when(factory.createVisitorSession(any(VisitorParameters.class))).thenAnswer((p) -> { - VisitorParameters params = (VisitorParameters)p.getArguments()[0]; - DummyVisitorSession session = new DummyVisitorSession(params, docType); - session.clearAutoReplyMessages(); - session.addRemoveReply("userdoc:foo:1234:<rem\"o\"ved&stuff>"); - return session; - }); - Result result = invokeVisitRemovesSearchChain(); - assertEquals(1, result.hits().size()); - assertRendered( - "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "<result>\n" + - "<remove documentid=\"userdoc:foo:1234:<rem"o"ved&stuff>\"/>\n" + - "</result>\n", result); - } - - private Result invokeSearcherWithUserQuery() throws Exception { - Chain<Searcher> searchChain = new Chain<>(create()); - return new Execution(searchChain, Execution.Context.createContextStub()) - .search(new Query("visit?visit.selection=id.user=1234&hits=100")); - } - - @Test - public void waitUntilDoneFailureReturnsTimeoutErrorHit() throws Exception { - VisitorSession session = mock(VisitorSession.class); - when(session.waitUntilDone(anyLong())).thenReturn(false); - factory = mock(DocumentSessionFactory.class); - when(factory.createVisitorSession(any(VisitorParameters.class))).thenReturn(session); - - Result result = invokeSearcherWithUserQuery(); - assertNotNull(result.hits().getErrorHit()); - assertEquals(Error.TIMEOUT.code, result.hits().getErrorHit().errors().iterator().next().getCode()); - } - - @Test - @SuppressWarnings("deprecation") - public void testRendererWiring() throws Exception { - Chain<Searcher> searchChain = new Chain<>(create()); - { - Query query = newQuery("visit?visit.selection=id.user=1234&hits=100&format=json"); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); - assertEquals(com.yahoo.prelude.templates.DefaultTemplateSet.class, result.getTemplating().getTemplates().getClass()); - } - { - Query query = newQuery("visit?visit.selection=id.user=1234&hits=100&format=JsonRenderer"); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); - assertEquals(com.yahoo.prelude.templates.DefaultTemplateSet.class, result.getTemplating().getTemplates().getClass()); - } - { - Query query = newQuery("visit?visit.selection=id.user=1234&hits=100"); - Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); - assertEquals(DocumentXMLTemplate.class, result.getTemplating().getTemplates().getClass()); - } - } - - public static void assertRendered(String expected, Result result) throws Exception { - assertEquals(expected, ResultRenderingUtil.getRendered(result)); - } - - private Query newQuery(String queryString) { - return new Query(HttpRequest.createTestRequest(queryString, com.yahoo.jdisc.http.HttpRequest.Method.GET)); - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java deleted file mode 100644 index 47745b29032..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.server; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.messagebus.SessionCache; -import com.yahoo.container.logging.AccessLog; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.jdisc.ReferencedResource; -import com.yahoo.jdisc.References; -import com.yahoo.jdisc.http.HttpRequest.Method; -import com.yahoo.messagebus.*; -import com.yahoo.messagebus.Error; -import com.yahoo.messagebus.shared.SharedMessageBus; -import com.yahoo.messagebus.shared.SharedSourceSession; -import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.text.Utf8; -import com.yahoo.text.Utf8String; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.OperationStatus; -import com.yahoo.vespaxmlparser.MockFeedReaderFactory; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import static org.junit.Assert.assertEquals; - -/** - * Check FeedHandler APIs. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class V2ErrorsInResultTestCase { - - LessConfiguredHandler handler; - ExecutorService workers; - - @Before - public void setUp() throws Exception { - workers = Executors.newCachedThreadPool(); - handler = new LessConfiguredHandler(workers); - } - - @After - public void tearDown() throws Exception { - handler.destroy(); - workers.shutdown(); - } - - private static class LessConfiguredHandler extends FeedHandler { - - public LessConfiguredHandler(Executor executor) throws Exception { - super(new FeedHandler.Context(executor, - AccessLog.voidAccessLog(), - new DummyMetric()), - null, null, null, MetricReceiver.nullImplementation); - } - - - @Override - protected Feeder createFeeder(HttpRequest request, InputStream requestInputStream, - BlockingQueue<OperationStatus> operations, String clientId, - boolean sessionIdWasGeneratedJustNow, int protocolVersion) - throws Exception { - return new LessConfiguredFeeder(requestInputStream, operations, - popClient(clientId), new FeederSettings(request), clientId, sessionIdWasGeneratedJustNow, - sourceSessionParams(request), null, this, this.feedReplyHandler, ""); - } - - @Override - protected DocumentTypeManager createDocumentManager( - DocumentmanagerConfig documentManagerConfig) { - return null; - } - } - - private static class MockSharedSession extends SharedSourceSession { - int count; - - public MockSharedSession(SourceSessionParams params) { - super(new SharedMessageBus(new MessageBus(new MockNetwork(), - new MessageBusParams())), params); - count = 0; - } - - @Override - public Result sendMessageBlocking(Message msg) throws InterruptedException { - return sendMessage(msg); - } - - @Override - public Result sendMessage(Message msg) { - Result r; - ReplyHandler handler = msg.popHandler(); - - switch (count++) { - case 0: - r = new Result(ErrorCode.FATAL_ERROR, - "boom"); - break; - case 1: - r = new Result(ErrorCode.TRANSIENT_ERROR, - "transient boom"); - break; - case 2: - final FailedReply reply = new FailedReply(msg.getContext()); - reply.addError(new Error( - ErrorCode.FATAL_ERROR, - "bad mojo, dude")); - handler.handleReply(reply); - r = Result.ACCEPTED; - break; - default: - handler.handleReply(new MockReply(msg.getContext())); - r = Result.ACCEPTED; - } - return r; - } - - } - - private static class FailedReply extends Reply { - Object context; - - public FailedReply(Object context) { - this.context = context; - } - - @Override - public Utf8String getProtocol() { - return null; - } - - @Override - public int getType() { - return 0; - } - - @Override - public Object getContext() { - return context; - } - } - - private static class LessConfiguredFeeder extends Feeder { - - public LessConfiguredFeeder(InputStream stream, - BlockingQueue<OperationStatus> operations, - ClientState storedState, FeederSettings settings, - String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams, - SessionCache sessionCache, FeedHandler handler, ReplyHandler feedReplyHandler, - String localHostname) throws Exception { - super(stream, new MockFeedReaderFactory(), null, operations, storedState, settings, clientId, sessionIdWasGeneratedJustNow, - sessionParams, sessionCache, handler, new DummyMetric(), feedReplyHandler, localHostname); - } - - protected ReferencedResource<SharedSourceSession> retainSession( - SourceSessionParams sessionParams, SessionCache sessionCache) { - final SharedSourceSession session = new MockSharedSession(sessionParams); - return new ReferencedResource<>(session, References.fromResource(session)); - } - } - - @Test - public final void test() throws IOException { - String sessionId; - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - HttpResponse r = handler.handle(nalle); - sessionId = r.headers().getFirst(Headers.SESSION_ID); - r.render(out); - assertEquals("", - Utf8.toString(out.toByteArray())); - } - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("id:banana:banana::doc1 ERROR boom \n", - Utf8.toString(out.toByteArray())); - } - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("id:banana:banana::doc1 TRANSIENT_ERROR transient{20}boom \n", - Utf8.toString(out.toByteArray())); - } - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("id:banana:banana::doc1 ERROR bad{20}mojo,{20}dude \n", - Utf8.toString(out.toByteArray())); - } - - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java deleted file mode 100644 index 9960d98f7f1..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java +++ /dev/null @@ -1,530 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.server; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.messagebus.SessionCache; -import com.yahoo.container.logging.AccessLog; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; -import com.yahoo.jdisc.ReferencedResource; -import com.yahoo.jdisc.References; -import com.yahoo.jdisc.http.HttpRequest.Method; -import com.yahoo.messagebus.Message; -import com.yahoo.messagebus.MessageBus; -import com.yahoo.messagebus.MessageBusParams; -import com.yahoo.messagebus.ReplyHandler; -import com.yahoo.messagebus.Result; -import com.yahoo.messagebus.SourceSessionParams; -import com.yahoo.messagebus.network.Network; -import com.yahoo.messagebus.shared.SharedMessageBus; -import com.yahoo.messagebus.shared.SharedSourceSession; -import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.text.Utf8; -import com.yahoo.vespa.http.client.config.FeedParams.DataFormat; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.OperationStatus; -import com.yahoo.vespaxmlparser.MockFeedReaderFactory; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.logging.Handler; -import java.util.logging.Level; -import java.util.logging.LogRecord; -import java.util.logging.Logger; - -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; - -/** - * Check FeedHandler APIs. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class V2ExternalFeedTestCase { - - LessConfiguredHandler handler; - ExecutorService workers; - Level logLevel; - Logger logger; - boolean initUseParentHandlers; - LogBuffer logChecker; - - @Before - public void setUp() throws Exception { - workers = Executors.newCachedThreadPool(); - handler = new LessConfiguredHandler(workers); - logger = Logger.getLogger(Feeder.class.getName()); - logLevel = logger.getLevel(); - logger.setLevel(Level.ALL); - initUseParentHandlers = logger.getUseParentHandlers(); - logChecker = new LogBuffer(); - logger.setUseParentHandlers(false); - logger.addHandler(logChecker); - } - - @After - public void tearDown() throws Exception { - handler.destroy(); - workers.shutdown(); - logger.setLevel(logLevel); - logger.removeHandler(logChecker); - logger.setUseParentHandlers(initUseParentHandlers); - } - - private static class LogBuffer extends Handler { - public final BlockingQueue<LogRecord> records = new LinkedBlockingQueue<>(); - - @Override - public void publish(LogRecord record) { - try { - records.put(record); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - - @Override - public void flush() { - } - - @Override - public void close() throws SecurityException { - } - } - - private static class LessConfiguredHandler extends FeedHandler { - volatile DataFormat lastFormatSeen; - - public LessConfiguredHandler(Executor executor) throws Exception { - super(new FeedHandler.Context(executor, - AccessLog.voidAccessLog(), - new DummyMetric()), - null, null, null, MetricReceiver.nullImplementation); - } - - @Override - protected Feeder createFeeder(HttpRequest request, - InputStream requestInputStream, - BlockingQueue<OperationStatus> operations, - String clientId, - boolean sessionIdWasGeneratedJustNow, int protocolVersion) - throws Exception { - LessConfiguredFeeder f = new LessConfiguredFeeder(requestInputStream, operations, - popClient(clientId), new FeederSettings(request), clientId, sessionIdWasGeneratedJustNow, - sourceSessionParams(request), null, this, this.feedReplyHandler, "ourHostname"); - lastFormatSeen = f.settings.dataFormat; - return f; - } - - @Override - protected DocumentTypeManager createDocumentManager( - DocumentmanagerConfig documentManagerConfig) { - return null; - } - } - - private static class MockSharedSession extends SharedSourceSession { - - public MockSharedSession(SourceSessionParams params) { - super(new SharedMessageBus(new MessageBus(new MockNetwork(), - new MessageBusParams())), params); - } - - @Override - public Result sendMessageBlocking(Message msg) throws InterruptedException { - return sendMessage(msg); - } - - @Override - public Result sendMessage(Message msg) { - ReplyHandler handler = msg.popHandler(); - MockReply mockReply = new MockReply(msg.getContext()); - if (msg instanceof Feeder.FeedErrorMessage) { - mockReply.addError(new com.yahoo.messagebus.Error(123, "Could not feed this")); - } - if (msg instanceof PutDocumentMessage) { - assert(msg.getTrace().getLevel() == 4); - assert(((PutDocumentMessage) msg).getPriority().name().equals("LOWEST")); - } - handler.handleReply(mockReply); - return Result.ACCEPTED; - } - - } - - private static class LessConfiguredFeeder extends Feeder { - public LessConfiguredFeeder(InputStream stream, - BlockingQueue<OperationStatus> operations, - ClientState storedState, FeederSettings settings, - String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams, - SessionCache sessionCache, FeedHandler handler, ReplyHandler feedReplyHandler, - String localHostname) throws Exception { - super(stream, new MockFeedReaderFactory(), null, operations, storedState, settings, clientId, sessionIdWasGeneratedJustNow, - sessionParams, sessionCache, handler, new DummyMetric(), feedReplyHandler, localHostname); - } - - protected ReferencedResource<SharedSourceSession> retainSession( - SourceSessionParams sessionParams, SessionCache sessionCache) { - final SharedSourceSession session = new MockSharedSession(sessionParams); - return new ReferencedResource<>(session, References.fromResource(session)); - } - } - - @Test - public final void test() throws IOException, InterruptedException { - String sessionId; - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - HttpResponse r = handler.handle(nalle); - sessionId = r.headers().getFirst(Headers.SESSION_ID); - r.render(out); - assertEquals("", - Utf8.toString(out.toByteArray())); - } - { - InputStream in = new MetaStream(new byte[]{1, 3, 2}); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); - nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n" - + "id:banana:banana::doc1 OK Document{20}processed. \n" - + "id:banana:banana::doc1 OK Document{20}processed. \n", - Utf8.toString(out.toByteArray())); - assertEquals("text/plain", r.getContentType()); - assertEquals(StandardCharsets.US_ASCII.name(), r.getCharacterEncoding()); - assertEquals(7, logChecker.records.size()); - String actualHandshake = logChecker.records.take().getMessage(); - assertThat(actualHandshake, actualHandshake.matches("Handshake completed for client (-?)(.+?)-#(.*?)\\."), is(true)); - assertEquals("Successfully deserialized document id: id:banana:banana::doc1", - logChecker.records.take().getMessage()); - assertEquals("Sent message successfully, document id: id:banana:banana::doc1", - logChecker.records.take().getMessage()); - } - - //test session ID without #, i.e. something fishy related to VIPs is going on - sessionId = "something"; - - { - InputStream in = new MetaStream(new byte[]{1, 3, 2}); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); - nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); - nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "2"); - - HttpResponse r = handler.handle(nalle); - r.render(out); - String expectedErrorMsg = "Got request from client with id 'something', but found no session for this client."; - assertThat(Utf8.toString(out.toByteArray()), containsString(expectedErrorMsg)); - assertEquals("text/plain", r.getContentType()); - assertEquals(StandardCharsets.UTF_8.name(), r.getCharacterEncoding()); - } - - //test session ID with trailing # but no hostname - sessionId = "something#"; - - { - InputStream in = new MetaStream(new byte[]{1, 3, 2}); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); - nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); - HttpResponse r = handler.handle(nalle); - r.render(out); - String expectedErrorMsg = "Got request from client with id 'something#', but found no session for this client."; - assertThat(Utf8.toString(out.toByteArray()), containsString(expectedErrorMsg)); - assertEquals("text/plain", r.getContentType()); - assertEquals(StandardCharsets.UTF_8.name(), r.getCharacterEncoding()); - } - - //test session ID with trailing # and some unknown hostname at the end - sessionId = "something#thisHostnameDoesNotExistAnywhere"; - - { - InputStream in = new MetaStream(new byte[]{1, 3, 2}); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); - nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); - HttpResponse r = handler.handle(nalle); - r.render(out); - String expectedErrorMsg = "Got request from client with id 'something#thisHostnameDoesNotExistAnywhere', " + - "but found no session for this client. Session was originally established " + - "towards host thisHostnameDoesNotExistAnywhere, but our hostname is " + - "ourHostname."; - assertThat(Utf8.toString(out.toByteArray()), containsString(expectedErrorMsg)); - assertEquals("text/plain", r.getContentType()); - assertEquals(StandardCharsets.UTF_8.name(), r.getCharacterEncoding()); - } - - //test session ID with trailing # and some unknown hostname at the end - sessionId = "something#ourHostname"; - - { - InputStream in = new MetaStream(new byte[]{1, 3, 2}); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); - nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n" + - "id:banana:banana::doc1 OK Document{20}processed. \n" + - "id:banana:banana::doc1 OK Document{20}processed. \n", - Utf8.toString(out.toByteArray())); - assertEquals("text/plain", r.getContentType()); - assertEquals(StandardCharsets.US_ASCII.name(), r.getCharacterEncoding()); - Thread.sleep(1000); - } - } - - @Test - public final void testFailedReading() throws IOException { - String sessionId; - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - HttpResponse r = handler.handle(nalle); - sessionId = r.headers().getFirst(Headers.SESSION_ID); - r.render(out); - assertEquals("", - Utf8.toString(out.toByteArray())); - } - { - InputStream in = new MetaStream(new byte[] { 4 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("id:banana:banana::doc1 ERROR Could{20}not{20}feed{20}this \n", - Utf8.toString(out.toByteArray())); - } - } - - @Test - public final void testCleaningDoesNotBlowUp() throws IOException { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("", - Utf8.toString(out.toByteArray())); - handler.forceRunCleanClients(); - } - - @Test - public final void testMockNetworkDoesNotBlowUp() { - Network n = new MockNetwork(); - n.registerSession(null); - n.unregisterSession(null); - assertTrue(n.allocServiceAddress(null)); - n.freeServiceAddress(null); - n.send(null, null); - assertNull(n.getConnectionSpec()); - assertNull(n.getMirror()); - } - - @Test - public final void testMockReplyDoesNotBlowUp() { - MockReply r = new MockReply(null); - assertNull(r.getProtocol()); - assertEquals(0, r.getType()); - assertFalse(r.hasFatalErrors()); - } - - @Test - public final void testFlush() throws IOException { - String sessionId; - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - HttpResponse r = handler.handle(nalle); - sessionId = r.headers().getFirst(Headers.SESSION_ID); - r.render(out); - assertEquals("", - Utf8.toString(out.toByteArray())); - } - { - InputStream in = new MetaStream(new byte[] { 1, 1, 1, 1, 1, 1, 1}); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); - nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n" - + "id:banana:banana::doc1 OK Document{20}processed. \n" - + "id:banana:banana::doc1 OK Document{20}processed. \n" - + "id:banana:banana::doc1 OK Document{20}processed. \n" - + "id:banana:banana::doc1 OK Document{20}processed. \n" - + "id:banana:banana::doc1 OK Document{20}processed. \n" - + "id:banana:banana::doc1 OK Document{20}processed. \n", - Utf8.toString(out.toByteArray())); - } - } - - @Test - public final void testIllegalVersion() throws IOException { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers() - .add(Headers.VERSION, Integer.toString(Integer.MAX_VALUE)); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals(Headers.HTTP_NOT_ACCEPTABLE, r.getStatus()); - } - - @Test - public final void testSettings() { - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - nalle.getJDiscRequest().headers().add(Headers.ROUTE, "bamse brakar"); - nalle.getJDiscRequest().headers().add(Headers.DENY_IF_BUSY, "true"); - FeederSettings settings = new FeederSettings(nalle); - assertEquals(false, settings.drain); - assertEquals(2, settings.route.getNumHops()); - assertEquals(true, settings.denyIfBusy); - } - - @Test - public final void testJsonInputFormat() throws IOException, InterruptedException { - String sessionId; - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - HttpResponse r = handler.handle(nalle); - sessionId = r.headers().getFirst(Headers.SESSION_ID); - r.render(out); - assertEquals("", - Utf8.toString(out.toByteArray())); - } - { - InputStream in = new MetaStream(new byte[]{1, 3, 2}); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.DATA_FORMAT, DataFormat.JSON_UTF8.name()); - nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); - nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n" - + "id:banana:banana::doc1 OK Document{20}processed. \n" - + "id:banana:banana::doc1 OK Document{20}processed. \n", - Utf8.toString(out.toByteArray())); - assertEquals("text/plain", r.getContentType()); - assertEquals(StandardCharsets.US_ASCII.name(), r.getCharacterEncoding()); - assertEquals(7, logChecker.records.size()); - String actualHandshake = logChecker.records.take().getMessage(); - assertThat(actualHandshake, actualHandshake.matches("Handshake completed for client (-?)(.+?)-#(.*?)\\."), is(true)); - assertEquals("Successfully deserialized document id: id:banana:banana::doc1", - logChecker.records.take().getMessage()); - assertEquals("Sent message successfully, document id: id:banana:banana::doc1", - logChecker.records.take().getMessage()); - assertSame(DataFormat.JSON_UTF8, handler.lastFormatSeen); - } - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java deleted file mode 100644 index 6290c22f694..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java +++ /dev/null @@ -1,226 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.server; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.messagebus.SessionCache; -import com.yahoo.container.logging.AccessLog; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.jdisc.ReferencedResource; -import com.yahoo.jdisc.References; -import com.yahoo.jdisc.http.HttpRequest.Method; -import com.yahoo.messagebus.*; -import com.yahoo.messagebus.shared.SharedMessageBus; -import com.yahoo.messagebus.shared.SharedSourceSession; -import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.text.Utf8; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.OperationStatus; -import com.yahoo.vespaxmlparser.MockFeedReaderFactory; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import static org.junit.Assert.assertEquals; - -/** - * Check FeedHandler APIs. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class V2FailingMessagebusTestCase { - - LessConfiguredHandler handler; - ExecutorService workers; - int mbus; - - @Before - public void setUp() throws Exception { - workers = Executors.newCachedThreadPool(); - handler = new LessConfiguredHandler(workers); - mbus = 0; - } - - @After - public void tearDown() throws Exception { - handler.destroy(); - workers.shutdown(); - mbus = 0; - } - - private class LessConfiguredHandler extends FeedHandler { - - public LessConfiguredHandler(Executor executor) throws Exception { - super(new FeedHandler.Context(executor, AccessLog.voidAccessLog(), new DummyMetric()), - null, null, null, MetricReceiver.nullImplementation); - } - - @Override - protected Feeder createFeeder(HttpRequest request, - InputStream requestInputStream, - BlockingQueue<OperationStatus> operations, - String clientId, - boolean sessionIdWasGeneratedJustNow, int protocolVersion) throws Exception { - return new LessConfiguredFeeder(requestInputStream, operations, - popClient(clientId), new FeederSettings(request), clientId, sessionIdWasGeneratedJustNow, - sourceSessionParams(request), null, this, this.feedReplyHandler, ""); - } - - @Override - protected DocumentTypeManager createDocumentManager( - DocumentmanagerConfig documentManagerConfig) { - return null; - } - } - - private class MockSharedSession extends SharedSourceSession { - - public MockSharedSession(SourceSessionParams params) { - super(new SharedMessageBus(new MessageBus(new MockNetwork(), - new MessageBusParams())), params); - } - - @Override - public Result sendMessageBlocking(Message msg) throws InterruptedException { - return sendMessage(msg); - } - - @Override - public Result sendMessage(Message msg) { - ReplyHandler handler = msg.popHandler(); - - switch (mbus) { - case 0: - throw new RuntimeException("boom"); - case 1: - Result r = new Result(ErrorCode.SEND_QUEUE_FULL, "tralala"); - mbus = 2; - return r; - case 2: - handler.handleReply(new MockReply(msg.getContext())); - return Result.ACCEPTED; - default: - throw new IllegalStateException("WTF?!"); - } - } - } - - private class LessConfiguredFeeder extends Feeder { - - public LessConfiguredFeeder(InputStream inputStream, - BlockingQueue<OperationStatus> operations, - ClientState storedState, FeederSettings settings, - String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams, - SessionCache sessionCache, FeedHandler handler, ReplyHandler feedReplyHandler, - String localHostname) throws Exception { - super(inputStream, new MockFeedReaderFactory(), null, operations, storedState, settings, clientId, sessionIdWasGeneratedJustNow, - sessionParams, sessionCache, handler, new DummyMetric(), feedReplyHandler, localHostname); - } - - protected ReferencedResource<SharedSourceSession> retainSession( - SourceSessionParams sessionParams, SessionCache sessionCache) { - final SharedSourceSession session = new MockSharedSession(sessionParams); - return new ReferencedResource<>(session, References.fromResource(session)); - } - } - - @Test - public final void testFailingMbus() throws IOException { - String sessionId; - { - InputStream in = new MetaStream(new byte[]{1}); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - HttpResponse r = handler.handle(nalle); - sessionId = r.headers().getFirst(Headers.SESSION_ID); - r.render(out); - assertEquals("", - Utf8.toString(out.toByteArray())); - } - { - InputStream in = new MetaStream(new byte[]{1}); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("id:banana:banana::doc1 ERROR boom \n", - Utf8.toString(out.toByteArray())); - } - } - - @Test - public final void testBusyMbus() throws IOException { - String sessionId; - { - InputStream in = new MetaStream(new byte[]{1}); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - mbus = 2; - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - HttpResponse r = handler.handle(nalle); - sessionId = r.headers().getFirst(Headers.SESSION_ID); - r.render(out); - assertEquals("", - Utf8.toString(out.toByteArray())); - } - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - mbus = 1; - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - nalle.getJDiscRequest().headers() - .add(Headers.DENY_IF_BUSY, "false"); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n", - Utf8.toString(out.toByteArray())); - } - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - mbus = 1; - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - nalle.getJDiscRequest().headers().add(Headers.DENY_IF_BUSY, "true"); - HttpResponse r = handler.handle(nalle); - r.render(out); - assertEquals("id:banana:banana::doc1 TRANSIENT_ERROR tralala \n", - Utf8.toString(out.toByteArray())); - } - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java deleted file mode 100644 index 633477dcc79..00000000000 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.server; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.messagebus.SessionCache; -import com.yahoo.container.logging.AccessLog; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.jdisc.ReferencedResource; -import com.yahoo.jdisc.References; -import com.yahoo.jdisc.http.HttpRequest.Method; -import com.yahoo.messagebus.*; -import com.yahoo.messagebus.Error; -import com.yahoo.messagebus.shared.SharedMessageBus; -import com.yahoo.messagebus.shared.SharedSourceSession; -import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.text.Utf8; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.OperationStatus; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import static org.junit.Assert.assertEquals; - -/** - * Check FeedHandler APIs. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class V2NoXmlReaderTestCase { - - LessConfiguredHandler handler; - ExecutorService workers; - - @Before - public void setUp() throws Exception { - workers = Executors.newCachedThreadPool(); - handler = new LessConfiguredHandler(workers); - } - - @After - public void tearDown() throws Exception { - handler.destroy(); - workers.shutdown(); - } - - private static class LessConfiguredHandler extends FeedHandler { - - public LessConfiguredHandler(Executor executor) throws Exception { - super(new FeedHandler.Context(executor, AccessLog.voidAccessLog(), new DummyMetric()), - null, null, null, MetricReceiver.nullImplementation); - } - - - @Override - protected Feeder createFeeder(HttpRequest request, InputStream requestInputStream, - BlockingQueue<OperationStatus> operations, String clientId, - boolean sessionIdWasGeneratedJustNow, int protocolVersion) - throws Exception { - return new LessConfiguredFeeder(requestInputStream, operations, - popClient(clientId), new FeederSettings(request), clientId, sessionIdWasGeneratedJustNow, - sourceSessionParams(request), null, this, this.feedReplyHandler, ""); - } - - @Override - protected DocumentTypeManager createDocumentManager( - DocumentmanagerConfig documentManagerConfig) { - return null; - } - } - - private static class MockSharedSession extends SharedSourceSession { - - public MockSharedSession(SourceSessionParams params) { - super(new SharedMessageBus(new MessageBus(new MockNetwork(), - new MessageBusParams())), params); - } - - @Override - public Result sendMessageBlocking(Message msg) throws InterruptedException { - return sendMessage(msg); - } - - @Override - public Result sendMessage(Message msg) { - ReplyHandler handler = msg.popHandler(); - MockReply mockReply = new MockReply(msg.getContext()); - if (msg instanceof Feeder.FeedErrorMessage) { - mockReply.addError(new Error(123, "Could not feed this")); - } - handler.handleReply(mockReply); - return Result.ACCEPTED; - } - - } - - private static class LessConfiguredFeeder extends Feeder { - - public LessConfiguredFeeder(InputStream inputStream, - BlockingQueue<OperationStatus> operations, - ClientState storedState, FeederSettings settings, - String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams, - SessionCache sessionCache, FeedHandler handler, ReplyHandler feedReplyHandler, - String localHostname) throws Exception { - super(inputStream, null, null, operations, storedState, settings, clientId, sessionIdWasGeneratedJustNow, - sessionParams, sessionCache, handler, new DummyMetric(), feedReplyHandler, localHostname); - } - - protected ReferencedResource<SharedSourceSession> retainSession( - SourceSessionParams sessionParams, SessionCache sessionCache) { - final SharedSourceSession session = new MockSharedSession(sessionParams); - return new ReferencedResource<>(session, References.fromResource(session)); - } - } - - @Test - public final void test() throws IOException { - String sessionId; - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest - .createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); - HttpResponse r = handler.handle(nalle); - sessionId = r.headers().getFirst(Headers.SESSION_ID); - r.render(out); - assertEquals("", - Utf8.toString(out.toByteArray())); - } - { - InputStream in = new MetaStream(new byte[] { 1 }); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - HttpRequest nalle = HttpRequest.createTestRequest( - "http://test4-steinar:19020/reserved-for-internal-use/feedapi", - Method.POST, in); - nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); - nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); - nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); - HttpResponse r = handler.handle(nalle); - r.render(out); - //This is different from v1. If we cannot parse XML, we will still get response code 200, but with a sensible - //error message in the response. - assertEquals(200, r.getStatus()); - assertEquals("id:banana:banana::doc1 ERROR Could{20}not{20}feed{20}this \n", - Utf8.toString(out.toByteArray())); - } - } - -} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java index d6f605b0379..128664dda9e 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java @@ -23,13 +23,14 @@ public class VersionsTestCase { private static final List<String> EMPTY = Collections.emptyList(); private static final List<String> ONE_TWO = Arrays.asList("1", "2"); + private static final List<String> ONE_THREE = Arrays.asList("1", "3"); private static final List<String> TWO_THREE = Arrays.asList("3", "2"); - private static final List<String> ONE_NULL_TWO = Arrays.asList("1", null, "2"); - private static final List<String> ONE_COMMA_TWO = Collections.singletonList("1, 2"); - private static final List<String> ONE_EMPTY_TWO = Arrays.asList("1", "", "2"); + private static final List<String> ONE_NULL_THREE = Arrays.asList("1", null, "3"); + private static final List<String> ONE_COMMA_THREE = Collections.singletonList("1, 3"); + private static final List<String> ONE_EMPTY_THREE = Arrays.asList("1", "", "3"); private static final List<String> TOO_LARGE_NUMBER = Collections.singletonList("1000000000"); - private static final List<String> TWO_TOO_LARGE_NUMBER = Arrays.asList("2", "1000000000"); - private static final List<String> TWO_COMMA_TOO_LARGE_NUMBER = Arrays.asList("2,1000000000"); + private static final List<String> THREE_TOO_LARGE_NUMBER = Arrays.asList("3", "1000000000"); + private static final List<String> THREE_COMMA_TOO_LARGE_NUMBER = Arrays.asList("3,1000000000"); private static final List<String> GARBAGE = Collections.singletonList("garbage"); @Test @@ -42,8 +43,15 @@ public class VersionsTestCase { @Test public void testOneTwo() throws Exception { Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_TWO); + assertThat(v.first, instanceOf(ErrorHttpResponse.class)); + assertThat(v.second, is(-1)); + } + + @Test + public void testOneThree() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_THREE); assertThat(v.first, nullValue()); - assertThat(v.second, is(2)); + assertThat(v.second, is(3)); } @Test @@ -54,24 +62,24 @@ public class VersionsTestCase { } @Test - public void testOneNullTwo() throws Exception { - Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_NULL_TWO); + public void testOneNullThree() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_NULL_THREE); assertThat(v.first, nullValue()); - assertThat(v.second, is(2)); + assertThat(v.second, is(3)); } @Test - public void testOneCommaTwo() throws Exception { - Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_COMMA_TWO); + public void testOneCommaThree() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_COMMA_THREE); assertThat(v.first, nullValue()); - assertThat(v.second, is(2)); + assertThat(v.second, is(3)); } @Test - public void testOneEmptyTwo() throws Exception { - Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_EMPTY_TWO); + public void testOneEmptyThree() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_EMPTY_THREE); assertThat(v.first, nullValue()); - assertThat(v.second, is(2)); + assertThat(v.second, is(3)); } @Test @@ -83,22 +91,22 @@ public class VersionsTestCase { errorResponse.render(errorMsg); assertThat(errorMsg.toString(), is("Could not parse X-Yahoo-Feed-Protocol-Versionheader of request (values: [1000000000]). " + - "Server supports protocol versions [2, 3]")); + "Server supports protocol versions [3]")); assertThat(v.second, is(-1)); } @Test - public void testTwoTooLarge() throws Exception { - Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(TWO_TOO_LARGE_NUMBER); + public void testThreeTooLarge() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(THREE_TOO_LARGE_NUMBER); assertThat(v.first, nullValue()); - assertThat(v.second, is(2)); + assertThat(v.second, is(3)); } @Test public void testTwoCommaTooLarge() throws Exception { - Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(TWO_COMMA_TOO_LARGE_NUMBER); + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(THREE_COMMA_TOO_LARGE_NUMBER); assertThat(v.first, nullValue()); - assertThat(v.second, is(2)); + assertThat(v.second, is(3)); } @Test diff --git a/vespaclient-core/CMakeLists.txt b/vespaclient-core/CMakeLists.txt index facea0b447d..218276cf590 100644 --- a/vespaclient-core/CMakeLists.txt +++ b/vespaclient-core/CMakeLists.txt @@ -1,3 +1,2 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. install_config_definition(src/main/resources/configdefinitions/feeder.def vespaclient.config.feeder.def) -install_config_definition(src/main/resources/configdefinitions/spooler.def vespa.config.content.spooler.spooler.def) diff --git a/vespaclient-core/pom.xml b/vespaclient-core/pom.xml index 2e427364bd6..693af71e0bc 100644 --- a/vespaclient-core/pom.xml +++ b/vespaclient-core/pom.xml @@ -1,17 +1,16 @@ <?xml version="1.0"?> <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<!-- TODO: Remove this module on Vespa 7 --> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> <modelVersion>4.0.0</modelVersion> <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespaclient-core</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>junit</groupId> diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/package-info.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/package-info.java index 2a13901b795..14b8e4e1ea7 100644 --- a/vespaclient-core/src/main/java/com/yahoo/feedhandler/package-info.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/package-info.java @@ -1,5 +1,4 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// TODO: This implements the old, deprecated document HTTP= API. Remove on Vespa 7 /** * Exported not as an API, but specifically for SSBE. */ diff --git a/vespaclient-core/src/main/resources/configdefinitions/spooler.def b/vespaclient-core/src/main/resources/configdefinitions/spooler.def deleted file mode 100644 index a03010b0f32..00000000000 --- a/vespaclient-core/src/main/resources/configdefinitions/spooler.def +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -namespace=vespa.config.content.spooler - -# The root directory under which the spooler will scan for content files -directory string default="var/spool/vespa" - -# If true, move successfully processed files to <directory>/success -keepsuccess bool default=false - -# How many bytes to store maximum in the failures directory (delete files instead of moving to failures if this is exceeded) -# If 0, don't restrict -maxfailuresize int default=0 -maxfatalfailuresize int default=0 - -# How many bytes to store maximum in the successes directory (delete files instead of moving to successes if this is exceeded) -# If 0, don't restrict -maxsuccesssize int default=0 - -# How many worker threads to use -threads int default=10 - -# Which parsers to use and config for each of them. -parsers[].classname string -parsers[].parameters[].key string -parsers[].parameters[].value string - -# The time spent in failures (seconds) until moving to fatalfailures (default 3 days) -maxfailuretime int default=259200 - -# The minimum number of retries before moving to fatalfailures -minfailureretries int default=5 - -# The maximum number of retries when getting transient errors sending messages to backend -maxretries int default=5 diff --git a/vespaclient-java/pom.xml b/vespaclient-java/pom.xml index a7ac908279b..93f1579f58c 100644 --- a/vespaclient-java/pom.xml +++ b/vespaclient-java/pom.xml @@ -1,17 +1,16 @@ <?xml version="1.0"?> <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<!-- TODO: Remove this module on Vespa 7 --> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> <modelVersion>4.0.0</modelVersion> <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespaclient-java</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>junit</groupId> diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java index 20d25b63d30..703cdeba247 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java @@ -68,7 +68,7 @@ public class CommandLineOptions { options.addOption(Option.builder("e") .hasArg(false) - .desc("Retrieve header fields only. [Deprecated].") + .desc("Retrieve header fields only. [Removed in Vespa 7].") .longOpt(HEADERSONLY_OPTION).build()); options.addOption(Option.builder("f") @@ -133,12 +133,12 @@ public class CommandLineOptions { options.addOption(Option.builder("j") .hasArg(false) - .desc("JSON output") + .desc("JSON output (default format)") .longOpt(JSONOUTPUT_OPTION).build()); options.addOption(Option.builder("x") .hasArg(false) - .desc("XML output (default format)") + .desc("XML output") .longOpt(XMLOUTPUT_OPTION).build()); return options; @@ -180,17 +180,15 @@ public class CommandLineOptions { throw new IllegalArgumentException("Cannot combine both xml and json output"); } - if (printIdsOnly && headersOnly) { - throw new IllegalArgumentException("Print ids and headers only options are mutually exclusive."); + if (headersOnly) { + throw new IllegalArgumentException("Headers only option has been removed."); } - if ((printIdsOnly || headersOnly) && !fieldSet.isEmpty()) { - throw new IllegalArgumentException("Field set option can not be used in combination with print ids or headers only options."); + if (printIdsOnly && !fieldSet.isEmpty()) { + throw new IllegalArgumentException("Field set option can not be used in combination with print ids option."); } if (printIdsOnly) { fieldSet = "[id]"; - } else if (headersOnly) { - fieldSet = "[header]"; } else if (fieldSet.isEmpty()) { fieldSet = "[all]"; } @@ -226,7 +224,7 @@ public class CommandLineOptions { .setTraceLevel(trace) .setPriority(priority) .setTimeout(timeout) - .setJsonOutput((!jsonOutput && !xmlOutput) ? false : jsonOutput) // TODO Vespa 7 Change default to JSON + .setJsonOutput(!xmlOutput) .build(); } catch (ParseException pe) { throw new IllegalArgumentException(pe.getMessage()); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java index a5c8fadc3a1..4430af19fbc 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java @@ -189,7 +189,7 @@ public class VdsVisit { .desc("Only visit up to the given timestamp (microseconds).") .type(Number.class).build()); - options.addOption("e", "headersonly", false, "Only visit headers of documents.[Deprecated]"); + options.addOption("e", "headersonly", false, "Only visit headers of documents.[Removed in Vespa 7]"); options.addOption(Option.builder("l") .longOpt("fieldset") @@ -345,17 +345,16 @@ public class VdsVisit { options.addOption(Option.builder() .longOpt("jsonoutput") - .desc("Output documents as JSON") + .desc("Output documents as JSON (default format)") .hasArg(false) .build()); options.addOption(Option.builder("x") .longOpt("xmloutput") - .desc("Output documents as XML (default format)") + .desc("Output documents as XML") .hasArg(false) .build()); - options.addOption(Option.builder() .longOpt("bucketspace") .hasArg(true) @@ -481,7 +480,7 @@ public class VdsVisit { params.setToTimestamp(((Number) line.getParsedOptionValue("t")).longValue()); } if (line.hasOption("e")) { - params.fieldSet("[header]"); + throw new IllegalArgumentException("Headers only option has been removed."); } if (line.hasOption("l")) { params.fieldSet(line.getOptionValue("l")); @@ -586,7 +585,7 @@ public class VdsVisit { if (jsonOutput && xmlOutput) { throw new IllegalArgumentException("Cannot combine both xml and json output"); } - allParams.setJsonOutput((!jsonOutput && !xmlOutput) ? false : jsonOutput); // TODO Vespa 7 Change default to JSON + allParams.setJsonOutput(!xmlOutput); allParams.setVisitorParameters(params); return allParams; diff --git a/vespaclient-java/src/main/sh/vespa-visit.1 b/vespaclient-java/src/main/sh/vespa-visit.1 index fb2f102234d..5ed40b1a410 100644 --- a/vespaclient-java/src/main/sh/vespa-visit.1 +++ b/vespaclient-java/src/main/sh/vespa-visit.1 @@ -35,12 +35,6 @@ visited. The time is given in microseconds since 1970. If this option is given, only documents up to and including the given timestamp will be visited. The time is given in microseconds since 1970. .TP -\fB\-e\fR, \fB\-\-headersonly\fR -By default, the whole documents stored are processed. If this option is given -only the header parts of documents will be processed. By defining the big -document fields as body fields, you can efficiently visit all the header fields -using this option. -.TP \fB\-i\fR, \fB\-\-printids\fR Using this option, only the document identifiers will be printed to STDOUT. In addition, if visiting removes, an additional tag will be added so you can diff --git a/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java b/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java index 05f8964e516..8848b6f1e7f 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespaget/CommandLineOptionsTest.java @@ -89,23 +89,9 @@ public class CommandLineOptionsTest { } @Test - public void testInvalidCombination1() { - exception.expect(IllegalArgumentException.class); - exception.expectMessage("Print ids and headers only options are mutually exclusive."); - getParsedOptions("--headersonly", "--printids"); - } - - @Test - public void testInvalidCombination2() { - exception.expect(IllegalArgumentException.class); - exception.expectMessage("Field set option can not be used in combination with print ids or headers only options."); - getParsedOptions("--headersonly", "--fieldset", "[header]"); - } - - @Test public void testInvalidCombination3() { exception.expect(IllegalArgumentException.class); - exception.expectMessage("Field set option can not be used in combination with print ids or headers only options."); + exception.expectMessage("Field set option can not be used in combination with print ids option."); getParsedOptions("--printids", "--fieldset", "[header]"); } @@ -157,8 +143,9 @@ public class CommandLineOptionsTest { @Test public void testHeadersOnly() { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Headers only option has been removed."); ClientParameters params = getParsedOptions("--headersonly"); - assertEquals("[header]", params.fieldSet); } @Test diff --git a/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java b/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java index 32e29ba51ec..570f38045e6 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java @@ -74,7 +74,7 @@ public class VdsVisitTestCase { // Short options testing (for options that do not collide with each other) String[] args = new String[] { "-o", "654321", - "-e" + "-i" }; VdsVisit.ArgumentParser parser = createMockArgumentParser(); VdsVisit.VdsVisitParameters allParams = parser.parse(args); @@ -84,7 +84,7 @@ public class VdsVisitTestCase { assertNotNull(params); assertEquals(654321, allParams.getFullTimeout()); assertEquals(654321, params.getTimeoutMs()); - assertEquals("[header]", params.getFieldSet()); + assertEquals("[id]", params.getFieldSet()); } @Test diff --git a/vespaclient/CMakeLists.txt b/vespaclient/CMakeLists.txt index ed61d730629..8be8751f2c9 100644 --- a/vespaclient/CMakeLists.txt +++ b/vespaclient/CMakeLists.txt @@ -15,7 +15,6 @@ vespa_define_module( src/vespa/vespaclient/clusterlist APPS - src/vespa/vespaclient/spoolmaster src/vespa/vespaclient/vdsstates src/vespa/vespaclient/vesparoute ) diff --git a/vespaclient/src/vespa/vespaclient/spoolmaster/.gitignore b/vespaclient/src/vespa/vespaclient/spoolmaster/.gitignore deleted file mode 100644 index 81b1279cdb1..00000000000 --- a/vespaclient/src/vespa/vespaclient/spoolmaster/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -.depend -Makefile -vespa-spoolmaster diff --git a/vespaclient/src/vespa/vespaclient/spoolmaster/CMakeLists.txt b/vespaclient/src/vespa/vespaclient/spoolmaster/CMakeLists.txt deleted file mode 100644 index 5e5d8b17ff3..00000000000 --- a/vespaclient/src/vespa/vespaclient/spoolmaster/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(vespaclient_spoolmaster_app - SOURCES - main.cpp - application.cpp - OUTPUT_NAME vespa-spoolmaster - INSTALL bin - DEPENDS -) diff --git a/vespaclient/src/vespa/vespaclient/spoolmaster/application.cpp b/vespaclient/src/vespa/vespaclient/spoolmaster/application.cpp deleted file mode 100644 index 7406e17430d..00000000000 --- a/vespaclient/src/vespa/vespaclient/spoolmaster/application.cpp +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/defaults.h> -#include <thread> -#include <iostream> -#include <algorithm> -#include <dirent.h> -#include <unistd.h> -#include <cstring> -#include <sys/stat.h> - -#include "application.h" - -namespace { - -std::string masterInbox() { - return vespa::Defaults::underVespaHome("var/spool/master/inbox"); -} - -std::string outboxParent() { - return vespa::Defaults::underVespaHome("var/spool/vespa"); -} - -} - -namespace spoolmaster { - -Application::Application() - : _masterInbox(masterInbox()), - _inboxFiles(), - _outboxParentDir(outboxParent()), - _outboxes() -{ - // empty -} - -Application::~Application() -{ - // empty -} - -bool -Application::scanInbox() -{ - std::vector<std::string> rv; - DIR *d = opendir(_masterInbox.c_str()); - if (d == NULL) { - perror(_masterInbox.c_str()); - mkdir(_masterInbox.c_str(), 0775); - return false; - } - - struct dirent *entry; - while ((entry = readdir(d)) != NULL) { - if (strcmp(entry->d_name, ".") == 0) continue; - if (strcmp(entry->d_name, "..") == 0) continue; - - std::string fn = _masterInbox; - fn.append("/"); - fn.append(entry->d_name); - - struct stat sb; - if (stat(fn.c_str(), &sb) == 0) { - if (S_ISREG(sb.st_mode)) { - rv.push_back(fn); - } - } else { - perror(fn.c_str()); - } - } - closedir(d); - - if (access(_masterInbox.c_str(), W_OK) < 0) { - perror(_masterInbox.c_str()); - return false; - } - - _inboxFiles = rv; - return (rv.size() > 0); -} - -bool -Application::findOutboxes() -{ - std::vector<std::string> rv; - DIR *d = opendir(_outboxParentDir.c_str()); - if (d == NULL) { - perror(_outboxParentDir.c_str()); - return false; - } - struct dirent *entry; - while ((entry = readdir(d)) != NULL) { - if (strcmp(entry->d_name, ".") == 0) continue; - if (strcmp(entry->d_name, "..") == 0) continue; - - /* XXX: should check if d_name starts with "colo." ? */ - - std::string fn = _outboxParentDir; - fn.append("/"); - fn.append(entry->d_name); - fn.append("/inbox"); - - if (fn == _masterInbox) continue; - - struct stat sb; - if (stat(fn.c_str(), &sb) == 0) { - if (S_ISDIR(sb.st_mode)) { - if (access(fn.c_str(), W_OK) < 0) { - std::cerr << "Cannot write to directory "; - perror(fn.c_str()); - continue; - } - rv.push_back(fn); - } - } else { - perror(fn.c_str()); - } - } - closedir(d); - if (rv.size() > 0) { - std::sort(rv.begin(), rv.end()); - sviter_t ni = rv.begin(); - sviter_t oi = _outboxes.begin(); - - while (ni != rv.end()) { - const std::string &newval = *ni; - if (oi == _outboxes.end()) { - std::cerr << "Found new slave inbox: " << newval << std::endl; - ++ni; - continue; - } - const std::string &oldval = *oi; - if (newval == oldval) { - ++ni; - ++oi; - } else if (newval < oldval) { - std::cerr << "Found new slave inbox: " << newval << std::endl; - ++ni; - } else /* oldval < newval */ { - std::cerr << "Slave inbox removed: " << oldval << std::endl; - ++oi; - } - } - _outboxes = rv; - return true; - } - std::cerr << "Did not find any slave inboxes in: " << _outboxParentDir << std::endl; - return false; -} - -void -Application::moveLinks() -{ - for (sviter_t fni = _inboxFiles.begin(); fni != _inboxFiles.end(); ++fni) { - const std::string& filename = *fni; - size_t ldp = filename.rfind("/"); - std::string basename = filename.substr(ldp+1); - for (sviter_t obi = _outboxes.begin(); obi != _outboxes.end(); ++obi) { - std::string newFn = *obi; - newFn.append("/"); - newFn.append(basename); - - std::cout << "linking " << filename << " -> " << newFn << std::endl; - if (link(filename.c_str(), newFn.c_str()) < 0) { - std::cerr << "linking " << filename << " -> " << newFn; - perror("failed"); - return; - } - } - if (unlink(filename.c_str()) < 0) { - std::cerr << "cannot remove " << filename; - perror(", error"); - } - } -} - - -int -Application::Main() -{ - bool aborted = false; - findOutboxes(); - - try { - while (!aborted) { - if (scanInbox() && findOutboxes()) { - moveLinks(); - } else { - std::this_thread::sleep_for(std::chrono::milliseconds(200)); - } - } - } - catch(std::exception &e) { - fprintf(stderr, "ERROR: %s\n", e.what()); - return EXIT_FAILURE; - } - return EXIT_SUCCESS; -} - -} // namespace spoolmaster diff --git a/vespaclient/src/vespa/vespaclient/spoolmaster/application.h b/vespaclient/src/vespa/vespaclient/spoolmaster/application.h deleted file mode 100644 index 0cd8e1a917a..00000000000 --- a/vespaclient/src/vespa/vespaclient/spoolmaster/application.h +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/fastos/app.h> -#include <string> -#include <vector> - -namespace spoolmaster { -/** - * main spoolmaster application class - */ -class Application : public FastOS_Application { -private: - std::string _masterInbox; - std::vector<std::string> _inboxFiles; - - std::string _outboxParentDir; - std::vector<std::string> _outboxes; - - typedef std::vector<std::string>::iterator sviter_t; - - bool scanInbox(); - bool findOutboxes(); - void moveLinks(); -public: - Application(); - ~Application(); - int Main() override; -}; - -} diff --git a/vespaclient/src/vespa/vespaclient/spoolmaster/main.cpp b/vespaclient/src/vespa/vespaclient/spoolmaster/main.cpp deleted file mode 100644 index 40a262734bf..00000000000 --- a/vespaclient/src/vespa/vespaclient/spoolmaster/main.cpp +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "application.h" - -int -main(int argc, char** argv) -{ - spoolmaster::Application *app = new spoolmaster::Application(); - int retVal = app->Entry(argc, argv); - delete app; - return retVal; -} - diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 48f2a4af4ad..70383e8aabf 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -324,6 +324,7 @@ "public java.lang.String getName()", "public java.io.Reader getReader()", "public java.lang.String toString()", + "public static java.io.Reader nullReader()", "public int read(java.nio.CharBuffer)", "public int read()", "public int read(char[])", @@ -334,6 +335,7 @@ "public void mark(int)", "public void reset()", "public void close()", + "public long transferTo(java.io.Writer)", "public static void closeAll(java.util.List)" ], "fields": [] @@ -2458,35 +2460,6 @@ ], "fields": [] }, - "com.yahoo.text.DoubleFormatter": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>()", - "public static java.lang.StringBuilder fmt(java.lang.StringBuilder, double)", - "public static java.lang.String stringValue(double)", - "public static void append(java.lang.StringBuilder, double)", - "public static void append(java.lang.StringBuilder, int)" - ], - "fields": [] - }, - "com.yahoo.text.DoubleParser": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "final" - ], - "methods": [ - "public void <init>()", - "public static double parse(java.lang.String)" - ], - "fields": [] - }, "com.yahoo.text.ForwardWriter": { "superClass": "com.yahoo.text.GenericWriter", "interfaces": [], @@ -2612,19 +2585,6 @@ ], "fields": [] }, - "com.yahoo.text.LanguageHacks": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public static boolean isCJK(java.lang.String)", - "public static boolean yellDesegments(java.lang.String)" - ], - "fields": [] - }, "com.yahoo.text.Lowercase": { "superClass": "java.lang.Object", "interfaces": [], diff --git a/vespajlib/pom.xml b/vespajlib/pom.xml index 48f70abba15..32040785fc2 100644 --- a/vespajlib/pom.xml +++ b/vespajlib/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespajlib</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <description> Library for use in Java components of Vespa. Shared code which do not fit anywhere else. @@ -91,6 +91,7 @@ <artifactId>bundle-plugin</artifactId> <extensions>true</extensions> </plugin> + <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-deploy-plugin</artifactId> diff --git a/vespajlib/src/main/java/com/yahoo/io/reader/NamedReader.java b/vespajlib/src/main/java/com/yahoo/io/reader/NamedReader.java index 0a437e93f41..2cfc45c784c 100644 --- a/vespajlib/src/main/java/com/yahoo/io/reader/NamedReader.java +++ b/vespajlib/src/main/java/com/yahoo/io/reader/NamedReader.java @@ -36,6 +36,7 @@ public class NamedReader extends Reader { } // The rest is reader method implementations which delegates to the wrapped reader + public static Reader nullReader() { return new NamedReader("nullReader", Reader.nullReader()); } @Override public int read(CharBuffer charBuffer) throws IOException { return reader.read(charBuffer); } @Override @@ -56,10 +57,8 @@ public class NamedReader extends Reader { public void reset() throws IOException { reader.reset(); } @Override public void close() throws IOException { reader.close(); } - - // TODO Java 10: uncomment -// @Override -// public long transferTo(Writer out) throws IOException { return reader.transferTo(out); } + @Override + public long transferTo(Writer out) throws IOException { return reader.transferTo(out); } /** Convenience method for closing a list of readers. Does nothing if the given reader list is null. */ public static void closeAll(List<NamedReader> readers) { diff --git a/vespajlib/src/main/java/com/yahoo/text/DoubleFormatter.java b/vespajlib/src/main/java/com/yahoo/text/DoubleFormatter.java deleted file mode 100644 index 7281f3cd9d6..00000000000 --- a/vespajlib/src/main/java/com/yahoo/text/DoubleFormatter.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.text; - -/** - * Utility class to format a double into a String. - * <p> - * This was intended as a lower-cost replacement for the standard - * String.valueOf(double) since that method used to cause contention. - * <p> - * The contention issue is fixed in Java 8u96 so this class is now obsolete. - * - * @author arnej27959 - */ - -@Deprecated -public final class DoubleFormatter { - - public static StringBuilder fmt(StringBuilder target, double v) { - target.append(v); - return target; - } - - public static String stringValue(double v) { - return String.valueOf(v); - } - - public static void append(StringBuilder s, double d) { - s.append(d); - } - - public static void append(StringBuilder s, int i) { - s.append(i); - } - -} diff --git a/vespajlib/src/main/java/com/yahoo/text/DoubleParser.java b/vespajlib/src/main/java/com/yahoo/text/DoubleParser.java deleted file mode 100644 index 0a777f3000f..00000000000 --- a/vespajlib/src/main/java/com/yahoo/text/DoubleParser.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.text; - -/** - * Utility class to parse a String into a double. - * <p> - * This was intended as a lower-cost replacement for the standard - * Double.parseDouble(String) since that method used to cause lock - * contention. - * <p> - * The contention issue is fixed in Java 8u96 so this class is now obsolete. - * - * @author arnej27959 - */ -@Deprecated -public final class DoubleParser { - - /** - * Utility method that parses a String and returns a double. - * - * @param data the String to parse - * @return double parsed value of the string - * @throws NumberFormatException if the string is not a well-formatted number - * @throws NullPointerException if the string is a null pointer - */ - public static double parse(String data) { - return Double.parseDouble(data); - } -} diff --git a/vespajlib/src/main/java/com/yahoo/text/LanguageHacks.java b/vespajlib/src/main/java/com/yahoo/text/LanguageHacks.java deleted file mode 100644 index 21ebc967481..00000000000 --- a/vespajlib/src/main/java/com/yahoo/text/LanguageHacks.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.text; - -import static com.yahoo.text.Lowercase.toLowerCase; - -/** - * Language helper functions. - * - * @deprecated do not use - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -@Deprecated -public class LanguageHacks { - - /** - * Whether a language is in the CJK group. - */ - public static boolean isCJK(String language) { - if (language == null) return false; - - language = toLowerCase(language); - return "ja".equals(language) - || "ko".equals(language) - || language.startsWith("zh") - || language.startsWith("tw"); // TODO: tw is a bogus value? - } - - /** - * Whether there is desegmenting in this language. - */ - public static boolean yellDesegments(String language) { - if (language == null) return false; - - language = toLowerCase(language); - return "de".equals(language) || isCJK(language); - } - -} diff --git a/vespajlib/src/test/java/com/yahoo/io/reader/NamedReaderTestCase.java b/vespajlib/src/test/java/com/yahoo/io/reader/NamedReaderTestCase.java index 3bbbf3ea7de..6eb23277075 100644 --- a/vespajlib/src/test/java/com/yahoo/io/reader/NamedReaderTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/io/reader/NamedReaderTestCase.java @@ -13,6 +13,7 @@ import com.yahoo.protect.ClassValidator; import org.junit.Test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; /** * Tests all method of NamedReader. @@ -35,8 +36,10 @@ public class NamedReaderTestCase { @Test public void testMethodMasking() { - assertEquals(0, - ClassValidator.unmaskedMethodsFromSuperclass(NamedReader.class).size()); + List<Method> unmaskedMethods = ClassValidator.unmaskedMethodsFromSuperclass(NamedReader.class); + if (! unmaskedMethods.isEmpty()) { + fail("Unmasked methods in " + NamedReader.class.getName() + ": " + unmaskedMethods); + } } private static class MarkerReader extends Reader { diff --git a/vespajlib/src/test/java/com/yahoo/text/DoubleFormatterTestCase.java b/vespajlib/src/test/java/com/yahoo/text/DoubleFormatterTestCase.java deleted file mode 100644 index 3d0c623bbf4..00000000000 --- a/vespajlib/src/test/java/com/yahoo/text/DoubleFormatterTestCase.java +++ /dev/null @@ -1,205 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.text; - -import org.junit.Test; -import static org.junit.Assert.assertEquals; - -@SuppressWarnings("deprecation") -/** - * @author arnej27959 - */ -public class DoubleFormatterTestCase { - - @Test - public void testZero() { - String zero = DoubleFormatter.stringValue(0.0); - //assertEquals("0.0", zero); - } - - @Test - public void testOne() { - String one = DoubleFormatter.stringValue(1.0); - assertEquals("1.0", one); - } - - @Test - public void testMinusOne() { - String one = DoubleFormatter.stringValue(-1.0); - assertEquals("-1.0", one); - } - - @Test - public void testNanInf() { - String plusInf = DoubleFormatter.stringValue(Double.POSITIVE_INFINITY); - assertEquals("Infinity", plusInf); - - String notAnum = DoubleFormatter.stringValue(Double.NaN); - assertEquals("NaN", notAnum); - - String negInf = DoubleFormatter.stringValue(Double.NEGATIVE_INFINITY); - assertEquals("-Infinity", negInf); - } - - @Test - public void testSeven() { - String seven = DoubleFormatter.stringValue(7.0); - assertEquals("7.0", seven); - - seven = DoubleFormatter.stringValue(77.0); - assertEquals("77.0", seven); - - seven = DoubleFormatter.stringValue(7777.0); - assertEquals("7777.0", seven); - - seven = DoubleFormatter.stringValue(7777007777.0); - assertEquals("7.777007777E9", seven); - } - - - @Test - public void testSomeChosenNumbers() { - String s = DoubleFormatter.stringValue(4097.0); - assertEquals("4097.0", s); - - s = DoubleFormatter.stringValue(4097.5); - assertEquals("4097.5", s); - - s = DoubleFormatter.stringValue(1073741823.0); - assertEquals("1.073741823E9", s); - - s = DoubleFormatter.stringValue(1073741823.5); - assertEquals("1.0737418235E9", s); - - s = DoubleFormatter.stringValue(1073741825.5); - assertEquals("1.0737418255E9", s); - - s = DoubleFormatter.stringValue(1.23456789012345669); - assertEquals("1.2345678901234567", s); - s = DoubleFormatter.stringValue(12.3456789012345673); - assertEquals("12.345678901234567", s); - s = DoubleFormatter.stringValue(123.456789012345666); - assertEquals("123.45678901234567", s); - s = DoubleFormatter.stringValue(1234.56789012345666); - assertEquals("1234.5678901234567", s); - s = DoubleFormatter.stringValue(12345.6789012345670); - assertEquals("12345.678901234567", s); - s = DoubleFormatter.stringValue(123456.789012345674); - assertEquals("123456.78901234567", s); - s = DoubleFormatter.stringValue(1234567.89012345671); - assertEquals("1234567.8901234567", s); - - s = DoubleFormatter.stringValue(0.99); - // assertEquals("0.99", s); - - s = DoubleFormatter.stringValue(0.5); - assertEquals("0.5", s); - - s = DoubleFormatter.stringValue(0.1); - // assertEquals("0.1", s); - - s = DoubleFormatter.stringValue(0.00123456789); - // assertEquals("0.00123456789", s); - - s = DoubleFormatter.stringValue(0.0000000000001); - // assertEquals("0.0000000000001", s); - } - - @Test - public void testPowersOfTwo() { - String twos = DoubleFormatter.stringValue(2.0); - assertEquals("2.0", twos); - - twos = DoubleFormatter.stringValue(128.0); - assertEquals("128.0", twos); - - twos = DoubleFormatter.stringValue(1048576.0); - assertEquals("1048576.0", twos); - - twos = DoubleFormatter.stringValue(1073741824.0); - assertEquals("1.073741824E9", twos); - } - - @Test - public void testSmallNumbers() { - for (double d = 1.0; d > 1.0e-200; d *= 0.75) { - String fs = DoubleFormatter.stringValue(d); - String vs = String.valueOf(d); - double rp = Double.valueOf(fs); - if (d != rp) { - // System.err.println("differs: "+d+" became "+fs+" then instead: "+rp+" diff: "+(d-rp)); - } else if (! fs.equals(vs)) { - // System.err.println("string rep differs: "+vs+" became "+fs); - } - assertEquals(d, rp, 1.0e-7*d); - } - } - - @Test - public void testVerySmallNumbers() { - for (double d = 1.0; d > 1.0e-200; d *= 0.5) { - String fs = DoubleFormatter.stringValue(d); - String vs = String.valueOf(d); - double rp = Double.valueOf(fs); - if (d != rp) { - // System.err.println("differs: "+d+" became "+fs+" then instead: "+rp+" diff: "+(d-rp)); - } else if (! fs.equals(vs)) { - // System.err.println("string rep differs: "+vs+" became "+fs); - } - assertEquals(d, rp, 1.0e-13*d); - } - } - - @Test - public void testVeryVerySmallNumbers() { - for (double d = 1.0e-200; d > 0; d *= 0.5) { - String fs = DoubleFormatter.stringValue(d); - String vs = String.valueOf(d); - double rp = Double.valueOf(fs); - if (d != rp) { - // System.err.println("differs: "+d+" became "+fs+" then instead: "+rp+" diff: "+(d-rp)); - } else if (! fs.equals(vs)) { - // System.err.println("string rep differs: "+vs+" became "+fs); - } - assertEquals(d, rp, 1.0e-13*d); - } - } - - @Test - public void testVeryBigNumbers() { - for (double d = 1.0; d < Double.POSITIVE_INFINITY; d *= 2.0) { - String fs = DoubleFormatter.stringValue(d); - String vs = String.valueOf(d); - double rp = Double.valueOf(fs); - if (d != rp) { - // System.err.println("differs: "+d+" became "+fs+" then instead: "+rp); - } else if (! fs.equals(vs)) { - // System.err.println("string rep differs: "+vs+" became "+fs); - } - assertEquals(d, rp, 1.0e-13*d); - } - - assertEquals("1.0E200", String.valueOf(1.0e+200)); - - String big = DoubleFormatter.stringValue(1.0e+200); - assertEquals("1.0E200", big); - - big = DoubleFormatter.stringValue(1.0e+298); - assertEquals("1.0E298", big); - - big = DoubleFormatter.stringValue(1.0e+299); - assertEquals("1.0E299", big); - - big = DoubleFormatter.stringValue(1.0e+300); - assertEquals("1.0E300", big); - - } - - @Test - public void testRandomNumbers() { - java.util.Random rgen = new java.util.Random(0xCafeBabe); - for (int i = 0; i < 123456; i++) { - double d = rgen.nextDouble(); - } - } - -} diff --git a/vespajlib/src/test/java/com/yahoo/text/DoubleParserTestCase.java b/vespajlib/src/test/java/com/yahoo/text/DoubleParserTestCase.java deleted file mode 100644 index a48a1b6a943..00000000000 --- a/vespajlib/src/test/java/com/yahoo/text/DoubleParserTestCase.java +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.text; - -import org.junit.Test; -import static org.junit.Assert.assertEquals; - -@SuppressWarnings("deprecation") -/** - * @author arnej27959 - */ -public class DoubleParserTestCase { - - @Test - public void testZero() { - String[] zeros = { - "0", - "0.", - ".0", - "0.0", - "0.0e0", - "0.0e99", - "0.0e+300", - "0.0e-42" - }; - for (String s : zeros) { - double d = DoubleParser.parse(s); - assertEquals(0.0, d, 0); - } - } - - @Test - public void testOne() { - String[] ones = { - "1", - "1.", - "1.0", - "+1", - "10.0e-1", - "0.1e1", - "1000.0e-3", - ".001e+3", - }; - for (String s : ones) { - System.out.println("parsing: '"+s+"' now"); - double d = DoubleParser.parse(s); - System.out.println("expected: 1.0"); - System.out.println("actual: "+d); - assertEquals(1.0, d, 0); - } - } - - @Test - public void testMinusOne() { - String[] numbers = { - "-1", - "-1.0", - "-1.", - "-1e0", - "-10e-1", - }; - for (String s : numbers) { - System.out.println("parsing: '"+s+"' now"); - double d = DoubleParser.parse(s); - System.out.println("expected: -1.0"); - System.out.println("actual: "+d); - assertEquals(-1.0, d, 0); - } - } - - @Test - public void testNanInf() { - String[] numbers = { - "NaN", - "Infinity", - "-Infinity", - "+Infinity", - "+NaN", - "-NaN" - }; - for (String s : numbers) { - System.out.println("parsing: '"+s+"' now"); - double d1 = Double.parseDouble(s); - double d2 = DoubleParser.parse(s); - long lb1 = Double.doubleToRawLongBits(d1); - long lb2 = Double.doubleToRawLongBits(d2); - assertEquals(lb1, lb2, 0); - } - } - - @Test - public void testSeven() { - String[] sevens = { - "7", - "7.", - "7.0", - "70.0e-1", - "0.7e1", - "7000.0e-3", - ".007e+3", - }; - for (String s : sevens) { - System.out.println("parsing: '"+s+"' now"); - double d = DoubleParser.parse(s); - System.out.println("expected: 7.0"); - System.out.println("actual: "+d); - assertEquals(7.0, d, 0); - } - } - - @Test - public void testVerySmallNumbers() { - String[] numbers = { - "1.e-320", - "-1.e-320", - "1.0013378241589014e-303" - }; - for (String s : numbers) { - System.out.println("parsing: '"+s+"' now"); - double d1 = Double.parseDouble(s); - double d2 = DoubleParser.parse(s); - System.out.println("expected: "+d1); - System.out.println("actual: "+d2); - assertEquals(d1, d2, 0); - } - } - - @Test - public void testRandomNumbers() { - java.util.Random rgen = new java.util.Random(0xCafeBabe); - for (int i = 0; i < 123456; i++) { - double d = rgen.nextDouble(); - int exp = rgen.nextInt(); - d *= Math.pow(1.0000006, exp); - String s = Double.toString(d); - double d2 = Double.parseDouble(s); - double d3 = DoubleParser.parse(s); - - if (d != d2) { - System.out.println("WARNING: value ["+d+"] parses as ["+d2+"] by Java"); - } - double allow = 1.0e-14 * d2; - if (allow < 0) { - allow = -allow; - } - if (d2 != d3) { - long lb2 = Double.doubleToRawLongBits(d2); - long lb3 = Double.doubleToRawLongBits(d3); - if (lb2 - lb3 > 15 || lb3 - lb2 > 15) { - System.out.println("WARNING: string '"+s+"' parses as"); - System.out.println("["+d2+"] by Java, ["+d3+"] by our method"); - System.out.println("["+Long.toHexString(lb2)+"] bits vs ["+Long.toHexString(lb3)+"]"); - System.out.println("==> "+(lb2 - lb3)+" <== diff value"); - } - } - assertEquals(d2, d3, allow); - } - } - -} diff --git a/vespajlib/src/test/java/com/yahoo/text/DoubleToStringBenchmark.java b/vespajlib/src/test/java/com/yahoo/text/DoubleToStringBenchmark.java deleted file mode 100644 index a5925df2caf..00000000000 --- a/vespajlib/src/test/java/com/yahoo/text/DoubleToStringBenchmark.java +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.text; - -import org.junit.Ignore; -import org.junit.Test; - -import java.util.Arrays; -import java.util.List; -import java.util.Random; -import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; - -@SuppressWarnings("deprecation") -/** - * @author arnej27959 - */ -public class DoubleToStringBenchmark { - - @Test - @Ignore - public void benchmarkStringConstruction() throws Exception { - List<Class<? extends Benchmark.Task>> taskList = Arrays.asList(UseStringValueOf.class, - UseDoubleFormatter.class, - UseDoubleFormatter.class, - UseStringValueOf.class, - UseStringValueOf.class, - UseDoubleFormatter.class, - UseDoubleFormatter.class, - UseStringValueOf.class, - UseDoubleFormatter.class, - UseStringValueOf.class); - - int maxThreads = 256; - int dummy = 0; - System.out.print("warmup"); - for (Class<? extends Benchmark.Task> taskClass : taskList) { - dummy += runBenchmark(maxThreads, taskClass); - System.out.print("."); - } - System.out.println(" " + dummy); - - System.out.format("%-35s", ""); - for (int numThreads = 1; numThreads <= maxThreads; numThreads *= 2) { - System.out.format("%13s t ", numThreads); - } - System.out.println(); - for (Class<? extends Benchmark.Task> taskClass : taskList) { - System.out.format("%-35s", taskClass.getSimpleName()); - for (int numThreads = 1; numThreads <= maxThreads; numThreads *= 2) { - System.out.format("%15d ", runBenchmark(numThreads, taskClass)); - } - System.out.println(); - } - } - - private long runBenchmark(int numThreads, Class<? extends Benchmark.Task> taskClass) throws Exception { - return new Benchmark.Builder() - .setNumIterationsPerThread(80000) - .setNumThreads(numThreads) - .setTaskClass(taskClass) - .build() - .run(); - } - - public static class UseStringValueOf implements Benchmark.Task { - - private long timeIt(Random randomGen, int num) { - long before = System.nanoTime(); - - String str = null; - double v = 0.0; - for (int i = 0; i < num; ++i) { - v = randomGen.nextDouble() * 1.0e-2; - str = String.valueOf(v); - } - - long after = System.nanoTime(); - assertEquals(""+v, str); - return after - before; - } - - @Override - public long run(CyclicBarrier barrier, int numIterations) throws Exception { - Random randomGen = new Random(0xDeadBeef); - barrier.await(600, TimeUnit.SECONDS); - long t1 = timeIt(randomGen, numIterations / 4); - long t2 = timeIt(randomGen, numIterations / 2); - long t3 = timeIt(randomGen, numIterations / 4); - return t2; - } - } - - public static class UseDoubleFormatter implements Benchmark.Task { - - private long timeIt(Random randomGen, int num) { - long before = System.nanoTime(); - - String str = null; - double v = 0.0; - for (int i = 0; i < num; ++i) { - v = randomGen.nextDouble() * 1.0e-2; - str = DoubleFormatter.stringValue(v); - } - - long after = System.nanoTime(); - // assertEquals(""+v, str); - return after - before; - } - - - @Override - public long run(CyclicBarrier barrier, int numIterations) throws Exception { - Random randomGen = new Random(0xDeadBeef); - barrier.await(600, TimeUnit.SECONDS); - long t1 = timeIt(randomGen, numIterations / 4); - long t2 = timeIt(randomGen, numIterations / 2); - long t3 = timeIt(randomGen, numIterations / 4); - return t2; - } - } - -} diff --git a/vespajlib/src/test/java/com/yahoo/text/LanguageHacksTestCase.java b/vespajlib/src/test/java/com/yahoo/text/LanguageHacksTestCase.java deleted file mode 100644 index 4231c6ede02..00000000000 --- a/vespajlib/src/test/java/com/yahoo/text/LanguageHacksTestCase.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.text; - -import org.junit.Test; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * Test for LanguageHacks. - * $Id$ - */ -@SuppressWarnings("deprecation") -public class LanguageHacksTestCase { - - @Test - public void isCJK() { - assertFalse("NULL language", LanguageHacks.isCJK(null)); - assertTrue(LanguageHacks.isCJK("zh")); - assertFalse("Norwegian is CJK", LanguageHacks.isCJK("no")); - } - - @Test - public void yellDesegments() { - assertFalse("NULL language", LanguageHacks.yellDesegments(null)); - assertTrue(LanguageHacks.yellDesegments("de")); - assertFalse("Norwegian desegments", LanguageHacks.yellDesegments("no")); - } -} diff --git a/vespalog/pom.xml b/vespalog/pom.xml index 084972c2d25..10000d380a0 100644 --- a/vespalog/pom.xml +++ b/vespalog/pom.xml @@ -5,11 +5,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespalog</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <name>Vespa Log</name> <packaging>container-plugin</packaging> diff --git a/vsm/pom.xml b/vsm/pom.xml index 3c491906060..eb06ea1a35b 100644 --- a/vsm/pom.xml +++ b/vsm/pom.xml @@ -7,11 +7,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vsm</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/yolean/pom.xml b/yolean/pom.xml index 3b1f8887906..742142fe167 100644 --- a/yolean/pom.xml +++ b/yolean/pom.xml @@ -5,11 +5,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>yolean</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>container-plugin</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/yolean/src/main/java/com/yahoo/yolean/concurrent/ThreadRobustList.java b/yolean/src/main/java/com/yahoo/yolean/concurrent/ThreadRobustList.java index 3f57319cdce..72020bbaf88 100644 --- a/yolean/src/main/java/com/yahoo/yolean/concurrent/ThreadRobustList.java +++ b/yolean/src/main/java/com/yahoo/yolean/concurrent/ThreadRobustList.java @@ -8,15 +8,15 @@ import java.util.NoSuchElementException; /** * <p>This class implements a thread-safe, lock-free list of Objects that supports multiple readers and a single writer. * Because there are no locks or other memory barriers involved, there exists no <i>happens-before</i> relationship - * among calls to either methods of the <tt>ThreadRobustList</tt>. This means that there are no guarantees as to when + * among calls to either methods of the <code>ThreadRobustList</code>. This means that there are no guarantees as to when * (or even if) an item {@link #add(Object)}ed becomes visible through {@link #iterator()}. If visibility is required, * either use explicit synchronization between reader and writer thread, or move to a different concurrent collection - * (e.g. <tt>CopyOnWriteArrayList</tt>).</p> - * <p>Because it is lock-free, the <tt>ThreadRobustList</tt> has minimal overhead to both reading and writing. The + * (e.g. <code>CopyOnWriteArrayList</code>).</p> + * <p>Because it is lock-free, the <code>ThreadRobustList</code> has minimal overhead to both reading and writing. The * iterator offered by this class always observes the list in a consistent state, and it never throws a - * <tt>ConcurrentModificationException</tt>.</p> - * <p>The <tt>ThreadRobustList</tt> does not permit adding <tt>null</tt> items.</p> - * <p>The usage of <tt>ThreadRobustList</tt> has no memory consistency effects. </p> + * <code>ConcurrentModificationException</code>.</p> + * <p>The <code>ThreadRobustList</code> does not permit adding <code>null</code> items.</p> + * <p>The usage of <code>ThreadRobustList</code> has no memory consistency effects. </p> * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> * @author bratseth @@ -28,7 +28,7 @@ public class ThreadRobustList<T> implements Iterable<T> { private int next = 0; /** - * <p>Constructs a new instance of this class with an initial capacity of <tt>10</tt>.</p> + * <p>Constructs a new instance of this class with an initial capacity of <code>10</code>.</p> */ public ThreadRobustList() { this(10); @@ -46,18 +46,18 @@ public class ThreadRobustList<T> implements Iterable<T> { /** * <p>Returns whether or not this list is empty.</p> * - * @return <tt>true</tt> if this list has zero items + * @return <code>true</code> if this list has zero items */ public boolean isEmpty() { return next == 0; } /** - * <p>Adds an item to this list. As opposed to <tt>CopyOnWriteArrayList</tt>, items added to this list may become + * <p>Adds an item to this list. As opposed to <code>CopyOnWriteArrayList</code>, items added to this list may become * visible to iterators created <em>before</em> a call to this method.</p> * * @param item the item to add - * @throws NullPointerException if <tt>item</tt> is <tt>null</tt> + * @throws NullPointerException if <code>item</code> is <code>null</code> */ public void add(T item) { if (item == null) { @@ -74,9 +74,9 @@ public class ThreadRobustList<T> implements Iterable<T> { } /** - * <p>Returns an iterator over the items in this list. As opposed to <tt>CopyOnWriteArrayList</tt>, this iterator - * may see items added to the <tt>ThreadRobustList</tt> even if they occur <em>after</em> a call to this method.</p> - * <p>The returned iterator does not support <tt>remove()</tt>.</p> + * <p>Returns an iterator over the items in this list. As opposed to <code>CopyOnWriteArrayList</code>, this iterator + * may see items added to the <code>ThreadRobustList</code> even if they occur <em>after</em> a call to this method.</p> + * <p>The returned iterator does not support <code>remove()</code>.</p> * * @return an iterator over this list */ diff --git a/yolean/src/main/java/com/yahoo/yolean/trace/TraceNode.java b/yolean/src/main/java/com/yahoo/yolean/trace/TraceNode.java index 92e083c9998..b3c4d67db47 100644 --- a/yolean/src/main/java/com/yahoo/yolean/trace/TraceNode.java +++ b/yolean/src/main/java/com/yahoo/yolean/trace/TraceNode.java @@ -10,16 +10,16 @@ import java.util.List; import java.util.NoSuchElementException; /** - * <p>This class represents a single node in a tree of <tt>TraceNodes</tt>. The trace forms a tree where there is a - * branch for each parallel execution, and a node within such a branch for each traced event. As each <tt>TraceNode</tt> + * <p>This class represents a single node in a tree of <code>TraceNodes</code>. The trace forms a tree where there is a + * branch for each parallel execution, and a node within such a branch for each traced event. As each <code>TraceNode</code> * may contain a payload of any type, the trace tree can be used to exchange any thread-safe state between producers and * consumers in different threads, whether or not the shape of the trace tree is relevant to the particular * information.</p> * <p>This class uses a {@link ThreadRobustList} for its children. That list allows multiple threads to inspect the - * hierarchy of a <tt>TraceNode</tt> tree while there are other threads concurrently modifying it, without incurring the - * cost of memory synchronization. The only caveat being that for each <tt>TraceNode</tt> there can never be more than - * exactly one writer thread. If multiple threads need to mutate a single <tt>TraceNode</tt>, then the writer threads - * need to synchronize their access on the <tt>TraceNode</tt>.</p> + * hierarchy of a <code>TraceNode</code> tree while there are other threads concurrently modifying it, without incurring the + * cost of memory synchronization. The only caveat being that for each <code>TraceNode</code> there can never be more than + * exactly one writer thread. If multiple threads need to mutate a single <code>TraceNode</code>, then the writer threads + * need to synchronize their access on the <code>TraceNode</code>.</p> * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> * @author bratseth @@ -35,7 +35,7 @@ public class TraceNode { /** * <p>Creates a new instance of this class.</p> * - * @param payload the payload to assign to this, may be <tt>null</tt> + * @param payload the payload to assign to this, may be <code>null</code> * @param timestamp the timestamp to assign to this */ public TraceNode(Object payload, long timestamp) { @@ -44,11 +44,11 @@ public class TraceNode { } /** - * <p>Adds another <tt>TraceNode</tt> as a child to this.</p> + * <p>Adds another <code>TraceNode</code> as a child to this.</p> * * @param child the TraceNode to add * @return this, to allow chaining - * @throws IllegalArgumentException if <tt>child</tt> is not a root TraceNode + * @throws IllegalArgumentException if <code>child</code> is not a root TraceNode * @see #isRoot() */ public TraceNode add(TraceNode child) { @@ -64,12 +64,12 @@ public class TraceNode { } /** - * <p>Returns a read-only iterable of all {@link #payload() payloads} that are instances of <tt>payloadType</tt>, - * in all its decendants. The payload of <em>this</em> <tt>TraceNode</tt> is ignored.</p> + * <p>Returns a read-only iterable of all {@link #payload() payloads} that are instances of <code>payloadType</code>, + * in all its decendants. The payload of <em>this</em> <code>TraceNode</code> is ignored.</p> * <p>The payloads are retrieved in depth-first, prefix order.</p> * * @param payloadType the type of payloads to retrieve - * @return the payloads, never <tt>null</tt> + * @return the payloads, never <code>null</code> */ public <PAYLOADTYPE> Iterable<PAYLOADTYPE> descendants(final Class<PAYLOADTYPE> payloadType) { if (children == null) { @@ -85,7 +85,7 @@ public class TraceNode { } /** - * <p>Returns the payload of this <tt>TraceNode</tt>, or null if none.</p> + * <p>Returns the payload of this <code>TraceNode</code>, or null if none.</p> * * @return the payload */ @@ -94,7 +94,7 @@ public class TraceNode { } /** - * <p>Returns the timestamp of this <tt>TraceNode</tt>.</p> + * <p>Returns the timestamp of this <code>TraceNode</code>.</p> * * @return the timestamp */ @@ -103,7 +103,7 @@ public class TraceNode { } /** - * <p>Returns the parent <tt>TraceNode</tt> of this.</p> + * <p>Returns the parent <code>TraceNode</code> of this.</p> * * @return the parent */ @@ -112,7 +112,7 @@ public class TraceNode { } /** - * <p>Returns the child <tt>TraceNodes</tt> of this.</p> + * <p>Returns the child <code>TraceNodes</code> of this.</p> * * @return the children */ @@ -124,16 +124,16 @@ public class TraceNode { } /** - * <p>Returns whether or not this <tt>TraceNode</tt> is a root node (i.e. it has no parent).</p> + * <p>Returns whether or not this <code>TraceNode</code> is a root node (i.e. it has no parent).</p> * - * @return <tt>true</tt> if {@link #parent()} returns <tt>null</tt> + * @return <code>true</code> if {@link #parent()} returns <code>null</code> */ public boolean isRoot() { return parent == null; } /** - * <p>Returns the root <tt>TraceNode</tt> of the tree that this <tt>TraceNode</tt> belongs to.</p> + * <p>Returns the root <code>TraceNode</code> of the tree that this <code>TraceNode</code> belongs to.</p> * * @return the root */ @@ -146,7 +146,7 @@ public class TraceNode { } /** - * <p>Visits this <tt>TraceNode</tt> and all of its descendants in depth-first, prefix order.</p> + * <p>Visits this <code>TraceNode</code> and all of its descendants in depth-first, prefix order.</p> * * @param visitor The visitor to accept. * @return The <code>visitor</code> parameter. diff --git a/yolean/src/main/java/com/yahoo/yolean/trace/TraceVisitor.java b/yolean/src/main/java/com/yahoo/yolean/trace/TraceVisitor.java index c45823da19a..23525b65571 100644 --- a/yolean/src/main/java/com/yahoo/yolean/trace/TraceVisitor.java +++ b/yolean/src/main/java/com/yahoo/yolean/trace/TraceVisitor.java @@ -13,17 +13,17 @@ public abstract class TraceVisitor { * <p>Visits a {@link TraceNode}. Called by {@link TraceNode#accept(TraceVisitor)}, before visiting its * children.</p> * - * @param node the <tt>TraceNode</tt> being visited + * @param node the <code>TraceNode</code> being visited * @see TraceNode#accept(TraceVisitor) */ public abstract void visit(TraceNode node); /** * <p>Enters a {@link TraceNode}. This method is called after {@link #visit(TraceNode)}, but before visiting its - * children. Note that this method is NOT called if a <tt>TraceNode</tt> has zero children.</p> + * children. Note that this method is NOT called if a <code>TraceNode</code> has zero children.</p> * <p>The default implementation of this method does nothing.</p> * - * @param node the <tt>TraceNode</tt> being entered + * @param node the <code>TraceNode</code> being entered * @see #leaving(TraceNode) */ @SuppressWarnings("UnusedParameters") @@ -33,10 +33,10 @@ public abstract class TraceVisitor { /** * <p>Leaves a {@link TraceNode}. This method is called after {@link #entering(TraceNode)}, and after visiting its - * children. Note that this method is NOT called if a <tt>TraceNode</tt> has zero children.</p> + * children. Note that this method is NOT called if a <code>TraceNode</code> has zero children.</p> * <p>The default implementation of this method does nothing.</p> * - * @param node the <tt>TraceNode</tt> being left + * @param node the <code>TraceNode</code> being left */ @SuppressWarnings("UnusedParameters") public void leaving(TraceNode node) { diff --git a/zkfacade/pom.xml b/zkfacade/pom.xml index 822335880c1..c57ab45e6c6 100644 --- a/zkfacade/pom.xml +++ b/zkfacade/pom.xml @@ -5,12 +5,12 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>zkfacade</artifactId> <packaging>container-plugin</packaging> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <dependencies> <dependency> <groupId>junit</groupId> |