diff options
165 files changed, 928 insertions, 7161 deletions
diff --git a/configd/src/apps/cmd/main.cpp b/configd/src/apps/cmd/main.cpp index 53edaf80dba..49767ab47aa 100644 --- a/configd/src/apps/cmd/main.cpp +++ b/configd/src/apps/cmd/main.cpp @@ -4,6 +4,7 @@ #include <unistd.h> #include <vespa/vespalib/util/signalhandler.h> +#include <vespa/vespalib/util/exception.h> #include <vespa/fnet/frt/supervisor.h> #include <vespa/fnet/frt/target.h> #include <vespa/fnet/frt/rpcrequest.h> @@ -64,8 +65,13 @@ int Cmd::run(const char *cmd, const char *arg) { int retval = 0; - initRPC("tcp/localhost:19097"); - + try { + initRPC("tcp/localhost:19097"); + } catch (vespalib::Exception &e) { + fprintf(stderr, "vespa-sentinel-cmd: exception in network initialization: %s\n", + e.what()); + return 2; + } FRT_RPCRequest *req = _server->supervisor().AllocRPCRequest(); req->SetMethodName(cmd); diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index e7e626f3d22..2c5d3ec60da 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -53,6 +53,7 @@ ztsUrl string default="" maintainerIntervalMinutes int default=30 # TODO: Default set to a high value (1 year) => maintainer will not run, change when maintainer verified out in prod tenantsMaintainerIntervalMinutes int default=525600 +keepUnusedFileReferencesHours int default=48 # Bootstrapping # How long bootstrapping can take before giving up (in seconds) diff --git a/configdefinitions/src/vespa/zookeeper-server.def b/configdefinitions/src/vespa/zookeeper-server.def index 723246daab0..b8ff6494764 100644 --- a/configdefinitions/src/vespa/zookeeper-server.def +++ b/configdefinitions/src/vespa/zookeeper-server.def @@ -42,3 +42,8 @@ server[].id int server[].hostname string server[].quorumPort int default=2182 server[].electionPort int default=2183 + +# Needed when upgrading from ZooKeeper 3.4 to 3.5, see https://issues.apache.org/jira/browse/ZOOKEEPER-3056, +# and in general where there is a zookeeper ensemble running that has had few transactions. +# TODO: Consider setting this to false by default (and override where appropriate) +trustEmptySnapshot bool default=true diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 3a9e5a55191..2734e5ad6b8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -409,7 +409,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return fileDistributionStatus.status(getApplication(applicationId), timeout); } - public Set<String> deleteUnusedFiledistributionReferences(File fileReferencesPath) { + public Set<String> deleteUnusedFiledistributionReferences(File fileReferencesPath, Duration keepFileReferences) { if (!fileReferencesPath.isDirectory()) throw new RuntimeException(fileReferencesPath + " is not a directory"); Set<String> fileReferencesInUse = new HashSet<>(); @@ -429,7 +429,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye fileReferencesOnDisk.addAll(Arrays.stream(filesOnDisk).map(File::getName).collect(Collectors.toSet())); log.log(LogLevel.DEBUG, "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk); - Instant instant = Instant.now().minus(Duration.ofDays(14)); + Instant instant = Instant.now().minus(keepFileReferences); Set<String> fileReferencesToDelete = fileReferencesOnDisk .stream() .filter(fileReference -> ! fileReferencesInUse.contains(fileReference)) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java index c6accd04896..8388159ba07 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java @@ -20,6 +20,7 @@ public class FileDistributionMaintainer extends Maintainer { private final ApplicationRepository applicationRepository; private final File fileReferencesDir; + private final ConfigserverConfig configserverConfig; FileDistributionMaintainer(ApplicationRepository applicationRepository, Curator curator, @@ -27,11 +28,13 @@ public class FileDistributionMaintainer extends Maintainer { ConfigserverConfig configserverConfig) { super(applicationRepository, curator, interval); this.applicationRepository = applicationRepository; + this.configserverConfig = configserverConfig; this.fileReferencesDir = new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir())); } @Override protected void maintain() { - applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir); + applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, + Duration.ofHours(configserverConfig.keepUnusedFileReferencesHours())); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 5731caff5a7..0802db23ea7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -3,14 +3,11 @@ package com.yahoo.vespa.config.server; import com.google.common.io.Files; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.component.Version; -import com.yahoo.component.Vtag; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Deployment; -import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.TenantName; @@ -224,7 +221,7 @@ public class ApplicationRepositoryTest { PrepareParams prepareParams = new PrepareParams.Builder().applicationId(applicationId()).ignoreValidationErrors(true).build(); deployApp(new File("src/test/apps/app"), prepareParams); - Set<String> toBeDeleted = applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir); + Set<String> toBeDeleted = applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, Duration.ofHours(48)); assertEquals(Collections.singleton("foo"), toBeDeleted); assertFalse(filereferenceDir.exists()); assertTrue(filereferenceDir2.exists()); diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 9292db89eb5..e5875247855 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -931,6 +931,7 @@ "public" ], "methods": [ + "public varargs void <init>(java.net.URI, java.lang.String[])", "public varargs void <init>(com.yahoo.container.jdisc.HttpRequest, java.lang.String[])", "public void render(java.io.OutputStream)", "public java.lang.String getContentType()" diff --git a/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java b/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java index 0a97a4d5d2f..2b3a272fadc 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java @@ -46,9 +46,7 @@ public class BundleLoader { return osgi.install(file.getAbsolutePath()); } - /** - * @return the number of bundles installed by this call. - */ + /** Returns the number of bundles installed by this call. */ private int install(List<FileReference> references) { Set<FileReference> bundlesToInstall = new HashSet<>(references); bundlesToInstall.removeAll(reference2Bundles.keySet()); @@ -115,12 +113,12 @@ public class BundleLoader { } } - //all bundles must have been started first to ensure correct package resolution. + // All bundles must have been started first to ensure correct package resolution. private void startBundles() { for (List<Bundle> bundles : reference2Bundles.values()) { for (Bundle bundle : bundles) { try { - if (!isFragment(bundle)) + if ( ! isFragment(bundle)) bundle.start(); } catch(Exception e) { throw new RuntimeException("Could not start bundle '" + bundle.getSymbolicName() + "'", e); @@ -129,18 +127,15 @@ public class BundleLoader { } } - // The OSGi APIs are just getting worse... private boolean isFragment(Bundle bundle) { BundleRevision bundleRevision = bundle.adapt(BundleRevision.class); if (bundleRevision == null) throw new NullPointerException("Null bundle revision means that bundle has probably been uninstalled: " + - bundle.getSymbolicName() + ":" + bundle.getVersion()); + bundle.getSymbolicName() + ":" + bundle.getVersion()); return (bundleRevision.getTypes() & BundleRevision.TYPE_FRAGMENT) != 0; } - /** - * Returns the number of uninstalled bundles - */ + /** Returns the number of uninstalled bundles */ private int retainOnly(List<FileReference> newReferences) { Set<Bundle> bundlesToRemove = new HashSet<>(Arrays.asList(osgi.getBundles())); @@ -182,4 +177,5 @@ public class BundleLoader { sb.append("}"); return sb.toString(); } + } diff --git a/container-core/src/main/java/com/yahoo/osgi/Osgi.java b/container-core/src/main/java/com/yahoo/osgi/Osgi.java index 31f1146c311..c94eaf43deb 100644 --- a/container-core/src/main/java/com/yahoo/osgi/Osgi.java +++ b/container-core/src/main/java/com/yahoo/osgi/Osgi.java @@ -2,9 +2,7 @@ package com.yahoo.osgi; import com.yahoo.component.ComponentSpecification; -import com.yahoo.container.bundle.BundleInstantiationSpecification; import org.osgi.framework.Bundle; -import org.osgi.framework.ServiceReference; import java.util.List; diff --git a/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java b/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java index 2e37a278387..8b2f20a1c13 100644 --- a/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java +++ b/container-core/src/main/java/com/yahoo/osgi/OsgiImpl.java @@ -2,17 +2,17 @@ package com.yahoo.osgi; import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.Version; import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.jdisc.application.OsgiFramework; import org.osgi.framework.Bundle; import org.osgi.framework.BundleException; -import org.osgi.framework.ServiceReference; import java.util.List; -import java.util.logging.Logger; /** * @author Tony Vaagenes + * @author bratseth */ public class OsgiImpl implements Osgi { @@ -62,13 +62,13 @@ public class OsgiImpl implements Osgi { } private static void ensureBundleActive(Bundle bundle) throws IllegalStateException { - final int state = bundle.getState(); + int state = bundle.getState(); Throwable cause = null; if (state != Bundle.ACTIVE) { try { - //Get the reason why the bundle isn't active. - //Do not change this method to not fail if start is successful without carefully analyzing - //why there are non-active bundles. + // Get the reason why the bundle isn't active. + // Do not change this method to not fail if start is successful without carefully analyzing + // why there are non-active bundles. bundle.start(); } catch (BundleException e) { cause = e; @@ -81,29 +81,29 @@ public class OsgiImpl implements Osgi { * Returns the bundle of a given name having the highest matching version * * @param id the id of the component to return. May not include a version, or include - * an underspecified version, in which case the highest (mathcing) version which + * an underspecified version, in which case the highest (matching) version which * does not contain a qualifier is returned * @return the bundle match having the highest version, or null if there was no matches */ public Bundle getBundle(ComponentSpecification id) { - Bundle highestMatch=null; + Bundle highestMatch = null; for (Bundle bundle : getBundles()) { assert bundle.getSymbolicName() != null : "ensureHasBundleSymbolicName not called during installation"; if ( ! bundle.getSymbolicName().equals(id.getName())) continue; if ( ! id.getVersionSpecification().matches(versionOf(bundle))) continue; - if (highestMatch==null || versionOf(highestMatch).compareTo(versionOf(bundle))<0) - highestMatch=bundle; + if (highestMatch == null || versionOf(highestMatch).compareTo(versionOf(bundle)) < 0) + highestMatch = bundle; } return highestMatch; } - /** returns the version of a bundle, as specified by Bundle-Version in the manifest */ - private static com.yahoo.component.Version versionOf(Bundle bundle) { - Object bundleVersion=bundle.getHeaders().get("Bundle-Version"); - if (bundleVersion==null) return com.yahoo.component.Version.emptyVersion; - return new com.yahoo.component.Version(bundleVersion.toString()); + /** Returns the version of a bundle, as specified by Bundle-Version in the manifest */ + private static Version versionOf(Bundle bundle) { + Object bundleVersion = bundle.getHeaders().get("Bundle-Version"); + if (bundleVersion == null) return Version.emptyVersion; + return new Version(bundleVersion.toString()); } @Override @@ -116,8 +116,8 @@ public class OsgiImpl implements Osgi { } private static String normalizeLocation(String location) { - if (location.indexOf(':')<0) - location="file:" + location; + if (location.indexOf(':') < 0) + location = "file:" + location; return location; } @@ -134,4 +134,5 @@ public class OsgiImpl implements Osgi { public void refreshPackages() { jdiscOsgi.refreshPackages(); } + } diff --git a/container-core/src/main/java/com/yahoo/restapi/MessageResponse.java b/container-core/src/main/java/com/yahoo/restapi/MessageResponse.java index 8669d4f9b0c..17ed321331b 100644 --- a/container-core/src/main/java/com/yahoo/restapi/MessageResponse.java +++ b/container-core/src/main/java/com/yahoo/restapi/MessageResponse.java @@ -9,6 +9,8 @@ import java.io.IOException; import java.io.OutputStream; /** + * A 200 ok response with a message in JSON. + * * @author bratseth */ public class MessageResponse extends HttpResponse { diff --git a/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java b/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java index 4852bfafa60..ff301d44798 100644 --- a/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java +++ b/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java @@ -9,6 +9,7 @@ import com.yahoo.slime.Slime; import java.io.IOException; import java.io.OutputStream; +import java.net.URI; /** * Returns a response containing an array of links to sub-resources @@ -19,18 +20,21 @@ public class ResourceResponse extends HttpResponse { private final Slime slime = new Slime(); - public ResourceResponse(HttpRequest request, String ... subResources) { + public ResourceResponse(URI parentUrl, String ... subResources) { super(200); Cursor resourceArray = slime.setObject().setArray("resources"); for (String subResource : subResources) { Cursor resourceEntry = resourceArray.addObject(); - resourceEntry.setString("url", new Uri(request.getUri()) - .append(subResource) - .withTrailingSlash() - .toString()); + resourceEntry.setString("url", new Uri(parentUrl).append(subResource) + .withTrailingSlash() + .toString()); } } + public ResourceResponse(HttpRequest request, String ... subResources) { + this(request.getUri(), subResources); + } + @Override public void render(OutputStream stream) throws IOException { new JsonFormat(true).encode(stream, slime); diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml index 9a4e3704ecd..92407aa9c68 100644 --- a/container-dependencies-enforcer/pom.xml +++ b/container-dependencies-enforcer/pom.xml @@ -68,7 +68,7 @@ </excludes> <includes> <include>com.yahoo.vespa</include> - <include>aopalliance:aopalliance:[1.0]:jar:provided</include> + <include>aopalliance:aopalliance:[${aopalliance.version}]:jar:provided</include> <include>com.fasterxml.jackson.core:jackson-annotations:[${jackson2.version}]:jar:provided</include> <include>com.fasterxml.jackson.core:jackson-core:[${jackson2.version}]:jar:provided</include> <include>com.fasterxml.jackson.core:jackson-databind:[${jackson-databind.version}]:jar:provided</include> @@ -89,18 +89,18 @@ <include>com.sun.activation:javax.activation:[1.2.0]:jar:provided</include> <include>com.sun.xml.bind:jaxb-core:[${jaxb.version}]:jar:provided</include> <include>com.sun.xml.bind:jaxb-impl:[${jaxb.version}]:jar:provided</include> - <include>commons-daemon:commons-daemon:[1.0.3]:jar:provided</include> + <include>commons-daemon:commons-daemon:[${commons-daemon.version}]:jar:provided</include> <include>commons-logging:commons-logging:[1.1.1]:jar:provided</include> <include>javax.annotation:javax.annotation-api:[${javax.annotation-api.version}]:jar:provided</include> - <include>javax.inject:javax.inject:[1]:jar:provided</include> - <include>javax.servlet:javax.servlet-api:[3.1.0]:jar:provided</include> + <include>javax.inject:javax.inject:[${javax.inject.version}]:jar:provided</include> + <include>javax.servlet:javax.servlet-api:[${javax.servlet-api.version}]:jar:provided</include> <include>javax.validation:validation-api:[${javax.validation-api.version}]:jar:provided</include> <include>javax.ws.rs:javax.ws.rs-api:[${javax.ws.rs-api.version}]:jar:provided</include> <include>javax.xml.bind:jaxb-api:[${jaxb.version}]:jar:provided</include> <include>net.jcip:jcip-annotations:[1.0]:jar:provided</include> - <include>net.jpountz.lz4:lz4:[1.3.0]:jar:provided</include> + <include>net.jpountz.lz4:lz4:[${lz4.version}]:jar:provided</include> <include>org.apache.felix:org.apache.felix.framework:[${felix.version}]:jar:provided</include> - <include>org.apache.felix:org.apache.felix.log:[1.0.1]:jar:provided</include> + <include>org.apache.felix:org.apache.felix.log:[${felix.log.version}]:jar:provided</include> <include>org.apache.felix:org.apache.felix.main:[${felix.version}]:jar:provided</include> <include>org.bouncycastle:bcpkix-jdk15on:[${bouncycastle.version}]:jar:provided</include> <include>org.bouncycastle:bcprov-jdk15on:[${bouncycastle.version}]:jar:provided</include> @@ -125,13 +125,13 @@ <include>org.glassfish.jersey.media:jersey-media-json-jackson:[${jersey2.version}]:jar:provided</include> <include>org.glassfish.jersey.media:jersey-media-multipart:[${jersey2.version}]:jar:provided</include> <include>org.javassist:javassist:[${javassist.version}]:jar:provided</include> - <include>org.json:json:[20090211]:jar:provided</include> + <include>org.json:json:[${org.json.version}]:jar:provided</include> <include>org.jvnet.mimepull:mimepull:[${mimepull.version}]:jar:provided</include> <include>org.slf4j:jcl-over-slf4j:[${slf4j.version}]:jar:provided</include> <include>org.slf4j:log4j-over-slf4j:[${slf4j.version}]:jar:provided</include> <include>org.slf4j:slf4j-api:[${slf4j.version}]:jar:provided</include> <include>org.slf4j:slf4j-jdk14:[${slf4j.version}]:jar:provided</include> - <include>xml-apis:xml-apis:[1.4.01]:jar:provided</include> + <include>xml-apis:xml-apis:[${xml-apis.version}]:jar:provided</include> </includes> </bannedDependencies> </rules> diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index c16f1b51ce2..8093fa41e43 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -39,7 +39,7 @@ <dependency> <groupId>aopalliance</groupId> <artifactId>aopalliance</artifactId> - <version>1.0</version> + <version>${aopalliance.version}</version> </dependency> <dependency> <groupId>com.fasterxml.jackson.core</groupId> @@ -117,12 +117,12 @@ <dependency> <groupId>commons-daemon</groupId> <artifactId>commons-daemon</artifactId> - <version>1.0.3</version> + <version>${commons-daemon.version}</version> </dependency> <dependency> - <!-- This version is exported by jdisc via jcl-over-slf4j. --> <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId> + <!-- This version is exported by jdisc via jcl-over-slf4j. --> <version>1.1.1</version> </dependency> <dependency> @@ -133,12 +133,12 @@ <dependency> <groupId>javax.inject</groupId> <artifactId>javax.inject</artifactId> - <version>1</version> + <version>${javax.inject.version}</version> </dependency> <dependency> <groupId>javax.servlet</groupId> <artifactId>javax.servlet-api</artifactId> - <version>3.1.0</version> + <version>${javax.servlet-api.version}</version> </dependency> <dependency> <groupId>javax.validation</groupId> @@ -186,7 +186,7 @@ <dependency> <groupId>net.jpountz.lz4</groupId> <artifactId>lz4</artifactId> - <version>1.3.0</version> + <version>${lz4.version}</version> </dependency> <dependency> <groupId>org.apache.felix</groupId> @@ -196,7 +196,7 @@ <dependency> <groupId>org.apache.felix</groupId> <artifactId>org.apache.felix.log</artifactId> - <version>1.0.1</version> + <version>${felix.log.version}</version> </dependency> <dependency> <groupId>org.apache.felix</groupId> @@ -321,7 +321,7 @@ <dependency> <groupId>org.json</groupId> <artifactId>json</artifactId> - <version>20090211</version> + <version>${org.json.version}</version> </dependency> <dependency> <groupId>org.jvnet.mimepull</groupId> @@ -351,7 +351,7 @@ <dependency> <groupId>xml-apis</groupId> <artifactId>xml-apis</artifactId> - <version>1.4.01</version> + <version>${xml-apis.version}</version> </dependency> <!-- NOTE: The dependencies below are not provided from the jdisc container runtime, but had to be moved @@ -415,15 +415,47 @@ </dependencies> </dependencyManagement> + <profiles> + <profile> + <id>check-dependency-properties</id> + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>versions-maven-plugin</artifactId> + <version>${versions-maven-plugin.version}</version> + <executions> + <execution> + <phase>verify</phase> + <goals> + <goal>display-property-updates</goal> + <goal>property-updates-report</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> + <properties> + <aopalliance.version>1.0</aopalliance.version> <bouncycastle.version>1.58</bouncycastle.version> + <commons-daemon.version>1.0.3</commons-daemon.version> <felix.version>6.0.3</felix.version> + <felix.log.version>1.0.1</felix.log.version> <findbugs.version>1.3.9</findbugs.version> <guava.version>20.0</guava.version> <guice.version>3.0</guice.version> + <javax.inject.version>1</javax.inject.version> + <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <jaxb.version>2.3.0</jaxb.version> <jetty.version>9.4.20.v20190813</jetty.version> + <lz4.version>1.3.0</lz4.version> + <org.json.version>20090211</org.json.version> <slf4j.version>1.7.5</slf4j.version> + <xml-apis.version>1.4.01</xml-apis.version> <!-- These must be kept in sync with version used by current jersey2.version. --> <!-- MUST be updated each time jersey2 is upgraded! --> @@ -439,6 +471,9 @@ <javax.ws.rs-api.version>2.0.1</javax.ws.rs-api.version> <jersey2.version>2.25</jersey2.version> <mimepull.version>1.9.6</mimepull.version> + + <!-- Not a dependency. Only included to allow the versions-maven-plugin to check for updates of itself --> + <versions-maven-plugin.version>2.7</versions-maven-plugin.version> </properties> </project> diff --git a/container-di/src/main/java/com/yahoo/container/bundle/BundleInstantiationSpecification.java b/container-di/src/main/java/com/yahoo/container/bundle/BundleInstantiationSpecification.java index 440a687a671..0fb8a99a957 100644 --- a/container-di/src/main/java/com/yahoo/container/bundle/BundleInstantiationSpecification.java +++ b/container-di/src/main/java/com/yahoo/container/bundle/BundleInstantiationSpecification.java @@ -63,13 +63,13 @@ public final class BundleInstantiationSpecification { new ComponentSpecification(idSpec), (classSpec == null || classSpec.isEmpty())? null : new ComponentSpecification(classSpec), (bundleSpec == null || bundleSpec.isEmpty())? null : new ComponentSpecification(bundleSpec)); - } /** * Return a new instance of the specification with bundle name altered - * @param bundleName New name of bundle - * @return the new instance of the specification. + * + * @param bundleName the new name of the bundle + * @return the new instance of the specification */ public BundleInstantiationSpecification inBundle(String bundleName) { return new BundleInstantiationSpecification(this.id, this.classId, new ComponentSpecification(bundleName)); diff --git a/container-disc/abi-spec.json b/container-disc/abi-spec.json index a0c8b74b0e3..35280e12146 100644 --- a/container-disc/abi-spec.json +++ b/container-disc/abi-spec.json @@ -41,7 +41,8 @@ ], "methods": [ "public abstract java.lang.String getSecret(java.lang.String)", - "public abstract java.lang.String getSecret(java.lang.String, int)" + "public abstract java.lang.String getSecret(java.lang.String, int)", + "public java.util.List listSecretVersions(java.lang.String)" ], "fields": [] } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/secretstore/SecretStore.java b/container-disc/src/main/java/com/yahoo/container/jdisc/secretstore/SecretStore.java index 902d924c431..7cd8e11c677 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/secretstore/SecretStore.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/secretstore/SecretStore.java @@ -2,6 +2,8 @@ package com.yahoo.container.jdisc.secretstore; +import java.util.List; + /** * @author mortent */ @@ -11,4 +13,9 @@ public interface SecretStore { /** Returns the secret for this key and version */ String getSecret(String key, int version); + + /** Lists the existing versions of this secret (nonnegative integers) */ + default List<Integer> listSecretVersions(String key) { + throw new UnsupportedOperationException("Secret store does not support listing versions"); + } } diff --git a/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 b/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 index 2e8aac43a65..639ff8255d2 100644 --- a/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 +++ b/container-search/src/main/antlr4/com/yahoo/search/yql/yqlplus.g4 @@ -298,7 +298,7 @@ ident keyword_as_ident : SELECT | TABLE | DELETE | INTO | VALUES | LIMIT | OFFSET | WHERE | 'order' | 'by' | DESC | MERGE | LEFT | JOIN | ON | OUTPUT | COUNT | BEGIN | END | APPLY | TYPE_BYTE | TYPE_INT16 | TYPE_INT32 | TYPE_INT64 | TYPE_BOOLEAN | TYPE_TIMESTAMP | TYPE_DOUBLE | TYPE_STRING | TYPE_ARRAY | TYPE_MAP - | VIEW | CREATE | IMPORT | PROGRAM | NEXT | PAGED | SOURCES | SET | MATCHES + | VIEW | CREATE | IMPORT | PROGRAM | NEXT | PAGED | SOURCES | SET | MATCHES | LIKE ; //program : params? (import_statement SEMI)* (ddl SEMI)* (statement SEMI)* EOF diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 66a75b7092c..ca52c517bbf 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -464,8 +464,8 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { trace("No query profile is used", false, 1); else trace("Using " + profile.toString(), false, 1); - if (traceLevel < 4) return; + if (traceLevel < 4) return; StringBuilder b = new StringBuilder("Resolved properties:\n"); Set<String> mentioned = new HashSet<>(); for (Map.Entry<String,String> requestProperty : requestProperties().entrySet() ) { @@ -495,10 +495,10 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { return httpRequest.propertyMap(); } - private void appendQueryProfileProperties(CompiledQueryProfile profile,Set<String> mentioned,StringBuilder b) { - for (Map.Entry<String,Object> property : profile.listValues("", requestProperties()).entrySet()) { + private void appendQueryProfileProperties(CompiledQueryProfile profile, Set<String> mentioned, StringBuilder b) { + for (Map.Entry<String,Object> property : profile.listValues(CompoundName.empty, requestProperties(), properties()).entrySet()) { if ( ! mentioned.contains(property.getKey())) - b.append(property.getKey()).append("=").append(property.getValue()).append(" (value from query profile)<br/>\n"); + b.append(property.getKey()).append("=").append(property.getValue()).append(" (value from query profile)\n"); } } diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 1c58081e4f1..0981c6e8dad 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -353,9 +353,7 @@ public class SearchHandler extends LoggingRequestHandler { } } - /** - * For internal use only - */ + /** For internal use only */ public Renderer<Result> getRendererCopy(ComponentSpecification spec) { Renderer<Result> renderer = executionFactory.rendererRegistry().getRenderer(spec); return perRenderingCopy(renderer); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java index 00c7cf98b47..0059b761734 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java @@ -15,7 +15,7 @@ import java.util.Map; public class DimensionBinding { /** The dimensions of this */ - private List<String> dimensions=null; + private List<String> dimensions; /** The values matching those dimensions */ private DimensionValues values; @@ -24,30 +24,32 @@ public class DimensionBinding { private Map<String,String> context; public static final DimensionBinding nullBinding = - new DimensionBinding(Collections.<String>unmodifiableList(Collections.<String>emptyList()), DimensionValues.empty, null); + new DimensionBinding(Collections.unmodifiableList(Collections.emptyList()), DimensionValues.empty, null); public static final DimensionBinding invalidBinding = - new DimensionBinding(Collections.<String>unmodifiableList(Collections.<String>emptyList()), DimensionValues.empty, null); + new DimensionBinding(Collections.unmodifiableList(Collections.emptyList()), DimensionValues.empty, null); /** Whether the value array contains only nulls */ private boolean containsAllNulls; /** Creates a binding from a variant and a context. Any of the arguments may be null. */ public static DimensionBinding createFrom(List<String> dimensions, Map<String,String> context) { - if (dimensions==null || dimensions.size()==0) { - if (context==null) return nullBinding; - if (dimensions==null) return new DimensionBinding(null,DimensionValues.empty,context); // Null, but must preserve context + if (dimensions == null || dimensions.size() == 0) { + if (context == null) return nullBinding; + if (dimensions == null) return new DimensionBinding(null, DimensionValues.empty, context); // Null, but must preserve context } - return new DimensionBinding(dimensions,extractDimensionValues(dimensions,context),context); + return new DimensionBinding(dimensions, extractDimensionValues(dimensions, context), context); } /** Creates a binding from a variant and a context. Any of the arguments may be null. */ public static DimensionBinding createFrom(List<String> dimensions, DimensionValues dimensionValues) { - if (dimensionValues==null || dimensionValues==DimensionValues.empty) return nullBinding; - if (dimensions==null) return new DimensionBinding(null,dimensionValues,null); // Null, but preserve raw material for creating a context later (in createFor) + if (dimensionValues==null || dimensionValues == DimensionValues.empty) return nullBinding; - return new DimensionBinding(dimensions,dimensionValues,null); + // If null, preserve raw material for creating a context later (in createFor) + if (dimensions == null) return new DimensionBinding(null, dimensionValues, null); + + return new DimensionBinding(dimensions, dimensionValues, null); } /** Returns a binding for a (possibly) new set of variants. Variants may be null, but not bindings */ diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/DimensionValues.java b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionValues.java index c7c020220dc..1a36f4917d9 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/DimensionValues.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionValues.java @@ -10,12 +10,14 @@ import java.util.Map; * An immutable set of dimension values. * Note that this may contain more or fewer values than needed given a set of dimensions. * Any missing values are treated as null. + * + * @author bratseth */ public class DimensionValues implements Comparable<DimensionValues> { private final String[] values; - public static final DimensionValues empty=new DimensionValues(new String[] {}); + public static final DimensionValues empty = new DimensionValues(new String[] {}); public static DimensionValues createFrom(String[] values) { if (values==null || values.length==0 || containsAllNulls(values)) return empty; @@ -27,7 +29,7 @@ public class DimensionValues implements Comparable<DimensionValues> { * the right size, and where no copying is done. * * @param values the dimension values. This need not be normalized to the right size. - * The input array is copied by this. + * The input array is copied by this. */ private DimensionValues(String[] values) { if (values==null) throw new NullPointerException("Dimension values cannot be null"); @@ -115,26 +117,25 @@ public class DimensionValues implements Comparable<DimensionValues> { } public Map<String,String> asContext(List<String> dimensions) { - Map<String,String> context=new HashMap<>(); - if (dimensions==null) return context; - for (int i=0; i<dimensions.size(); i++) { - context.put(dimensions.get(i),get(i)); - } + Map<String,String> context = new HashMap<>(); + if (dimensions == null) return context; + for (int i = 0; i < dimensions.size(); i++) + context.put(dimensions.get(i), get(i)); return context; } - /** Returns the string at the given index, <b>or null if it has no value at this index.</b> */ + /** Returns the string at the given index, or null if it has no value at this index */ public String get(int index) { - if (index>=values.length) return null; + if (index >= values.length) return null; return values[index]; } /** Returns the number of values in this (some of which may be null) */ public int size() { return values.length; } - /** Returns copy of the values in this in an array */ + /** Returns a copy of the values in this in an array */ public String[] getValues() { - return Arrays.copyOf(values,values.length); + return Arrays.copyOf(values, values.length); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java index 826c9949bcf..accef7ba154 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java @@ -10,6 +10,7 @@ import com.yahoo.search.query.profile.types.QueryProfileType; import java.util.Collection; import java.util.HashSet; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; @@ -40,14 +41,12 @@ public class QueryProfileCompiler { // Resolve values for each existing variant and combine into a single data structure Set<DimensionBindingForPath> variants = collectVariants(CompoundName.empty, in, DimensionBinding.nullBinding); variants.add(new DimensionBindingForPath(DimensionBinding.nullBinding, CompoundName.empty)); // if this contains no variants - if (log.isLoggable(Level.FINE)) - log.fine("Compiling " + in.toString() + " having " + variants.size() + " variants"); - int i = 0; + log.fine(() -> "Compiling " + in.toString() + " having " + variants.size() + " variants"); for (DimensionBindingForPath variant : variants) { - if (log.isLoggable(Level.FINER)) - log.finer(" Compiling variant " + i++ + ": " + variant); - for (Map.Entry<String, Object> entry : in.listValues(variant.path(), variant.binding().getContext(), null).entrySet()) + log.finer(() -> " Compiling variant " + variant); + for (Map.Entry<String, Object> entry : in.listValues(variant.path(), variant.binding().getContext(), null).entrySet()) { values.put(variant.path().append(entry.getKey()), variant.binding(), entry.getValue()); + } for (Map.Entry<CompoundName, QueryProfileType> entry : in.listTypes(variant.path(), variant.binding().getContext()).entrySet()) types.put(variant.path().append(entry.getKey()), variant.binding(), entry.getValue()); for (CompoundName reference : in.listReferences(variant.path(), variant.binding().getContext())) @@ -62,7 +61,7 @@ public class QueryProfileCompiler { } /** - * Returns all the unique combinations of dimension values which have values set reachable from this profile. + * Returns all the unique combinations of dimension values which have values reachable from this profile. * * @param profile the profile we are collecting the variants of * @param currentVariant the variant we must have to arrive at this point in the query profile graph @@ -74,15 +73,59 @@ public class QueryProfileCompiler { if (profile instanceof BackedOverridableQueryProfile) variants.addAll(collectVariantsInThis(path, ((BackedOverridableQueryProfile) profile).getBacking(), currentVariant)); - Set<DimensionBindingForPath> parentVariants = new HashSet<>(); + Set<DimensionBindingForPath> parentVariants; for (QueryProfile inheritedProfile : profile.inherited()) { parentVariants = collectVariants(path, inheritedProfile, currentVariant); variants.addAll(parentVariants); variants.addAll(combined(variants, parentVariants)); // parents and children may have different variant dimensions } + + variants.addAll(leftExpanded(variants)); return variants; } + /** + * For variants which are underspecified on the left we must explicitly resolve each possible combination + * of actual left-side values. + * + * I.e if we have the variants [-,b=b1], [a=a1,-], [a=a2,-], + * this returns the variants [a=a1,b=b1], [a=a2,b=b1] + * + * This is necessary because left-specified values takes precedence, such that resolving [a=a1,b=b1] would + * lead us to the compiled profile [a=a1,-], which may contain default values for properties where + * we should have preferred variant values in [-,b=b1]. + */ + private static Set<DimensionBindingForPath> leftExpanded(Set<DimensionBindingForPath> variants) { + Set<DimensionBindingForPath> expanded = new HashSet<>(); + for (var variant : variants) { + if (hasLeftWildcard(variant.binding())) + expanded.addAll(leftExpanded(variant, variants)); + } + return expanded; + } + + private static boolean hasLeftWildcard(DimensionBinding variant) { + for (int i = 0; i < variant.getValues().size() - 1; i++) { // -1 to not check the rightmost + if (variant.getValues().get(i) == null) + return true; + } + return false; + } + + private static Set<DimensionBindingForPath> leftExpanded(DimensionBindingForPath variantToExpand, + Set<DimensionBindingForPath> variants) { + Set<DimensionBindingForPath> expanded = new HashSet<>(); + for (var variant : variants) { + if ( ! variantToExpand.path().equals(variant.path())) continue; + + DimensionBinding combined = variantToExpand.binding().combineWith(variant.binding); + if ( ! combined.isInvalid() ) + expanded.add(new DimensionBindingForPath(combined, variantToExpand.path())); + } + return expanded; + } + + /** Generates a set of all the (legal) combinations of the variants in the given sets */ private static Set<DimensionBindingForPath> combined(Set<DimensionBindingForPath> v1s, Set<DimensionBindingForPath> v2s) { diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java index 7ac73947905..d6e93701ca1 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java @@ -23,7 +23,7 @@ import java.util.regex.Pattern; */ public class CompiledQueryProfile extends AbstractComponent implements Cloneable { - private static final Pattern namePattern=Pattern.compile("[$a-zA-Z_/][-$a-zA-Z0-9_/()]*"); + private static final Pattern namePattern = Pattern.compile("[$a-zA-Z_/][-$a-zA-Z0-9_/()]*"); private final CompiledQueryProfileRegistry registry; @@ -102,15 +102,15 @@ public class CompiledQueryProfile extends AbstractComponent implements Cloneable * For example, if {a.d => "a.d-value" ,a.e => "a.e-value", b.d => "b.d-value", then calling listValues("a") * will return {"d" => "a.d-value","e" => "a.e-value"} */ - public final Map<String, Object> listValues(final CompoundName prefix) { return listValues(prefix, Collections.<String,String>emptyMap()); } - public final Map<String, Object> listValues(final String prefix) { return listValues(new CompoundName(prefix)); } + public final Map<String, Object> listValues(CompoundName prefix) { return listValues(prefix, Collections.<String,String>emptyMap()); } + public final Map<String, Object> listValues(String prefix) { return listValues(new CompoundName(prefix)); } /** * Return all objects that start with the given prefix path. Use "" to list all. * <p> * For example, if {a.d => "a.d-value" ,a.e => "a.e-value", b.d => "b.d-value", then calling listValues("a") * will return {"d" => "a.d-value","e" => "a.e-value"} */ - public final Map<String, Object> listValues(final String prefix,Map<String, String> context) { + public final Map<String, Object> listValues(String prefix, Map<String, String> context) { return listValues(new CompoundName(prefix), context); } /** @@ -119,7 +119,7 @@ public class CompiledQueryProfile extends AbstractComponent implements Cloneable * For example, if {a.d => "a.d-value" ,a.e => "a.e-value", b.d => "b.d-value", then calling listValues("a") * will return {"d" => "a.d-value","e" => "a.e-value"} */ - public final Map<String, Object> listValues(final CompoundName prefix,Map<String, String> context) { + public final Map<String, Object> listValues(CompoundName prefix, Map<String, String> context) { return listValues(prefix, context, null); } /** diff --git a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java index 78868f37718..7b3327c04af 100644 --- a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java @@ -35,9 +35,6 @@ import static org.junit.Assume.assumeTrue; */ public class QueryTestCase { - /** - * Basic test - */ @Test public void testSimpleQueryParsing () { Query q = newQuery("/search?query=foobar&offset=10&hits=20"); @@ -46,9 +43,6 @@ public class QueryTestCase { assertEquals(20,q.getHits()); } - /** - * Not quite so basic - */ @Test public void testAdvancedQueryParsing () { Query q = newQuery("/search?query=fOObar and kanoo&offset=10&hits=20&filter=-foo +bar&type=adv&suggestonly=true"); @@ -58,9 +52,6 @@ public class QueryTestCase { assertEquals(true, q.properties().getBoolean("suggestonly", false)); } - /** - * Not quite so basic - */ @Test public void testAnyQueryParsing () { Query q = newQuery("/search?query=foobar and kanoo&offset=10&hits=10&type=any&suggestonly=true&filter=-fast.type:offensive&encoding=latin1"); @@ -71,9 +62,6 @@ public class QueryTestCase { assertEquals("latin1",q.getModel().getEncoding()); } - /** - * A long string - */ @Test public void testLongQueryParsing() { Query q = newQuery("/p13n?query=news" @@ -108,18 +96,12 @@ public class QueryTestCase { } } - /** - * Test UTF-8 decoding - */ @Test public void testUtf8Decoding() { Query q = new Query("/?query=beyonc%C3%A9"); assertEquals("beyonc\u00e9",((WordItem) q.getModel().getQueryTree().getRoot()).getWord()); } - /** - * Check sortspec "parsing" is correct. - */ @Test public void testSortSpec() { Query q = newQuery("?query=test&sortspec=+a -b c +[rank]"); @@ -321,7 +303,7 @@ public class QueryTestCase { /** Test a vertical specific patch, see Tokenizer */ @Test @Ignore - public void testOtherharactersParsing() { + public void testOtherCharactersParsing() { Query query=newQuery(com.yahoo.search.test.QueryTestCase.httpEncode("?query=\u3007\u2e80\u2eff\u2ed0")); assertEquals(Character.UnicodeBlock.CJK_SYMBOLS_AND_PUNCTUATION, Character.UnicodeBlock.of(query.getModel().getQueryTree().getRoot().toString().charAt(0))); diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java index 9f890b249d2..3c276157f72 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java @@ -9,6 +9,7 @@ import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileProperties; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; +import com.yahoo.yolean.trace.TraceNode; import org.junit.Test; import java.util.Arrays; @@ -592,4 +593,23 @@ public class QueryProfileTestCase { assertTrue(q.properties().getBoolean("clustering.timeline", false)); } + @Test + public void testSubstitutionInTrace() { + QueryProfile profile = new QueryProfile("test"); + profile.set("property", "%{foo}", null); + CompiledQueryProfile cProfile = profile.compile(null); + + Query query = new Query("?foo=value&tracelevel=4", cProfile); + assertEquals("value", query.properties().get("property")); + assertTrue(traceContains("foo=value", query)); + } + + // NB: NOT RECURSIVE + private boolean traceContains(String string, Query query) { + for (TraceNode node : query.getContext(true).getTrace().traceNode().children()) + if (node.payload().toString().contains(string)) + return true; + return false; + } + } diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java index 46dbac859a2..d9bf4a1db97 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java @@ -9,6 +9,7 @@ import com.yahoo.search.query.profile.BackedOverridableQueryProfile; import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileProperties; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; +import com.yahoo.yolean.trace.TraceNode; import org.junit.Test; import java.util.Arrays; @@ -18,6 +19,7 @@ import java.util.Map; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * @author bratseth @@ -68,6 +70,20 @@ public class QueryProfileVariantsTestCase { } @Test + public void testMultipleMatchingVariantsAndDefault() { + QueryProfile profile = new QueryProfile("test"); + profile.setDimensions(new String[] { "a", "b" }); + profile.set("property1", "property1_default", null); + profile.set("property1", "property1_variantB", new String[] { null, "b1" }, null); + profile.set("property2", "property2_variantA", new String[] { "a1", null }, null); + CompiledQueryProfile cProfile = profile.compile(null); + + Query query = new Query("?a=a1&b=b1&b=b1", cProfile); + assertEquals("property1_variantB", query.properties().get("property1")); + assertEquals("property2_variantA", query.properties().get("property2")); + } + + @Test public void testInheritedVariants() { QueryProfile parent = new QueryProfile("parent"); parent.setDimensions(new String[] { "parentDim" }); @@ -149,13 +165,13 @@ public class QueryProfileVariantsTestCase { @Test public void testVariantsOfExplicitCompound() { QueryProfile a1 = new QueryProfile("a1"); - a1.set("b","a.b", null); + a1.set("b", "a.b", null); - QueryProfile profile=new QueryProfile("test"); + QueryProfile profile = new QueryProfile("test"); profile.setDimensions(new String[] {"x"}); - profile.set("a",a1, null); - profile.set("a.b","a.b.x1",new String[] {"x1"}, null); - profile.set("a.b","a.b.x2",new String[] {"x2"}, null); + profile.set("a", a1, null); + profile.set("a.b", "a.b.x1", new String[] {"x1"}, null); + profile.set("a.b", "a.b.x2", new String[] {"x2"}, null); CompiledQueryProfile cprofile = profile.compile(null); @@ -254,53 +270,53 @@ public class QueryProfileVariantsTestCase { @Test public void testVariantNotInBase() { - QueryProfile test=new QueryProfile("test"); + QueryProfile test = new QueryProfile("test"); test.setDimensions(new String[] {"x"}); - test.set("InX1Only","x1",new String[] {"x1"}, null); + test.set("InX1Only", "x1", new String[] { "x1" }, null); CompiledQueryProfile ctest = test.compile(null); - assertEquals("x1",ctest.get("InX1Only", toMap("x=x1"))); - assertEquals(null,ctest.get("InX1Only", toMap("x=x2"))); - assertEquals(null,ctest.get("InX1Only")); + assertEquals("x1", ctest.get("InX1Only", toMap("x=x1"))); + assertEquals(null, ctest.get("InX1Only", toMap("x=x2"))); + assertEquals(null, ctest.get("InX1Only")); } @Test public void testVariantNotInBaseSpaceVariantValue() { - QueryProfile test=new QueryProfile("test"); - test.setDimensions(new String[] {"x"}); - test.set("InX1Only","x1",new String[] {"x 1"}, null); + QueryProfile test = new QueryProfile("test"); + test.setDimensions(new String[] { "x" }); + test.set("InX1Only", "x1", new String[] { "x 1" }, null); CompiledQueryProfile ctest = test.compile(null); - assertEquals("x1",ctest.get("InX1Only", toMap("x=x 1"))); - assertEquals(null,ctest.get("InX1Only", toMap("x=x 2"))); - assertEquals(null,ctest.get("InX1Only")); + assertEquals("x1", ctest.get("InX1Only", toMap("x=x 1"))); + assertEquals(null, ctest.get("InX1Only", toMap("x=x 2"))); + assertEquals(null, ctest.get("InX1Only")); } @Test public void testDimensionsInSuperType() { - QueryProfile parent=new QueryProfile("parent"); - parent.setDimensions(new String[] {"x","y"}); - QueryProfile child=new QueryProfile("child"); + QueryProfile parent = new QueryProfile("parent"); + parent.setDimensions(new String[] {"x", "y"}); + QueryProfile child = new QueryProfile("child"); child.addInherited(parent); - child.set("a","a.default", null); - child.set("a","a.x1.y1",new String[] {"x1","y1"}, null); - child.set("a","a.x1.y2",new String[] {"x1","y2"}, null); + child.set("a", "a.default", null); + child.set("a", "a.x1.y1", new String[] {"x1", "y1"}, null); + child.set("a", "a.x1.y2", new String[] {"x1", "y2"}, null); CompiledQueryProfile cchild = child.compile(null); assertEquals("a.default", cchild.get("a")); - assertEquals("a.x1.y1", cchild.get("a", toMap("x=x1","y=y1"))); - assertEquals("a.x1.y2", cchild.get("a", toMap("x=x1","y=y2"))); + assertEquals("a.x1.y1", cchild.get("a", toMap("x=x1", "y=y1"))); + assertEquals("a.x1.y2", cchild.get("a", toMap("x=x1", "y=y2"))); } @Test public void testDimensionsInSuperTypeRuntime() { - QueryProfile parent=new QueryProfile("parent"); - parent.setDimensions(new String[] {"x","y"}); - QueryProfile child=new QueryProfile("child"); + QueryProfile parent = new QueryProfile("parent"); + parent.setDimensions(new String[] { "x", "y" }); + QueryProfile child = new QueryProfile("child"); child.addInherited(parent); - child.set("a","a.default", null); + child.set("a", "a.default", null); child.set("a", "a.x1.y1", new String[]{"x1", "y1"}, null); child.set("a", "a.x1.y2", new String[]{"x1", "y2"}, null); Properties overridable=new QueryProfileProperties(child.compile(null)); @@ -1126,15 +1142,15 @@ public class QueryProfileVariantsTestCase { } public static Map<String,String> toMap(QueryProfile profile, String[] dimensionValues) { - Map<String,String> context=new HashMap<>(); + Map<String, String> context = new HashMap<>(); List<String> dimensions; - if (profile.getVariants()!=null) - dimensions=profile.getVariants().getDimensions(); + if (profile.getVariants() != null) + dimensions = profile.getVariants().getDimensions(); else - dimensions=((BackedOverridableQueryProfile)profile).getBacking().getVariants().getDimensions(); + dimensions = ((BackedOverridableQueryProfile)profile).getBacking().getVariants().getDimensions(); - for (int i=0; i<dimensionValues.length; i++) - context.put(dimensions.get(i),dimensionValues[i]); // Lookup dim. names to ease test... + for (int i = 0; i < dimensionValues.length; i++) + context.put(dimensions.get(i), dimensionValues[i]); // Lookup dim. names to ease test... return context; } diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 00a17f963c6..1d25c2301e2 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -980,6 +980,12 @@ public class YqlParserTestCase { assertUrlQuery("urlfield", yql, false, false, false); } + @Test + public void testReservedWordInSource() { + parse("select * from sources like where text contains \"test\";"); + // success: parsed without exception + } + private void assertUrlQuery(String field, Query query, boolean startAnchor, boolean endAnchor, boolean endAnchorIsDefault) { boolean startAnchorIsDefault = false; // Always diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java index 5f8e2f886a2..9881002659d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java @@ -26,12 +26,14 @@ public enum JobType { systemTest ("system-test", Map.of(main , ZoneId.from("test", "us-east-1"), cd , ZoneId.from("test", "cd-us-central-1"), - PublicCd, ZoneId.from("test", "aws-us-east-1c"))), + PublicCd, ZoneId.from("test", "aws-us-east-1c"), + Public , ZoneId.from("test", "aws-us-east-1c"))), stagingTest ("staging-test", Map.of(main , ZoneId.from("staging", "us-east-3"), cd , ZoneId.from("staging", "cd-us-central-1"), - PublicCd, ZoneId.from("staging", "aws-us-east-1c"))), + PublicCd, ZoneId.from("staging", "aws-us-east-1c"), + Public , ZoneId.from("staging", "aws-us-east-1c"))), productionUsEast3 ("production-us-east-3", Map.of(main, ZoneId.from("prod" , "us-east-3"))), @@ -58,7 +60,8 @@ public enum JobType { Map.of(main, ZoneId.from("prod" , "aws-us-east-1a"))), productionAwsUsEast1c ("production-aws-us-east-1c", - Map.of(PublicCd, ZoneId.from("prod", "aws-us-east-1c"))), + Map.of(PublicCd, ZoneId.from("prod", "aws-us-east-1c"), + Public, ZoneId.from("prod", "aws-us-east-1c"))), productionAwsUsWest2a ("production-aws-us-west-2a", Map.of(main, ZoneId.from("prod" , "aws-us-west-2a"))), diff --git a/dist/vespa.spec b/dist/vespa.spec index 2d6e8cbfd13..b55d7e4125a 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -57,13 +57,13 @@ BuildRequires: gtest-devel BuildRequires: gmock-devel %endif %if 0%{?fc31} -BuildRequires: llvm-devel >= 8.0.0 +BuildRequires: llvm-devel >= 9.0.0 BuildRequires: boost-devel >= 1.69 BuildRequires: gtest-devel BuildRequires: gmock-devel %endif %if 0%{?fc32} -BuildRequires: llvm-devel >= 8.0.0 +BuildRequires: llvm-devel >= 9.0.0 BuildRequires: boost-devel >= 1.69 BuildRequires: gtest-devel BuildRequires: gmock-devel @@ -133,12 +133,12 @@ Requires: llvm-libs >= 8.0.0 %define _vespa_llvm_version 8 %endif %if 0%{?fc31} -Requires: llvm-libs >= 8.0.0 -%define _vespa_llvm_version 8 +Requires: llvm-libs >= 9.0.0 +%define _vespa_llvm_version 9 %endif %if 0%{?fc32} -Requires: llvm-libs >= 8.0.0 -%define _vespa_llvm_version 8 +Requires: llvm-libs >= 9.0.0 +%define _vespa_llvm_version 9 %endif %define _extra_link_directory %{_vespa_deps_prefix}/lib64 %define _extra_include_directory %{_vespa_deps_prefix}/include diff --git a/docker/build-vespa.sh b/docker/build-vespa.sh index 48f816051a4..4570bdf5b5a 100755 --- a/docker/build-vespa.sh +++ b/docker/build-vespa.sh @@ -11,7 +11,7 @@ DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) cd $DIR VESPA_VERSION=$1 -DOCKER_IMAGE="centos:latest" +DOCKER_IMAGE="centos:7" docker pull ${DOCKER_IMAGE} diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocprocThreadManager.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocprocThreadManager.java index 4fb3c8f0913..6fd4beac056 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocprocThreadManager.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocprocThreadManager.java @@ -1,11 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.docproc.jdisc; -import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.document.DocumentUtil; -import com.yahoo.jdisc.Metric; import com.yahoo.log.LogLevel; -import com.yahoo.statistics.*; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Logger; @@ -22,12 +19,6 @@ class DocprocThreadManager { private final AtomicLong bytesFinished = new AtomicLong(0); DocprocThreadManager(double maxConcurrentFactor, double documentExpansionFactor, int containerCoreMemoryMb) { - this(maxConcurrentFactor, documentExpansionFactor, containerCoreMemoryMb, Statistics.nullImplementation, - new NullMetric()); - } - - DocprocThreadManager(double maxConcurrentFactor, double documentExpansionFactor, int containerCoreMemoryMb, - Statistics statistics, Metric metric) { this((long) (((double) DocumentUtil.calculateMaxPendingSize(maxConcurrentFactor, documentExpansionFactor, containerCoreMemoryMb)) * maxConcurrentFactor)); } diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocprocThreadPoolExecutor.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocprocThreadPoolExecutor.java index 23ab73c7f1d..e1a902c8d5c 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocprocThreadPoolExecutor.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocprocThreadPoolExecutor.java @@ -20,7 +20,7 @@ public class DocprocThreadPoolExecutor extends ThreadPoolExecutor { public DocprocThreadPoolExecutor(int maxNumThreads, BlockingQueue<Runnable> queue, DocprocThreadManager threadMgr) { super((maxNumThreads > 0) ? maxNumThreads : Runtime.getRuntime().availableProcessors(), - (maxNumThreads > 0) ? maxNumThreads : 8192, + (maxNumThreads > 0) ? maxNumThreads : 2048, 1, TimeUnit.SECONDS, queue, new DaemonThreadFactory("docproc-")); diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java index ec74fb9246d..5b7b9d85a91 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java @@ -19,7 +19,6 @@ import com.yahoo.docproc.jdisc.messagebus.MbusRequestContext; import com.yahoo.docproc.proxy.SchemaMap; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.documentapi.ThroughputLimitQueue; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Request; import com.yahoo.jdisc.handler.AbstractRequestHandler; @@ -33,7 +32,6 @@ import com.yahoo.statistics.Statistics; import java.util.TimerTask; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.PriorityBlockingQueue; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.SynchronousQueue; @@ -96,29 +94,20 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { DocumentProcessingHandlerParameters params) { this(docprocServiceRegistry, documentProcessorComponentRegistry, docFactoryRegistry, new DocprocThreadPoolExecutor(params.getMaxNumThreads(), - chooseQueueType(params), + chooseQueueType(params.getMaxNumThreads()), new DocprocThreadManager(params.getMaxConcurrentFactor(), params.getDocumentExpansionFactor(), - params.getContainerCoreMemoryMb(), - params.getStatisticsManager(), - params.getMetric())), + params.getContainerCoreMemoryMb())), params.getDocumentTypeManager(), params.getChainsModel(), params.getSchemaMap(), params.getStatisticsManager(), params.getMetric(), params.getContainerDocConfig()); } - private static BlockingQueue<Runnable> chooseQueueType(DocumentProcessingHandlerParameters params) { - if (params.getMaxQueueTimeMs() > 0) { - return new ThroughputLimitQueue<>(params.getMaxQueueTimeMs()); - } - if (params.getMaxQueueTimeMs() == 0) { - return new PriorityBlockingQueue<>(); // Probably no need to bound this queue, see bug #4254537 - } - if (params.getMaxNumThreads() > 0) { - return new LinkedBlockingQueue<>(); - } - return new SynchronousQueue<>(); + private static BlockingQueue<Runnable> chooseQueueType(int maxNumThreads) { + return (maxNumThreads > 0) + ? new LinkedBlockingQueue<>() + : new SynchronousQueue<>(); } @Inject @@ -138,7 +127,6 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { .setMaxConcurrentFactor(containerMbusConfig.maxConcurrentFactor()) .setDocumentExpansionFactor(containerMbusConfig.documentExpansionFactor()) .setContainerCoreMemoryMb(containerMbusConfig.containerCoreMemory()) - .setMaxQueueTimeMs(docprocConfig.maxqueuetimems()) .setDocumentTypeManager(new DocumentTypeManager(docManConfig)) .setChainsModel(buildFromConfig(chainsConfig)).setSchemaMap(configureMapping(mappingConfig)) .setStatisticsManager(manager) diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerParameters.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerParameters.java index cec943306c8..bf308e39218 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerParameters.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerParameters.java @@ -21,7 +21,6 @@ public class DocumentProcessingHandlerParameters { private double maxConcurrentFactor = 0.2; private double documentExpansionFactor = 20.0; private int containerCoreMemoryMb = 50; - private long maxQueueTimeMs = 0; private DocumentTypeManager documentTypeManager = null; private ChainsModel chainsModel = null; private SchemaMap schemaMap = null; @@ -86,21 +85,6 @@ public class DocumentProcessingHandlerParameters { } /** - * Returns the maximum time (in milliseconds) that a document may stay in the input queue. The default value - * of 0 disables this functionality. - * - * @return the maximum time (in milliseconds) that a document may stay in the input queue. - */ - public long getMaxQueueTimeMs() { - return maxQueueTimeMs; - } - - public DocumentProcessingHandlerParameters setMaxQueueTimeMs(long maxQueueTimeMs) { - this.maxQueueTimeMs = maxQueueTimeMs; - return this; - } - - /** * Returns the maximum number of thread that the thread pool will ever attempt to run simultaneously. * * @return the maximum number of thread that the thread pool will ever attempt to run simultaneously. diff --git a/docproc/src/main/resources/configdefinitions/docproc.def b/docproc/src/main/resources/configdefinitions/docproc.def index 645fbcce1b7..53f5c2de21f 100644 --- a/docproc/src/main/resources/configdefinitions/docproc.def +++ b/docproc/src/main/resources/configdefinitions/docproc.def @@ -6,7 +6,9 @@ namespace=config.docproc # 0 Gives a PriorityQueue # Negative values gives a SynchronousQueue if numthreads <= 0 (which increases thread pool size as required), # otherwise a LinkedBlockingQueue +# Depreacated and a noop maxqueuetimems int default=-1 # The number of threads in the DocprocHandler worker thread pool +# Default is number of cpu's, but is dynamic up to 2048 numthreads int default=-1 diff --git a/documentapi/abi-spec.json b/documentapi/abi-spec.json index dc8daf737e4..80fccbd9161 100644 --- a/documentapi/abi-spec.json +++ b/documentapi/abi-spec.json @@ -524,28 +524,6 @@ ], "fields": [] }, - "com.yahoo.documentapi.ThroughputLimitQueue": { - "superClass": "java.util.concurrent.LinkedBlockingQueue", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(long)", - "public void <init>(com.yahoo.concurrent.Timer, long)", - "public boolean add(java.lang.Object)", - "public boolean offer(java.lang.Object)", - "public java.lang.Object poll()", - "public void put(java.lang.Object)", - "public boolean offer(java.lang.Object, long, java.util.concurrent.TimeUnit)", - "public java.lang.Object take()", - "public java.lang.Object poll(long, java.util.concurrent.TimeUnit)", - "public int capacity()", - "public int remainingCapacity()", - "public boolean addAll(java.util.Collection)" - ], - "fields": [] - }, "com.yahoo.documentapi.UpdateResponse": { "superClass": "com.yahoo.documentapi.Response", "interfaces": [], diff --git a/documentapi/src/main/java/com/yahoo/documentapi/ThroughputLimitQueue.java b/documentapi/src/main/java/com/yahoo/documentapi/ThroughputLimitQueue.java deleted file mode 100644 index 579cdaced04..00000000000 --- a/documentapi/src/main/java/com/yahoo/documentapi/ThroughputLimitQueue.java +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi; - -import com.yahoo.concurrent.SystemTimer; -import com.yahoo.concurrent.Timer; - -import java.util.Collection; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; - -/** - * Queue that limits it's size based on the throughput. Allows the queue to keep a certain number of - * seconds of processing in its queue. - */ -public class ThroughputLimitQueue<M> extends LinkedBlockingQueue<M> { - private static Logger log = Logger.getLogger(ThroughputLimitQueue.class.getName()); - - double averageWaitTime = 0; - long maxWaitTime = 0; - long startTime; - int capacity = 2; - Timer timer; - - /** - * Creates a new queue. - * - * @param queueSizeInMs The maximum time we wish to have objects waiting in the queue. - */ - public ThroughputLimitQueue(long queueSizeInMs) { - this(SystemTimer.INSTANCE, queueSizeInMs); - } - - /** - * Creates a new queue. Used for unit testing. - * - * @param t Used to measure time spent in the queue. Subclass for unit testing, or use SystemTimer for regular use. - * @param queueSizeInMs The maximum time we wish to have objects waiting in the queue. - */ - public ThroughputLimitQueue(Timer t, long queueSizeInMs) { - maxWaitTime = queueSizeInMs; - timer = t; - } - - // Doc inherited from BlockingQueue - public boolean add(M m) { - if (!offer(m)) { - throw new IllegalStateException("Queue full"); - } - return true; - } - - // Doc inherited from BlockingQueue - public boolean offer(M m) { - return remainingCapacity() > 0 && super.offer(m); - } - - /** - * Calculates the average waiting time and readjusts the queue capacity. - * - * @param m The last message that was read from queue, if any. - * @return Returns m. - */ - private M calculateAverage(M m) { - if (m == null) { - startTime = 0; - return null; - } - - if (startTime != 0) { - long waitTime = timer.milliTime() - startTime; - - if (averageWaitTime == 0) { - averageWaitTime = waitTime; - } else { - averageWaitTime = 0.99 * averageWaitTime + 0.01 * waitTime; - } - - int newCapacity = Math.max(2, (int)Math.round(maxWaitTime / averageWaitTime)); - if (newCapacity != capacity) { - log.fine("Capacity of throughput queue changed from " + capacity + " to " + newCapacity); - capacity = newCapacity; - } - } - - if (!isEmpty()) { - startTime = timer.milliTime(); - } else { - startTime = 0; - } - - return m; - } - - // Doc inherited from BlockingQueue - public M poll() { - return calculateAverage(super.poll()); - } - - // Doc inherited from BlockingQueue - public void put(M m) throws InterruptedException { - offer(m, Long.MAX_VALUE, TimeUnit.SECONDS); - } - - // Doc inherited from BlockingQueue - public boolean offer(M m, long l, TimeUnit timeUnit) throws InterruptedException { - long timeWaited = 0; - while (timeWaited < timeUnit.toMillis(l)) { - if (offer(m)) { - return true; - } - - Thread.sleep(10); - timeWaited += 10; - } - - return false; - } - - // Doc inherited from BlockingQueue - public M take() throws InterruptedException { - return poll(Long.MAX_VALUE, TimeUnit.SECONDS); - } - - // Doc inherited from BlockingQueue - public M poll(long l, TimeUnit timeUnit) throws InterruptedException { - long timeWaited = 0; - while (timeWaited < timeUnit.toMillis(l)) { - M elem = poll(); - if (elem != null) { - return elem; - } - - Thread.sleep(10); - timeWaited += 10; - } - - return null; - } - - /** - * @return Returns the maximum capacity of the queue - */ - public int capacity() { - return capacity; - } - - // Doc inherited from BlockingQueue - public int remainingCapacity() { - int sz = capacity - size(); - return (sz > 0) ? sz : 0; - } - - // Doc inherited from BlockingQueue - public boolean addAll(Collection<? extends M> ms) { - for (M m : ms) { - if (!offer(m)) { - return false; - } - } - - return true; - } -} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/ThroughputLimitQueueTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/ThroughputLimitQueueTestCase.java deleted file mode 100644 index 75e98bc895a..00000000000 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/test/ThroughputLimitQueueTestCase.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.test; - -import com.yahoo.documentapi.ThroughputLimitQueue; -import com.yahoo.concurrent.Timer; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author thomasg - */ -public class ThroughputLimitQueueTestCase { - - class TestTimer implements Timer { - public long milliTime = 0; - - public long milliTime() { - return milliTime; - } - } - - @Test - public void testCapacity() { - TestTimer t = new TestTimer(); - t.milliTime = 10; - ThroughputLimitQueue<Object> q = new ThroughputLimitQueue<Object>(t, 2000); - - q.add(new Object()); - q.add(new Object()); - q.remove(); - t.milliTime += 10; - q.remove(); - - assertEquals(200, q.capacity()); - - for (int i = 0; i < 1000; i++) { - q.add(new Object()); - q.add(new Object()); - t.milliTime += 100; - q.remove(); - t.milliTime += 100; - q.remove(); - } - - assertEquals(20, q.capacity()); - } - -} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 54c815b48a3..54531aeb0d7 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -80,6 +80,12 @@ public class Flags { "Takes effect on next node agent tick. Change is orchestrated, but does NOT require container restart", HOSTNAME, APPLICATION_ID); + public static final UnboundBooleanFlag INCLUDE_SIS_IN_TRUSTSTORE = defineFeatureFlag( + "include-sis-in-truststore", false, + "Whether to use the trust store backed by Athenz and Service Identity certificates.", + "Takes effect on next tick, but may get throttled due to orchestration.", + HOSTNAME); + public static final UnboundStringFlag TLS_INSECURE_MIXED_MODE = defineStringFlag( "tls-insecure-mixed-mode", "tls_client_mixed_server", "TLS insecure mixed mode. Allowed values: ['plaintext_client_mixed_server', 'tls_client_mixed_server', 'tls_client_tls_server']", diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/OsgiFramework.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/OsgiFramework.java index 5abc95cc610..f5346a21a4f 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/OsgiFramework.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/OsgiFramework.java @@ -8,9 +8,9 @@ import org.osgi.framework.BundleException; import java.util.List; /** - * <p>This is an abstraction of the OSGi framework that hides the actual implementation details. If you need access to + * This is an abstraction of the OSGi framework that hides the actual implementation details. If you need access to * this interface, simply inject it into your Application. In most cases, however, you are better of injecting a - * {@link BundleInstaller} since that provides common convenience methods.</p> + * {@link BundleInstaller} since that provides common convenience methods. * * @author Simon Thoresen Hult */ @@ -24,7 +24,7 @@ public interface OsgiFramework { * argument.</p> * * <p><b>NOTE:</b> When this method installs more than one bundle, <em>AND</em> one of those bundles throw an - * exception during installation, the bundles installed prior to throwing the expcetion will remain installed. To + * exception during installation, the bundles installed prior to throwing the exception will remain installed. To * enable the caller to recover from such a situation, this method wraps any thrown exception within a {@link * BundleInstallationException} that contains the list of successfully installed bundles.</p> * @@ -32,67 +32,66 @@ public interface OsgiFramework { * case of an exception), but that can not be implemented thread-safely since an <code>Application</code> may choose to * install bundles concurrently through any available <code>BundleContext</code>.</p> * - * @param bundleLocation The location identifier of the bundle to install. - * @return The list of Bundle objects installed, the object at index 0 matches the given location. - * @throws BundleInstallationException If the input stream cannot be read, or the installation of a bundle failed, + * @param bundleLocation the location identifier of the bundle to install + * @return the list of Bundle objects installed, the object at index 0 matches the given location + * @throws BundleInstallationException if the input stream cannot be read, or the installation of a bundle failed, * or the caller does not have the appropriate permissions, or the system {@link - * BundleContext} is no longer valid. + * BundleContext} is no longer valid */ List<Bundle> installBundle(String bundleLocation) throws BundleException; /** - * <p>Starts the given {@link Bundle}s. The parameter <code>privileged</code> tells the framework whether or not + * Starts the given {@link Bundle}s. The parameter <code>privileged</code> tells the framework whether or not * privileges are available, and is checked against the {@link OsgiHeader#PRIVILEGED_ACTIVATOR} header of each - * Bundle being started. Any bundle that is a fragment is silently ignored.</p> + * Bundle being started. Any bundle that is a fragment is silently ignored. * - * @param bundles The bundles to start. - * @param privileged Whether or not privileges are available. - * @throws BundleException If a bundle could not be started. This could be because a code dependency could not + * @param bundles the bundles to start + * @param privileged whether or not privileges are available + * @throws BundleException if a bundle could not be started. This could be because a code dependency could not * be resolved or the specified BundleActivator could not be loaded or threw an * exception. - * @throws SecurityException If the caller does not have the appropriate permissions. - * @throws IllegalStateException If this bundle has been uninstalled or this bundle tries to change its own state. + * @throws SecurityException if the caller does not have the appropriate permissions + * @throws IllegalStateException if this bundle has been uninstalled or this bundle tries to change its own state */ void startBundles(List<Bundle> bundles, boolean privileged) throws BundleException; /** - * <p>This method <em>synchronously</em> refreshes all bundles currently loaded. Once this method returns, the - * class loaders of all bundles will reflect on the current set of loaded bundles.</p> + * Synchronously refresh all bundles currently loaded. Once this method returns, the + * class loaders of all bundles will reflect on the current set of loaded bundles. */ void refreshPackages(); /** - * <p>Returns the BundleContext of this framework's system bundle. The returned BundleContext can be used by the + * Returns the BundleContext of this framework's system bundle. The returned BundleContext can be used by the * caller to act on behalf of this bundle. This method may return <code>null</code> if it has no valid - * BundleContext.</p> + * BundleContext. * - * @return A <code>BundleContext</code> for the system bundle, or <code>null</code>. - * @throws SecurityException If the caller does not have the appropriate permissions. + * @return a <code>BundleContext</code> for the system bundle, or <code>null</code> + * @throws SecurityException if the caller does not have the appropriate permissions */ BundleContext bundleContext(); /** - * <p>Returns an iterable collection of all installed bundles. This method returns a list of all bundles installed + * Returns an iterable collection of all installed bundles. This method returns a list of all bundles installed * in the OSGi environment at the time of the call to this method. However, since the OsgiFramework is a very - * dynamic environment, bundles can be installed or uninstalled at anytime.</p> + * dynamic environment, bundles can be installed or uninstalled at anytime. * - * @return An iterable collection of Bundle objects, one object per installed bundle. + * @return an iterable collection of Bundle objects, one object per installed bundle */ List<Bundle> bundles(); /** - * <p>This method starts the framework instance. Before this method is called, any call to {@link - * #installBundle(String)} or {@link #bundles()} will generate a {@link NullPointerException}.</p> + * This method starts the framework instance. Before this method is called, any call to {@link + * #installBundle(String)} or {@link #bundles()} will generate a {@link NullPointerException}. * - * @throws BundleException If any error occurs. + * @throws BundleException if any error occurs */ void start() throws BundleException; /** - * <p>This method <em>synchronously</em> shuts down the framework. It must be called at the end of a session in - * order to shutdown all active bundles.</p> + * Synchronously shut down the framework. It must be called at the end of a session to shutdown all active bundles. * - * @throws BundleException If any error occurs. + * @throws BundleException if any error occurs */ void stop() throws BundleException; diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/FelixFramework.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/FelixFramework.java index 7de4a9273bd..96fc0c91d2d 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/FelixFramework.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/FelixFramework.java @@ -117,19 +117,19 @@ public class FelixFramework implements OsgiFramework { wiring.refreshBundles(null, event -> { switch (event.getType()) { - case FrameworkEvent.PACKAGES_REFRESHED: - latch.countDown(); - break; - case FrameworkEvent.ERROR: - log.log(Level.SEVERE, "ERROR FrameworkEvent received.", event.getThrowable()); - break; + case FrameworkEvent.PACKAGES_REFRESHED: + latch.countDown(); + break; + case FrameworkEvent.ERROR: + log.log(Level.SEVERE, "ERROR FrameworkEvent received.", event.getThrowable()); + break; } }); try { long TIMEOUT_SECONDS = 60L; - if (!latch.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + if ( ! latch.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)) { log.warning("No PACKAGES_REFRESHED FrameworkEvent received within " + TIMEOUT_SECONDS + - " seconds of calling FrameworkWiring.refreshBundles()"); + " seconds of calling FrameworkWiring.refreshBundles()"); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -176,9 +176,10 @@ public class FelixFramework implements OsgiFramework { deleteDirContents(child); boolean deleted = child.delete(); if (! deleted) - throw new RuntimeException( - "Could not delete file '" + child.getAbsolutePath() +"'. Please check file permissions!"); + throw new RuntimeException("Could not delete file '" + child.getAbsolutePath() + + "'. Please check file permissions!"); } } } + } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java index 770c59dec35..c24e583ea46 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/StandaloneMain.java @@ -39,9 +39,9 @@ public class StandaloneMain { // We're not logging at this point since the application is responsible // for setting up logging. System.out.println("debug\tInitializing application without privileges."); + setupSignalHandlers(); loader.init(bundleLocation, false); loader.start(); - setupSignalHandlers(); waitForShutdown(); System.out.println("debug\tTrying to shutdown in a controlled manner."); log.log(Level.INFO, "JDisc shutting down"); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java index 2d2632bcf0f..71dcb7d0682 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java @@ -127,7 +127,7 @@ class HttpRequestDispatch { boolean reportedError = false; if (error != null) { - if (error instanceof CompletionException && error.getCause() instanceof EofException) { + if (isErrorOfType(error, EofException.class, IOException.class)) { log.log(Level.FINE, error, () -> "Network connection was unexpectedly terminated: " + parent.jettyRequest.getRequestURI()); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshot.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshot.java new file mode 100644 index 00000000000..4f227ccb6d4 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshot.java @@ -0,0 +1,83 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Optional; + +/** + * A snapshot of the attributes of the file for a given path, and file content if it is a regular file. + * + * @author hakonhall + */ +public class FileSnapshot { + private final Path path; + private final Optional<FileAttributes> attributes; + private final Optional<byte[]> content; + + public static FileSnapshot forPath(Path path) { return forNonExistingFile(path).snapshot(); } + + /** Guaranteed to not throw any exceptions. */ + public static FileSnapshot forNonExistingFile(Path path) { + return new FileSnapshot(path, Optional.empty(), Optional.empty()); + } + + private static FileSnapshot forRegularFile(Path path, FileAttributes attributes, byte[] content) { + if (!attributes.isRegularFile()) throw new IllegalArgumentException(path + " is not a regular file"); + return new FileSnapshot(path, Optional.of(attributes), Optional.of(content)); + } + + private static FileSnapshot forOtherFile(Path path, FileAttributes attributes) { + if (attributes.isRegularFile()) throw new IllegalArgumentException(path + " is a regular file"); + return new FileSnapshot(path, Optional.of(attributes), Optional.empty()); + } + + private FileSnapshot(Path path, Optional<FileAttributes> attributes, Optional<byte[]> content) { + this.path = path; + this.attributes = attributes; + this.content = content; + } + + public Path path() { return path; } + + /** Whether there was a file (or directory) at path. */ + public boolean exists() { return attributes.isPresent(); } + + /** Returns the file attributes if the file exists. */ + public Optional<FileAttributes> attributes() { return attributes; } + + /** Returns the file content if the file exists and is a regular file. */ + public Optional<byte[]> content() { return content; } + + /** Returns the file UTF-8 content if it exists and is a regular file. */ + public Optional<String> utf8Content() { return content.map(c -> new String(c, StandardCharsets.UTF_8)); } + + /** Returns an up-to-date snapshot of the path, possibly {@code this} if last modified time has not changed. */ + public FileSnapshot snapshot() { + Optional<FileAttributes> currentAttributes = new UnixPath(path).getAttributesIfExists(); + if (currentAttributes.isPresent()) { + + // 'this' may still be valid, depending on last modified times. + if (attributes.isPresent()) { + Instant previousModifiedTime = attributes.get().lastModifiedTime(); + Instant currentModifiedTime = currentAttributes.get().lastModifiedTime(); + if (currentModifiedTime.compareTo(previousModifiedTime) <= 0) { + return this; + } + } + + if (currentAttributes.get().isRegularFile()) { + Optional<byte[]> content = IOExceptionUtil.ifExists(() -> Files.readAllBytes(path)); + return content.map(bytes -> FileSnapshot.forRegularFile(path, currentAttributes.get(), bytes)) + // File was removed after getting attributes and before getting content. + .orElseGet(() -> FileSnapshot.forNonExistingFile(path)); + } else { + return FileSnapshot.forOtherFile(path, currentAttributes.get()); + } + } else { + return attributes.isPresent() ? FileSnapshot.forNonExistingFile(path) : this /* avoid allocation */; + } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java index f7eba68e455..afc0e7b5c22 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileWriter.java @@ -3,9 +3,11 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import java.io.File; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Optional; import java.util.function.Supplier; /** @@ -17,20 +19,30 @@ public class FileWriter { private final Path path; private final FileSync fileSync; private final PartialFileData.Builder fileDataBuilder = PartialFileData.builder(); - private final Supplier<byte[]> contentProducer; + private final Optional<ByteArraySupplier> contentProducer; private boolean overwriteExistingFile = true; + public FileWriter(Path path) { + this(path, Optional.empty()); + } + public FileWriter(Path path, Supplier<String> contentProducer) { this(path, () -> contentProducer.get().getBytes(StandardCharsets.UTF_8)); } public FileWriter(Path path, ByteArraySupplier contentProducer) { + this(path, Optional.of(contentProducer)); + } + + private FileWriter(Path path, Optional<ByteArraySupplier> contentProducer) { this.path = path; this.fileSync = new FileSync(path); this.contentProducer = contentProducer; } + public Path path() { return path; } + public FileWriter withOwner(String owner) { fileDataBuilder.withOwner(owner); return this; @@ -52,11 +64,19 @@ public class FileWriter { } public boolean converge(TaskContext context) { + return converge(context, contentProducer.orElseThrow().get()); + } + + public boolean converge(TaskContext context, String utf8Content) { + return converge(context, utf8Content.getBytes(StandardCharsets.UTF_8)); + } + + public boolean converge(TaskContext context, byte[] content) { if (!overwriteExistingFile && Files.isRegularFile(path)) { return false; } - fileDataBuilder.withContent(contentProducer.get()); + fileDataBuilder.withContent(content); PartialFileData fileData = fileDataBuilder.create(); return fileSync.convergeTo(context, fileData); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java index cf6c6c432f4..2cc74742463 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java @@ -231,6 +231,8 @@ public class UnixPath { return new UnixPath(link); } + public FileSnapshot getFileSnapshot() { return FileSnapshot.forPath(path); } + @Override public String toString() { return path.toString(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java new file mode 100644 index 00000000000..8c73d522f1d --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileSnapshotTest.java @@ -0,0 +1,64 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.Test; + +import java.nio.file.FileSystem; +import java.nio.file.Path; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * @author hakonhall + */ +public class FileSnapshotTest { + private final FileSystem fileSystem = TestFileSystem.create(); + private final UnixPath path = new UnixPath(fileSystem.getPath("/var/lib/file.txt")); + + private FileSnapshot fileSnapshot = FileSnapshot.forPath(path.toPath()); + + @Test + public void fileDoesNotExist() { + assertFalse(fileSnapshot.exists()); + assertFalse(fileSnapshot.attributes().isPresent()); + assertFalse(fileSnapshot.content().isPresent()); + assertEquals(path.toPath(), fileSnapshot.path()); + } + + @Test + public void directory() { + path.createParents().createDirectory(); + fileSnapshot = fileSnapshot.snapshot(); + assertTrue(fileSnapshot.exists()); + assertTrue(fileSnapshot.attributes().isPresent()); + assertTrue(fileSnapshot.attributes().get().isDirectory()); + } + + @Test + public void regularFile() { + path.createParents().writeUtf8File("file content"); + fileSnapshot = fileSnapshot.snapshot(); + assertTrue(fileSnapshot.exists()); + assertTrue(fileSnapshot.attributes().isPresent()); + assertTrue(fileSnapshot.attributes().get().isRegularFile()); + assertTrue(fileSnapshot.utf8Content().isPresent()); + assertEquals("file content", fileSnapshot.utf8Content().get()); + + FileSnapshot newFileSnapshot = fileSnapshot.snapshot(); + assertSame(fileSnapshot, newFileSnapshot); + } + + @Test + public void fileRemoval() { + path.createParents().writeUtf8File("file content"); + fileSnapshot = fileSnapshot.snapshot(); + assertTrue(fileSnapshot.exists()); + path.deleteIfExists(); + fileSnapshot = fileSnapshot.snapshot(); + assertFalse(fileSnapshot.exists()); + } +}
\ No newline at end of file diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersion.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersion.java index 99945ce46e8..8719a80e578 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersion.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersion.java @@ -16,7 +16,7 @@ public class OsVersion { private final boolean active; public OsVersion(Version version, boolean active) { - this.version = version; + this.version = requireNonEmpty(version); this.active = active; } @@ -49,4 +49,10 @@ public class OsVersion { return "OS version " + version + " [active: " + active + "]"; } + private static Version requireNonEmpty(Version version) { + Objects.requireNonNull(version, "version must be non-null"); + if (version.isEmpty()) throw new IllegalArgumentException("version must be non-empty"); + return version; + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java index 26e59040b95..91f619ffa91 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java @@ -5,7 +5,6 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.NodeType; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Slime; -import com.yahoo.slime.Type; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.provision.os.OsVersion; @@ -45,16 +44,8 @@ public class OsVersionsSerializer { var versions = new TreeMap<NodeType, OsVersion>(); // Use TreeMap to sort by node type var inspector = SlimeUtils.jsonToSlime(data).get(); inspector.traverse((ObjectTraverser) (key, value) -> { - Version version; - boolean active; - if (value.type() == Type.OBJECT) { - version = Version.fromString(value.field(VERSION_FIELD).asString()); - active = value.field(ACTIVE_FIELD).asBool(); - } else { - // TODO(mpolden): Remove support for legacy format after September 2019 - version = Version.fromString(value.asString()); - active = true; - } + var version = Version.fromString(value.field(VERSION_FIELD).asString()); + var active = value.field(ACTIVE_FIELD).asBool(); versions.put(NodeSerializer.nodeTypeFromString(key), new OsVersion(version, active)); }); return versions; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ErrorResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ErrorResponse.java deleted file mode 100644 index 5d7d720a1e2..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ErrorResponse.java +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; - -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.slime.JsonFormat; - -import java.io.IOException; -import java.io.OutputStream; - -import static com.yahoo.jdisc.Response.Status.*; - -/** - * Error responses with JSON bodies - * - * @author bratseth - */ -public class ErrorResponse extends HttpResponse { - - private final Slime slime = new Slime(); - private final String message; - - public enum ErrorCode { - FORBIDDEN, - UNAUTHORIZED, - NOT_FOUND, - BAD_REQUEST, - METHOD_NOT_ALLOWED, - INTERNAL_SERVER_ERROR - } - - private ErrorResponse(int code, ErrorCode errorCode, String message) { - super(code); - this.message = message; - Cursor root = slime.setObject(); - root.setString("error-code", errorCode.name()); - root.setString("message", message); - } - - public String message() { return message; } - - public static ErrorResponse notFoundError(String message) { - return new ErrorResponse(NOT_FOUND, ErrorCode.NOT_FOUND, message); - } - - public static ErrorResponse internalServerError(String message) { - return new ErrorResponse(INTERNAL_SERVER_ERROR, ErrorCode.INTERNAL_SERVER_ERROR, message); - } - - public static ErrorResponse badRequest(String message) { - return new ErrorResponse(BAD_REQUEST, ErrorCode.BAD_REQUEST, message); - } - - public static ErrorResponse methodNotAllowed(String message) { - return new ErrorResponse(METHOD_NOT_ALLOWED, ErrorCode.METHOD_NOT_ALLOWED, message); - } - - public static ErrorResponse unauthorized(String message) { - return new ErrorResponse(UNAUTHORIZED, ErrorCode.UNAUTHORIZED, message); - } - - public static ErrorResponse forbidden(String message) { - return new ErrorResponse(FORBIDDEN, ErrorCode.FORBIDDEN, message); - } - - @Override - public void render(OutputStream stream) throws IOException { - new JsonFormat(true).encode(stream, slime); - } - - @Override - public String getContentType() { return "application/json"; } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/HostCapacityResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/HostCapacityResponse.java index 7b0eb38b628..5402c22498a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/HostCapacityResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/HostCapacityResponse.java @@ -1,6 +1,5 @@ package com.yahoo.vespa.hosted.provision.restapi.v2; -import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; @@ -16,6 +15,9 @@ import java.util.Arrays; import java.util.List; import java.util.Optional; +/** + * @author mgimle + */ public class HostCapacityResponse extends HttpResponse { private final StringBuilder text; private final Slime slime; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersApiHandler.java index 6ffac2c0fbc..24aa1fe0b25 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersApiHandler.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.restapi.v2; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.restapi.ErrorResponse; import com.yahoo.vespa.hosted.provision.NoSuchNodeException; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.yolean.Exceptions; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/MessageResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/MessageResponse.java deleted file mode 100644 index 51b1eebf55f..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/MessageResponse.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; - -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; - -import java.io.IOException; -import java.io.OutputStream; - -/** - * A 200 ok response with a message in JSON - * - * @author bratseth - */ -public class MessageResponse extends HttpResponse { - - private final Slime slime = new Slime(); - - public MessageResponse(String message) { - super(200); - slime.setObject().setString("message", message); - } - - @Override - public void render(OutputStream stream) throws IOException { - new JsonFormat(true).encode(stream, slime); - } - - @Override - public String getContentType() { return "application/json"; } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index a2ebcc29078..d12150408c4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -12,7 +12,10 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.io.IOUtils; +import com.yahoo.restapi.ErrorResponse; +import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; +import com.yahoo.restapi.ResourceResponse; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; @@ -98,13 +101,13 @@ public class NodesApiHandler extends LoggingRequestHandler { private HttpResponse handleGET(HttpRequest request) { String path = request.getUri().getPath(); - if (path.equals( "/nodes/v2/")) return ResourcesResponse.fromStrings(request.getUri(), "state", "node", "command", "maintenance", "upgrade"); + if (path.equals( "/nodes/v2/")) return new ResourceResponse(request.getUri(), "state", "node", "command", "maintenance", "upgrade"); if (path.equals( "/nodes/v2/node/")) return new NodesResponse(ResponseType.nodeList, request, orchestrator, nodeRepository); if (path.startsWith("/nodes/v2/node/")) return new NodesResponse(ResponseType.singleNode, request, orchestrator, nodeRepository); if (path.equals( "/nodes/v2/state/")) return new NodesResponse(ResponseType.stateList, request, orchestrator, nodeRepository); if (path.startsWith("/nodes/v2/state/")) return new NodesResponse(ResponseType.nodesInStateList, request, orchestrator, nodeRepository); if (path.startsWith("/nodes/v2/acl/")) return new NodeAclResponse(request, nodeRepository); - if (path.equals( "/nodes/v2/command/")) return ResourcesResponse.fromStrings(request.getUri(), "restart", "reboot"); + if (path.equals( "/nodes/v2/command/")) return new ResourceResponse(request.getUri(), "restart", "reboot"); if (path.equals( "/nodes/v2/maintenance/")) return new JobsResponse(nodeRepository.jobControl()); if (path.equals( "/nodes/v2/upgrade/")) return new UpgradeResponse(nodeRepository.infrastructureVersions(), nodeRepository.osVersions(), nodeRepository.dockerImages()); if (path.startsWith("/nodes/v2/capacity")) return new HostCapacityResponse(nodeRepository, request); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ResourcesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ResourcesResponse.java deleted file mode 100644 index 3cc9a24dce3..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/ResourcesResponse.java +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; - -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; - -import java.io.IOException; -import java.io.OutputStream; -import java.net.URI; - -/** A response which lists a set of subresources as full urls */ -public class ResourcesResponse extends HttpResponse { - - private final URI parentUrl; - - private final String[] subResources; - - public ResourcesResponse(URI parentUrl, String[] subResources) { - super(200); - this.parentUrl = parentUrl; - this.subResources = subResources; - } - - @Override - public void render(OutputStream stream) throws IOException { - String parentUrlString = parentUrl.toString(); - if ( ! parentUrlString.endsWith("/")) - parentUrlString = parentUrlString + "/"; - - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor array = root.setArray("resources"); - for (String subResource : subResources) { - array.addObject().setString("url", parentUrlString + subResource + "/"); - } - new JsonFormat(true).encode(stream, slime); - } - - @Override - public String getContentType() { return "application/json"; } - - public static ResourcesResponse fromStrings(URI parentUrl, String ... subResources) { - return new ResourcesResponse(parentUrl, subResources); - } - -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializerTest.java index b41958b36db..c6583292da8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializerTest.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.os.OsVersion; import org.junit.Test; -import java.nio.charset.StandardCharsets; import java.util.Map; import static org.junit.Assert.assertEquals; @@ -17,21 +16,6 @@ import static org.junit.Assert.assertEquals; public class OsVersionsSerializerTest { @Test - public void legacy_format() { - var json = "{\"host\":\"1.2.3\",\"proxyhost\":\"4.5.6\",\"confighost\":\"7.8.9\"}"; - var serializedFromString = OsVersionsSerializer.fromJson(json.getBytes(StandardCharsets.UTF_8)); - var versions = Map.of( - NodeType.host, new OsVersion(Version.fromString("1.2.3"), true), - NodeType.proxyhost, new OsVersion(Version.fromString("4.5.6"), true), - NodeType.confighost, new OsVersion(Version.fromString("7.8.9"), true) - ); - assertEquals(versions, serializedFromString); - - var serialized = OsVersionsSerializer.fromJson(OsVersionsSerializer.toJson(versions)); - assertEquals(serialized, versions); - } - - @Test public void serialization() { var versions = Map.of( NodeType.host, new OsVersion(Version.fromString("1.2.3"), true), diff --git a/parent/pom.xml b/parent/pom.xml index 0404249571f..36443db2c28 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -123,7 +123,7 @@ <artifactId>maven-dependency-plugin</artifactId> <version>3.1.1</version> <dependencies> - <!-- TODO: Remove when upgrading to 3.1.2 --> + <!-- TODO: remove when upgrading to 3.1.2 --> <dependency> <groupId>org.apache.maven.shared</groupId> <artifactId>maven-dependency-analyzer</artifactId> @@ -405,10 +405,13 @@ </profiles> <dependencyManagement> <dependencies> + + <!-- Please keep this list lexically sorted by groupId, then artifactId.--> + <dependency> - <groupId>org.apache.maven.wagon</groupId> - <artifactId>wagon-ssh-external</artifactId> - <version>2.7</version> + <groupId>com.amazonaws</groupId> + <artifactId>aws-java-sdk-core</artifactId> + <version>${aws.sdk.version}</version> </dependency> <dependency> <groupId>com.github.cverges.expect4j</groupId> @@ -416,54 +419,50 @@ <version>1.6</version> </dependency> <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-compress</artifactId> - <version>1.18</version> - </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-exec</artifactId> - <version>1.3</version> + <groupId>com.github.tomakehurst</groupId> + <artifactId>wiremock-standalone</artifactId> + <version>2.6.0</version> </dependency> <dependency> - <groupId>org.apache.velocity</groupId> - <artifactId>velocity</artifactId> - <version>1.7</version> + <groupId>com.google.jimfs</groupId> + <artifactId>jimfs</artifactId> + <version>1.1</version> + <scope>test</scope> <!-- TODO: remove scope from parent pom --> </dependency> <dependency> - <groupId>io.airlift</groupId> - <artifactId>airline</artifactId> - <version>0.7</version> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + <version>${protobuf.version}</version> </dependency> <dependency> - <groupId>io.prometheus</groupId> - <artifactId>simpleclient</artifactId> - <version>${prometheus.client.version}</version> + <groupId>com.ibm.icu</groupId> + <artifactId>icu4j</artifactId> + <version>57.1</version> </dependency> <dependency> - <groupId>io.prometheus</groupId> - <artifactId>simpleclient_common</artifactId> - <version>${prometheus.client.version}</version> + <groupId>com.infradna.tool</groupId> + <artifactId>bridge-method-annotation</artifactId> + <version>1.4</version> </dependency> <dependency> - <groupId>org.ow2.asm</groupId> - <artifactId>asm</artifactId> - <version>${asm.version}</version> + <groupId>com.optimaize.languagedetector</groupId> + <artifactId>language-detector</artifactId> + <version>0.6</version> </dependency> <dependency> - <groupId>org.eclipse.collections</groupId> - <artifactId>eclipse-collections</artifactId> - <version>9.2.0</version> + <groupId>com.yahoo.athenz</groupId> + <artifactId>athenz-zms-java-client</artifactId> + <version>${athenz.version}</version> </dependency> <dependency> - <groupId>org.eclipse.collections</groupId> - <artifactId>eclipse-collections-api</artifactId> - <version>9.2.0</version> + <groupId>com.yahoo.athenz</groupId> + <artifactId>athenz-zpe-java-client</artifactId> + <version>${athenz.version}</version> </dependency> <dependency> - <groupId>com.infradna.tool</groupId> - <artifactId>bridge-method-annotation</artifactId> - <version>1.4</version> + <groupId>com.yahoo.athenz</groupId> + <artifactId>athenz-zts-java-client</artifactId> + <version>${athenz.version}</version> </dependency> <dependency> <groupId>commons-cli</groupId> @@ -505,39 +504,35 @@ <artifactId>commons-pool</artifactId> <version>1.5.6</version> </dependency> - <!-- Explicitly included to get Zookeeper version 3.4.14, - can be excluded if you want the Zookeeper version - used by curator by default - --> <dependency> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - <version>3.4.14</version> + <groupId>io.airlift</groupId> + <artifactId>airline</artifactId> + <version>0.7</version> </dependency> <dependency> - <groupId>org.apache.curator</groupId> - <artifactId>curator-recipes</artifactId> - <version>${curator.version}</version> + <groupId>io.prometheus</groupId> + <artifactId>simpleclient</artifactId> + <version>${prometheus.client.version}</version> </dependency> <dependency> - <groupId>org.apache.curator</groupId> - <artifactId>curator-test</artifactId> - <version>${curator.version}</version> + <groupId>io.prometheus</groupId> + <artifactId>simpleclient_common</artifactId> + <version>${prometheus.client.version}</version> </dependency> <dependency> - <groupId>org.junit.jupiter</groupId> - <artifactId>junit-jupiter-engine</artifactId> - <version>${junit.version}</version> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.12</version> </dependency> <dependency> - <groupId>org.junit.vintage</groupId> - <artifactId>junit-vintage-engine</artifactId> - <version>${junit.version}</version> + <groupId>net.java.dev.jna</groupId> + <artifactId>jna</artifactId> + <version>${jna.version}</version> </dependency> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <version>4.12</version> + <groupId>net.spy</groupId> + <artifactId>spymemcached</artifactId> + <version>2.10.1</version> </dependency> <dependency> <groupId>org.antlr</groupId> @@ -551,10 +546,30 @@ </dependency> <dependency> <groupId>org.apache.commons</groupId> + <artifactId>commons-compress</artifactId> + <version>1.18</version> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-exec</artifactId> + <version>1.3</version> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> <version>3.1</version> </dependency> <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-recipes</artifactId> + <version>${curator.version}</version> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-test</artifactId> + <version>${curator.version}</version> + </dependency> + <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>fluent-hc</artifactId> <version>4.3.6</version> @@ -590,11 +605,6 @@ <version>3.5.0</version> </dependency> <dependency> - <groupId>org.apache.maven.plugin-tools</groupId> - <artifactId>maven-plugin-annotations</artifactId> - <version>${maven-plugin-tools.version}</version> - </dependency> - <dependency> <groupId>org.apache.maven</groupId> <artifactId>maven-plugin-api</artifactId> <version>${maven-plugin-tools.version}</version> @@ -605,6 +615,11 @@ <version>2.2.1</version> </dependency> <dependency> + <groupId>org.apache.maven.plugin-tools</groupId> + <artifactId>maven-plugin-annotations</artifactId> + <version>${maven-plugin-tools.version}</version> + </dependency> + <dependency> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-jar-plugin</artifactId> <version>3.1.0</version> @@ -621,6 +636,35 @@ <type>pom</type> </dependency> <dependency> + <groupId>org.apache.maven.wagon</groupId> + <artifactId>wagon-ssh-external</artifactId> + <version>2.7</version> + </dependency> + <dependency> + <groupId>org.apache.opennlp</groupId> + <artifactId>opennlp-tools</artifactId> + <version>1.8.4</version> + </dependency> + <dependency> + <groupId>org.apache.velocity</groupId> + <artifactId>velocity</artifactId> + <version>1.7</version> + </dependency> + <dependency> + <!-- Explicitly force Zookeeper version, + can be excluded if you want the Zookeeper version + used by curator by default + --> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + <version>${zookeeper.version}</version> + </dependency> + <dependency> + <groupId>org.assertj</groupId> + <artifactId>assertj-core</artifactId> + <version>3.11.1</version> + </dependency> + <dependency> <groupId>org.codehaus.jettison</groupId> <artifactId>jettison</artifactId> <version>1.3.1</version> @@ -629,31 +673,35 @@ <groupId>org.cthul</groupId> <artifactId>cthul-matchers</artifactId> <version>1.0</version> - <scope>test</scope> + <scope>test</scope> <!-- TODO: remove scope from parent pom --> + </dependency> + <dependency> + <groupId>org.eclipse.collections</groupId> + <artifactId>eclipse-collections</artifactId> + <version>9.2.0</version> + </dependency> + <dependency> + <groupId>org.eclipse.collections</groupId> + <artifactId>eclipse-collections-api</artifactId> + <version>9.2.0</version> </dependency> <dependency> <groupId>org.hamcrest</groupId> <artifactId>hamcrest-all</artifactId> <version>1.3</version> - <scope>test</scope> + <scope>test</scope> <!-- TODO: remove scope from parent pom --> </dependency> <dependency> <groupId>org.hamcrest</groupId> <artifactId>hamcrest-core</artifactId> <version>1.3</version> - <scope>test</scope> + <scope>test</scope> <!-- TODO: remove scope from parent pom --> </dependency> <dependency> <groupId>org.hamcrest</groupId> <artifactId>hamcrest-library</artifactId> <version>1.3</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>uk.co.datumedge</groupId> - <artifactId>hamcrest-json</artifactId> - <version>0.2</version> - <scope>test</scope> + <scope>test</scope> <!-- TODO: remove scope from parent pom --> </dependency> <dependency> <groupId>org.hdrhistogram</groupId> @@ -661,16 +709,25 @@ <version>2.1.8</version> </dependency> <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + <version>${junit.version}</version> + </dependency> + <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <version>${junit.version}</version> + </dependency> + <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <version>1.10.19</version> - <scope>test</scope> + <scope>test</scope> <!-- TODO: remove scope from parent pom --> </dependency> <dependency> - <groupId>com.google.jimfs</groupId> - <artifactId>jimfs</artifactId> - <version>1.1</version> - <scope>test</scope> + <groupId>org.ow2.asm</groupId> + <artifactId>asm</artifactId> + <version>${asm.version}</version> </dependency> <dependency> <groupId>org.springframework</groupId> @@ -678,66 +735,6 @@ <version>4.0.6.RELEASE</version> </dependency> <dependency> - <groupId>org.twdata.maven</groupId> - <artifactId>mojo-executor</artifactId> - <version>2.3.0</version> - </dependency> - <dependency> - <groupId>net.spy</groupId> - <artifactId>spymemcached</artifactId> - <version>2.10.1</version> - </dependency> - <dependency> - <groupId>xerces</groupId> - <artifactId>xercesImpl</artifactId> - <version>2.11.0</version> - </dependency> - <dependency> - <groupId>com.ibm.icu</groupId> - <artifactId>icu4j</artifactId> - <version>57.1</version> - </dependency> - <dependency> - <groupId>com.yahoo.athenz</groupId> - <artifactId>athenz-zms-java-client</artifactId> - <version>${athenz.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.athenz</groupId> - <artifactId>athenz-zts-java-client</artifactId> - <version>${athenz.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.athenz</groupId> - <artifactId>athenz-zpe-java-client</artifactId> - <version>${athenz.version}</version> - </dependency> - <dependency> - <groupId>com.github.tomakehurst</groupId> - <artifactId>wiremock-standalone</artifactId> - <version>2.6.0</version> - </dependency> - <dependency> - <groupId>org.apache.opennlp</groupId> - <artifactId>opennlp-tools</artifactId> - <version>1.8.4</version> - </dependency> - <dependency> - <groupId>com.optimaize.languagedetector</groupId> - <artifactId>language-detector</artifactId> - <version>0.6</version> - </dependency> - <dependency> - <groupId>net.java.dev.jna</groupId> - <artifactId>jna</artifactId> - <version>${jna.version}</version> - </dependency> - <dependency> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - <version>${protobuf.version}</version> - </dependency> - <dependency> <groupId>org.tensorflow</groupId> <artifactId>proto</artifactId> <version>${tensorflow.version}</version> @@ -748,14 +745,20 @@ <version>${tensorflow.version}</version> </dependency> <dependency> - <groupId>org.assertj</groupId> - <artifactId>assertj-core</artifactId> - <version>3.11.1</version> + <groupId>org.twdata.maven</groupId> + <artifactId>mojo-executor</artifactId> + <version>2.3.0</version> </dependency> <dependency> - <groupId>com.amazonaws</groupId> - <artifactId>aws-java-sdk-core</artifactId> - <version>${aws.sdk.version}</version> + <groupId>uk.co.datumedge</groupId> + <artifactId>hamcrest-json</artifactId> + <version>0.2</version> + <scope>test</scope> <!-- TODO: remove scope from parent pom --> + </dependency> + <dependency> + <groupId>xerces</groupId> + <artifactId>xercesImpl</artifactId> + <version>2.11.0</version> </dependency> </dependencies> </dependencyManagement> @@ -785,6 +788,7 @@ <protobuf.version>3.7.0</protobuf.version> <surefire.version>2.22.0</surefire.version> <tensorflow.version>1.12.0</tensorflow.version> + <zookeeper.version>3.4.14</zookeeper.version> <doclint>all</doclint> <test.hide>true</test.hide> diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index ee262a5fa5c..53435780d9a 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -40,7 +40,6 @@ vespa_define_module( src/vespa/searchcore/proton/server src/vespa/searchcore/proton/summaryengine src/vespa/searchcore/proton/test - src/vespa/searchcore/util APPS src/apps/proton diff --git a/searchcore/src/apps/proton/CMakeLists.txt b/searchcore/src/apps/proton/CMakeLists.txt index a8bfa9a39a8..c42ed048ce1 100644 --- a/searchcore/src/apps/proton/CMakeLists.txt +++ b/searchcore/src/apps/proton/CMakeLists.txt @@ -24,7 +24,6 @@ vespa_add_executable(searchcore_proton_app searchcore_grouping searchcore_proton_metrics searchcore_fconfig - searchcore_util storageserver_storageapp searchlib_searchlib_uca ) diff --git a/searchcore/src/apps/tests/CMakeLists.txt b/searchcore/src/apps/tests/CMakeLists.txt index 55c371791a3..42be4091f66 100644 --- a/searchcore/src/apps/tests/CMakeLists.txt +++ b/searchcore/src/apps/tests/CMakeLists.txt @@ -19,7 +19,6 @@ vespa_add_executable(searchcore_persistenceconformance_test_app TEST searchcore_grouping searchcore_proton_metrics searchcore_fconfig - searchcore_util vdstestlib persistence_persistence_conformancetest searchlib_searchlib_uca diff --git a/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp b/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp index 2e097e7141f..3b3b5f412d2 100644 --- a/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp +++ b/searchcore/src/apps/vespa-proton-cmd/vespa-proton-cmd.cpp @@ -216,9 +216,12 @@ public: fprintf(stderr, "Config system is not up. Verify that vespa is started."); return 3; } - - initRPC(); - + try { + initRPC(); + } catch (vespalib::Exception &e) { + fprintf(stderr, "Exception in network initialization: %s", e.what()); + return 2; + } int port = 0; std::string spec = _argv[1]; diff --git a/searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt b/searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt index d25a88c1a71..8b552cb2b47 100644 --- a/searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt +++ b/searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt @@ -4,6 +4,5 @@ vespa_add_executable(searchcore_attribute_updater_test_app TEST attribute_updater_test.cpp DEPENDS searchcore_pcommon - searchcore_util ) vespa_add_test(NAME searchcore_attribute_updater_test_app COMMAND searchcore_attribute_updater_test_app) diff --git a/searchcore/src/tests/proton/docsummary/CMakeLists.txt b/searchcore/src/tests/proton/docsummary/CMakeLists.txt index 906a1e642f5..ce1a0b3d68c 100644 --- a/searchcore/src/tests/proton/docsummary/CMakeLists.txt +++ b/searchcore/src/tests/proton/docsummary/CMakeLists.txt @@ -22,7 +22,6 @@ vespa_add_executable(searchcore_docsummary_test_app TEST searchcore_grouping searchcore_proton_metrics searchcore_fconfig - searchcore_util searchlib_searchlib_uca ) vespa_add_executable(searchcore_summaryfieldconverter_test_app diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index 0a9f3127844..0e521e473ae 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -23,7 +23,6 @@ #include <vespa/searchcore/proton/server/searchview.h> #include <vespa/searchcore/proton/server/summaryadapter.h> #include <vespa/searchlib/common/gatecallback.h> -#include <vespa/searchlib/common/transport.h> #include <vespa/searchlib/engine/docsumapi.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> @@ -359,7 +358,7 @@ Test::assertTensor(const Tensor::UP & exp, const std::string & fieldName, uint32_t classId; ASSERT_LESS_EQUAL(sizeof(classId), docsum.data.size()); memcpy(&classId, docsum.data.c_str(), sizeof(classId)); - ASSERT_EQUAL(::search::fs4transport::SLIME_MAGIC_ID, classId); + ASSERT_EQUAL(::search::docsummary::SLIME_MAGIC_ID, classId); vespalib::Slime slime; vespalib::Memory serialized(docsum.data.c_str() + sizeof(classId), docsum.data.size() - sizeof(classId)); @@ -384,7 +383,7 @@ getSlime(const DocsumReply &reply, uint32_t id, bool relaxed) uint32_t classId; ASSERT_LESS_EQUAL(sizeof(classId), docsum.data.size()); memcpy(&classId, docsum.data.c_str(), sizeof(classId)); - ASSERT_EQUAL(::search::fs4transport::SLIME_MAGIC_ID, classId); + ASSERT_EQUAL(search::docsummary::SLIME_MAGIC_ID, classId); vespalib::Slime slime; vespalib::Memory serialized(docsum.data.c_str() + sizeof(classId), docsum.data.size() - sizeof(classId)); diff --git a/searchcore/src/tests/proton/documentdb/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/CMakeLists.txt index cd67bd82a06..d4a08e9de5c 100644 --- a/searchcore/src/tests/proton/documentdb/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/CMakeLists.txt @@ -19,7 +19,6 @@ vespa_add_executable(searchcore_documentdb_test_app TEST searchcore_grouping searchcore_proton_metrics searchcore_fconfig - searchcore_util searchlib_searchlib_uca ) vespa_add_test(NAME searchcore_documentdb_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/documentdb_test.sh diff --git a/searchcore/src/tests/proton/documentdb/buckethandler/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/buckethandler/CMakeLists.txt index d48778ffd1b..e955aa76d37 100644 --- a/searchcore/src/tests/proton/documentdb/buckethandler/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/buckethandler/CMakeLists.txt @@ -12,7 +12,6 @@ vespa_add_executable(searchcore_buckethandler_test_app TEST searchcore_bucketdb searchcore_pcommon searchcore_grouping - searchcore_util searchcore_fconfig ) vespa_add_test(NAME searchcore_buckethandler_test_app COMMAND searchcore_buckethandler_test_app) diff --git a/searchcore/src/tests/proton/documentdb/clusterstatehandler/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/clusterstatehandler/CMakeLists.txt index 28b79100eda..77bdae70ebd 100644 --- a/searchcore/src/tests/proton/documentdb/clusterstatehandler/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/clusterstatehandler/CMakeLists.txt @@ -10,7 +10,6 @@ vespa_add_executable(searchcore_clusterstatehandler_test_app TEST searchcore_attribute searchcore_pcommon searchcore_grouping - searchcore_util searchcore_fconfig ) vespa_add_test(NAME searchcore_clusterstatehandler_test_app COMMAND searchcore_clusterstatehandler_test_app) diff --git a/searchcore/src/tests/proton/documentdb/combiningfeedview/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/combiningfeedview/CMakeLists.txt index e046b81b8bd..e810b2191bb 100644 --- a/searchcore/src/tests/proton/documentdb/combiningfeedview/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/combiningfeedview/CMakeLists.txt @@ -13,7 +13,6 @@ vespa_add_executable(searchcore_combiningfeedview_test_app TEST searchcore_pcommon searchcore_grouping searchcore_proton_metrics - searchcore_util searchcore_fconfig ) vespa_add_test(NAME searchcore_combiningfeedview_test_app COMMAND searchcore_combiningfeedview_test_app) diff --git a/searchcore/src/tests/proton/documentdb/configurer/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/configurer/CMakeLists.txt index fe09e017ba5..a7cdfff8085 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/configurer/CMakeLists.txt @@ -18,7 +18,6 @@ vespa_add_executable(searchcore_configurer_test_app TEST searchcore_grouping searchcore_proton_metrics searchcore_fconfig - searchcore_util searchlib_searchlib_uca ) vespa_add_test(NAME searchcore_configurer_test_app COMMAND searchcore_configurer_test_app) diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/document_subdbs/CMakeLists.txt index f4a9d90add9..697666f8e02 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/CMakeLists.txt @@ -18,7 +18,6 @@ vespa_add_executable(searchcore_document_subdbs_test_app TEST searchcore_pcommon searchcore_grouping searchcore_proton_metrics - searchcore_util searchcore_fconfig searchlib_searchlib_uca ) diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt index b23ab73cd6e..1c11ed745a6 100644 --- a/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/CMakeLists.txt @@ -13,7 +13,6 @@ vespa_add_executable(searchcore_documentbucketmover_test_app TEST searchcore_bucketdb searchcore_pcommon searchcore_grouping - searchcore_util searchcore_fconfig ) vespa_add_test(NAME searchcore_documentbucketmover_test_app COMMAND searchcore_documentbucketmover_test_app) diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/feedhandler/CMakeLists.txt index 8f93629b6a6..84027130cdb 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/feedhandler/CMakeLists.txt @@ -13,7 +13,6 @@ vespa_add_executable(searchcore_feedhandler_test_app TEST searchcore_pcommon searchcore_grouping searchcore_proton_metrics - searchcore_util searchcore_fconfig ) vespa_add_test(NAME searchcore_feedhandler_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/feedhandler_test.sh diff --git a/searchcore/src/tests/proton/documentdb/feedview/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/feedview/CMakeLists.txt index ebf28f3bf16..a46ca246d9a 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/feedview/CMakeLists.txt @@ -14,7 +14,6 @@ vespa_add_executable(searchcore_feedview_test_app TEST searchcore_pcommon searchcore_grouping searchcore_proton_metrics - searchcore_util searchcore_fconfig ) vespa_add_test(NAME searchcore_feedview_test_app COMMAND searchcore_feedview_test_app) diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/maintenancecontroller/CMakeLists.txt index 6b25971df83..0864f444ecf 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/CMakeLists.txt @@ -14,7 +14,6 @@ vespa_add_executable(searchcore_maintenancecontroller_test_app TEST searchcore_persistenceengine searchcore_grouping searchcore_proton_metrics - searchcore_util searchcore_fconfig searchlib_test ) @@ -33,7 +32,6 @@ vespa_add_executable(searchcore_frozenbucketsmap_test_app TEST searchcore_persistenceengine searchcore_grouping searchcore_proton_metrics - searchcore_util searchcore_fconfig ) vespa_add_test(NAME searchcore_frozenbucketsmap_test_app COMMAND searchcore_frozenbucketsmap_test_app) diff --git a/searchcore/src/tests/proton/feed_and_search/CMakeLists.txt b/searchcore/src/tests/proton/feed_and_search/CMakeLists.txt index fdc2a925ae1..2c37e4f4a71 100644 --- a/searchcore/src/tests/proton/feed_and_search/CMakeLists.txt +++ b/searchcore/src/tests/proton/feed_and_search/CMakeLists.txt @@ -3,6 +3,5 @@ vespa_add_executable(searchcore_feed_and_search_test_app TEST SOURCES feed_and_search.cpp DEPENDS - searchcore_util ) vespa_add_test(NAME searchcore_feed_and_search_test_app COMMAND searchcore_feed_and_search_test_app) diff --git a/searchcore/src/tests/proton/index/CMakeLists.txt b/searchcore/src/tests/proton/index/CMakeLists.txt index ef40e291e18..1ffad6cdbf8 100644 --- a/searchcore/src/tests/proton/index/CMakeLists.txt +++ b/searchcore/src/tests/proton/index/CMakeLists.txt @@ -7,7 +7,6 @@ vespa_add_executable(searchcore_indexmanager_test_app TEST searchcore_index searchcore_flushengine searchcore_pcommon - searchcore_util gtest ) vespa_add_executable(searchcore_fusionrunner_test_app TEST @@ -17,7 +16,6 @@ vespa_add_executable(searchcore_fusionrunner_test_app TEST searchcore_server searchcore_index searchcore_pcommon - searchcore_util ) vespa_add_executable(searchcore_diskindexcleaner_test_app TEST SOURCES diff --git a/searchcore/src/tests/proton/matching/CMakeLists.txt b/searchcore/src/tests/proton/matching/CMakeLists.txt index 0fce2f6aca2..e3c5fa2bd97 100644 --- a/searchcore/src/tests/proton/matching/CMakeLists.txt +++ b/searchcore/src/tests/proton/matching/CMakeLists.txt @@ -12,7 +12,6 @@ vespa_add_executable(searchcore_matching_test_app TEST searchcore_bucketdb searchcore_pcommon searchcore_grouping - searchcore_util searchlib_searchlib_uca searchlib_test ) diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index 3f68b54aca2..950321533b0 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -18,7 +18,6 @@ #include <vespa/searchlib/aggregation/perdocexpression.h> #include <vespa/searchlib/attribute/extendableattributes.h> #include <vespa/searchlib/common/featureset.h> -#include <vespa/searchlib/common/transport.h> #include <vespa/searchlib/engine/docsumrequest.h> #include <vespa/searchlib/engine/searchrequest.h> #include <vespa/searchlib/engine/docsumreply.h> @@ -573,27 +572,22 @@ TEST("require that re-ranking is diverse with diversity = 1/10") { } TEST("require that sortspec can be used (multi-threaded)") { - for (bool drop_sort_data: {false, true}) { - for (size_t threads = 1; threads <= 16; ++threads) { - MyWorld world; - world.basicSetup(); - world.basicResults(); - SearchRequest::SP request = world.createSimpleRequest("f1", "spread"); - request->sortSpec = "+a1"; - if (drop_sort_data) { - request->queryFlags |= fs4transport::QFLAG_DROP_SORTDATA; - } - SearchReply::UP reply = world.performSearch(request, threads); - ASSERT_EQUAL(9u, reply->hits.size()); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::100").getGlobalId(), reply->hits[0].gid); - EXPECT_EQUAL(zero_rank_value, reply->hits[0].metric); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::200").getGlobalId(), reply->hits[1].gid); - EXPECT_EQUAL(zero_rank_value, reply->hits[1].metric); - EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::300").getGlobalId(), reply->hits[2].gid); - EXPECT_EQUAL(zero_rank_value, reply->hits[2].metric); - EXPECT_EQUAL(drop_sort_data, reply->sortIndex.empty()); - EXPECT_EQUAL(drop_sort_data, reply->sortData.empty()); - } + for (size_t threads = 1; threads <= 16; ++threads) { + MyWorld world; + world.basicSetup(); + world.basicResults(); + SearchRequest::SP request = world.createSimpleRequest("f1", "spread"); + request->sortSpec = "+a1"; + SearchReply::UP reply = world.performSearch(request, threads); + ASSERT_EQUAL(9u, reply->hits.size()); + EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::100").getGlobalId(), reply->hits[0].gid); + EXPECT_EQUAL(zero_rank_value, reply->hits[0].metric); + EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::200").getGlobalId(), reply->hits[1].gid); + EXPECT_EQUAL(zero_rank_value, reply->hits[1].metric); + EXPECT_EQUAL(document::DocumentId("id:ns:searchdocument::300").getGlobalId(), reply->hits[2].gid); + EXPECT_EQUAL(zero_rank_value, reply->hits[2].metric); + EXPECT_FALSE(reply->sortIndex.empty()); + EXPECT_FALSE(reply->sortData.empty()); } } diff --git a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp index 23ef86a46b7..a9cae7d8ab7 100644 --- a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp +++ b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp @@ -7,7 +7,7 @@ #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/util/compressor.h> -#include <vespa/searchlib/common/transport.h> +#include <vespa/searchsummary/docsummary/docsumwriter.h> #include <vespa/metrics/metricset.h> #include <vespa/fnet/frt/rpcrequest.h> @@ -336,7 +336,7 @@ void createSummary(search::RawBuf &buf) { vespalib::Slime summary; summary.setObject().setLong("long", 982); - uint32_t magic = search::fs4transport::SLIME_MAGIC_ID; + uint32_t magic = search::docsummary::SLIME_MAGIC_ID; buf.append(&magic, sizeof(magic)); search::SlimeOutputRawBufAdapter adapter(buf); BinaryFormat::encode(summary, adapter); diff --git a/searchcore/src/vespa/searchcore/common/.gitignore b/searchcore/src/vespa/searchcore/common/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/searchcore/src/vespa/searchcore/common/.gitignore +++ /dev/null diff --git a/searchcore/src/vespa/searchcore/common/OWNERS b/searchcore/src/vespa/searchcore/common/OWNERS deleted file mode 100644 index 9dc0c2d970d..00000000000 --- a/searchcore/src/vespa/searchcore/common/OWNERS +++ /dev/null @@ -1 +0,0 @@ -baldersheim diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp index c65257e7f6a..5951d07a2cb 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp @@ -5,7 +5,6 @@ #include <vespa/searchlib/attribute/iattributemanager.h> #include <vespa/searchlib/common/location.h> #include <vespa/searchlib/common/matching_elements.h> -#include <vespa/searchlib/common/transport.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/util/stringfmt.h> @@ -50,7 +49,6 @@ DocsumContext::initState() { const DocsumRequest & req = _request; _docsumState._args.initFromDocsumRequest(req); - _docsumState._args.SetQueryFlags(req.queryFlags & ~search::fs4transport::QFLAG_DROP_SORTDATA); _docsumState._docsumcnt = req.hits.size(); _docsumState._docsumbuf = (_docsumState._docsumcnt > 0) @@ -176,7 +174,7 @@ DocsumContext::FillRankFeatures(search::docsummary::GetDocsumsState * state, sea { assert(&_docsumState == state); // check if we are allowed to run - if ((state->_args.GetQueryFlags() & search::fs4transport::QFLAG_DUMP_FEATURES) == 0) { + if ( ! state->_args.dumpFeatures()) { return; } state->_rankFeatures = _matcher->getRankFeatures(_request, _searchCtx, _attrCtx, _sessionMgr); diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index fe7a7616c18..87c0283d228 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -8,7 +8,6 @@ #include "matcher.h" #include "sessionmanager.h" #include <vespa/searchcore/grouping/groupingcontext.h> -#include <vespa/searchlib/engine/errorcodes.h> #include <vespa/searchlib/engine/docsumrequest.h> #include <vespa/searchlib/engine/searchrequest.h> #include <vespa/searchlib/engine/searchreply.h> @@ -214,8 +213,6 @@ Matcher::match(const SearchRequest &request, vespalib::ThreadBundle &threadBundl metaStore, *feature_overrides); traceQuery(6, request.trace(), mtf->query()); if (!mtf->valid()) { - reply->errorCode = ECODE_QUERY_PARSE_ERROR; - reply->errorMessage = "query execution failed (invalid query)"; return reply; } @@ -227,7 +224,7 @@ Matcher::match(const SearchRequest &request, vespalib::ThreadBundle &threadBundl !_rankSetup->getSecondPhaseRank().empty(), !willNotNeedRanking(request, groupingContext)); ResultProcessor rp(attrContext, metaStore, sessionMgr, groupingContext, sessionId, - request.sortSpec, params.offset, params.hits, request.should_drop_sort_data()); + request.sortSpec, params.offset, params.hits); size_t numThreadsPerSearch = computeNumThreadsPerSearch(mtf->estimate(), rankProperties); LimitedThreadBundleWrapper limitedThreadBundle(threadBundle, numThreadsPerSearch); diff --git a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp index dedda1504a5..445aab310d9 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp @@ -61,8 +61,7 @@ ResultProcessor::ResultProcessor(IAttributeContext &attrContext, GroupingContext &groupingContext, const vespalib::string &sessionId, const vespalib::string &sortSpec, - size_t offset, size_t hits, - bool drop_sort_data) + size_t offset, size_t hits) : _attrContext(attrContext), _metaStore(metaStore), _sessionMgr(sessionMgr), @@ -71,7 +70,6 @@ ResultProcessor::ResultProcessor(IAttributeContext &attrContext, _sortSpec(sortSpec), _offset(offset), _hits(hits), - _drop_sort_data(drop_sort_data), _wasMerged(false) { if (!_groupingContext.empty()) { @@ -140,7 +138,7 @@ ResultProcessor::makeReply(PartialResultUP full_result) dst.metric = src._rankValue; LOG(debug, "convertLidToGid: hit[%zu]: lid(%u) -> gid(%s)", i, docId, dst.gid.toString().c_str()); } - if (result.hasSortData() && (hitcnt > 0) && !_drop_sort_data) { + if (result.hasSortData() && (hitcnt > 0)) { size_t sortDataSize = result.sortDataSize(); for (size_t i = 0; i < hitOffset; ++i) { sortDataSize -= result.sortData(i).second; diff --git a/searchcore/src/vespa/searchcore/proton/matching/result_processor.h b/searchcore/src/vespa/searchcore/proton/matching/result_processor.h index 41c764d247b..48a8aecebe1 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/result_processor.h +++ b/searchcore/src/vespa/searchcore/proton/matching/result_processor.h @@ -88,7 +88,6 @@ private: const vespalib::string &_sortSpec; size_t _offset; size_t _hits; - bool _drop_sort_data; bool _wasMerged; public: @@ -98,8 +97,7 @@ public: GroupingContext & groupingContext, const vespalib::string & sessionId, const vespalib::string & sortSpec, - size_t offset, size_t hits, - bool drop_sort_data); + size_t offset, size_t hits); ~ResultProcessor(); size_t countFS4Hits(); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 0a49f494406..a8f5cb32375 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -23,6 +23,7 @@ #include <vespa/searchcore/proton/matchengine/matchengine.h> #include <vespa/searchlib/transactionlog/trans_log_server_explorer.h> #include <vespa/searchlib/util/fileheadertk.h> +#include <vespa/searchlib/common/packets.h> #include <vespa/document/base/exceptions.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/repo/documenttyperepo.h> @@ -30,7 +31,6 @@ #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/host_name.h> #include <vespa/vespalib/util/random.h> -#include <vespa/searchlib/engine/transportserver.h> #include <vespa/vespalib/net/state_server.h> #include <vespa/searchlib/aggregation/forcelink.hpp> @@ -203,7 +203,6 @@ Proton::Proton(const config::ConfigUri & configUri, _customComponentBindToken(), _customComponentRootToken(), _stateServer(), - _fs4Server(), // This executor can only have 1 thread as it is used for // serializing startup. _executor(1, 128 * 1024), @@ -289,9 +288,6 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) _tls->start(); _flushEngine = std::make_unique<FlushEngine>(std::make_shared<flushengine::TlsStatsFactory>(_tls->getTransLogServer()), strategy, flush.maxconcurrent, flush.idleinterval*1000); - _fs4Server = std::make_unique<TransportServer>(*_matchEngine, *_summaryEngine, *this, protonConfig.ptport, TransportServer::DEBUG_ALL); - _fs4Server->setTCPNoDelay(true); - _metricsEngine->addExternalMetrics(_fs4Server->getMetrics()); _metricsEngine->addExternalMetrics(_summaryEngine->getMetrics()); char tmp[1024]; @@ -333,11 +329,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) waitForOnlineState(); _isReplayDone = true; _rpcHooks->set_online(); - if ( ! _fs4Server->start() ) { - throw vespalib::PortListenException(protonConfig.ptport, "FS4"); - } - int port = _fs4Server->getListenPort(); - LOG(debug, "Started fs4 interface on port %d", port); + _flushEngine->start(); _isInitializing = false; _protonConfigurer.setAllowReconfig(true); @@ -445,13 +437,7 @@ Proton::~Proton() if (_sharedExecutor) { _sharedExecutor->sync(); } - LOG(debug, "Shutting down fs4 interface"); - if (_metricsEngine && _fs4Server) { - _metricsEngine->removeExternalMetrics(_fs4Server->getMetrics()); - } - if (_fs4Server) { - _fs4Server->shutDown(); - } + if ( ! _documentDBMap.empty()) { size_t numCores = 4; const std::shared_ptr<proton::ProtonConfigSnapshot> pcsp = _protonConfigurer.getActiveConfigSnapshot(); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index fe7e8fe2219..487a596e7e6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -29,8 +29,6 @@ #include <mutex> #include <shared_mutex> -namespace search::engine { class TransportServer; } - namespace vespalib { class StateServer; } namespace proton { @@ -54,7 +52,6 @@ class Proton : public IProtonConfigurerOwner, { private: typedef search::transactionlog::TransLogServerApp TLS; - using TransportServer = search::engine::TransportServer; typedef search::engine::MonitorRequest MonitorRequest; typedef search::engine::MonitorReply MonitorReply; typedef search::engine::MonitorClient MonitorClient; @@ -109,7 +106,6 @@ private: vespalib::JsonHandlerRepo::Token::UP _customComponentBindToken; vespalib::JsonHandlerRepo::Token::UP _customComponentRootToken; std::unique_ptr<vespalib::StateServer> _stateServer; - std::unique_ptr<TransportServer> _fs4Server; vespalib::ThreadStackExecutor _executor; std::unique_ptr<IProtonDiskLayout> _protonDiskLayout; ProtonConfigurer _protonConfigurer; diff --git a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.h b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.h index 78e3c5d3f3c..d8dc3d5f4db 100644 --- a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.h +++ b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.h @@ -7,7 +7,6 @@ #include <vespa/vespalib/util/closure.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/threadstackexecutor.h> -#include <vespa/searchlib/common/packets.h> #include <vespa/searchlib/engine/proto_rpc_adapter.h> #include <mutex> #include <condition_variable> diff --git a/searchcore/src/vespa/searchcore/util/.gitignore b/searchcore/src/vespa/searchcore/util/.gitignore deleted file mode 100644 index 51f1b16bde5..00000000000 --- a/searchcore/src/vespa/searchcore/util/.gitignore +++ /dev/null @@ -1,14 +0,0 @@ -*.exp -*.ilk -*.lib -*.pdb -.depend -ID -Makefile -extcase -extcase.exe -extprop -extprop.exe -mklicense -result -util.lib diff --git a/searchcore/src/vespa/searchcore/util/CMakeLists.txt b/searchcore/src/vespa/searchcore/util/CMakeLists.txt deleted file mode 100644 index 0d02385be94..00000000000 --- a/searchcore/src/vespa/searchcore/util/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchcore_util STATIC - SOURCES - eventloop.cpp - log.cpp - DEPENDS -) diff --git a/searchcore/src/vespa/searchcore/util/OWNERS b/searchcore/src/vespa/searchcore/util/OWNERS deleted file mode 100644 index 9dc0c2d970d..00000000000 --- a/searchcore/src/vespa/searchcore/util/OWNERS +++ /dev/null @@ -1 +0,0 @@ -baldersheim diff --git a/searchcore/src/vespa/searchcore/util/autoptr.h b/searchcore/src/vespa/searchcore/util/autoptr.h deleted file mode 100644 index ba294e097e4..00000000000 --- a/searchcore/src/vespa/searchcore/util/autoptr.h +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -template <class T> -class FastS_AutoPtr -{ -private: - FastS_AutoPtr(const FastS_AutoPtr &); - FastS_AutoPtr& operator=(const FastS_AutoPtr &); - - T *_val; - void Clean() { - if (_val != NULL) { - delete _val; - _val = NULL; - } - } -public: - FastS_AutoPtr() : _val(NULL) { } - explicit FastS_AutoPtr(T *val) - : _val(val) - { - } - ~FastS_AutoPtr() { Clean(); } - void Set(T *val) { Clean(); _val = val; } - T *Get() const { return _val; } - T *HandOver() { T *ret = _val; _val = NULL; return ret; } - void Drop() { - if (_val != NULL) { - delete _val; - _val = NULL; - } - } -}; - - -template <class T> -class FastS_AutoRefCntPtr -{ -private: - FastS_AutoRefCntPtr(const FastS_AutoRefCntPtr &); - FastS_AutoRefCntPtr& operator=(const FastS_AutoRefCntPtr &); - - T *_val; - void Clean() { - if (_val != NULL) - _val->subRef(); - } -public: - FastS_AutoRefCntPtr() : _val(NULL) { } - explicit FastS_AutoRefCntPtr(T *val) {_val = val; } - ~FastS_AutoRefCntPtr() { Clean(); } - void Set(T *val) { Clean(); _val = val; } - void SetDup(T *val) { - Clean(); - if (val != NULL) - val->addRef(); - _val = val; - } - T *Get() const { return _val; } - T *GetDup() { - if (_val != NULL) - _val->addRef(); - return _val; - } - T *HandOver() { T *ret = _val; _val = NULL; return ret; } - void Drop() { - if (_val != NULL) { - _val->subRef(); - _val = NULL; - } - } -}; - - -class FastS_AutoCharPtr -{ -private: - FastS_AutoCharPtr(const FastS_AutoCharPtr &); - FastS_AutoCharPtr& operator=(const FastS_AutoCharPtr &); - - char *_val; - void Clean() { - if (_val != NULL) - free(_val); - } -public: - FastS_AutoCharPtr() - : _val(NULL) - { - } - explicit FastS_AutoCharPtr(char *val) - : _val(val) - { - } - ~FastS_AutoCharPtr() { Clean(); } - void Set(char *val) { Clean(); _val = val; } - char *Get() const { return _val; } - char *HandOver() { char *ret = _val; _val = NULL; return ret; } - void Drop() { - if (_val != NULL) { - free(_val); - _val = NULL; - } - } -}; - diff --git a/searchcore/src/vespa/searchcore/util/description.html b/searchcore/src/vespa/searchcore/util/description.html deleted file mode 100644 index 025ae212095..00000000000 --- a/searchcore/src/vespa/searchcore/util/description.html +++ /dev/null @@ -1 +0,0 @@ -<!-- Short description for make kdoc. --> diff --git a/searchcore/src/vespa/searchcore/util/eventloop.cpp b/searchcore/src/vespa/searchcore/util/eventloop.cpp deleted file mode 100644 index 3cc9bf3bfc6..00000000000 --- a/searchcore/src/vespa/searchcore/util/eventloop.cpp +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "eventloop.h" -#include <cstdio> - -double FastS_TimeOut::_val[FastS_TimeOut::valCnt]; - -void -FastS_TimeOut::WriteTime(char* buffer, size_t bufsize, double xtime) -{ - snprintf(buffer, bufsize, "%.3fs ", xtime); -} diff --git a/searchcore/src/vespa/searchcore/util/eventloop.h b/searchcore/src/vespa/searchcore/util/eventloop.h deleted file mode 100644 index 0a17ec1287a..00000000000 --- a/searchcore/src/vespa/searchcore/util/eventloop.h +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <cstddef> - -class FastS_TimeOut -{ -public: - enum ValName { - maxSockSilent, // 0 - valCnt // 1 - Must be last, used as array size: - }; - static double _val[valCnt]; - - static void WriteTime(char* buffer, size_t bufsize, double xtime); -}; - - diff --git a/searchcore/src/vespa/searchcore/util/log.cpp b/searchcore/src/vespa/searchcore/util/log.cpp deleted file mode 100644 index fc25fa1f346..00000000000 --- a/searchcore/src/vespa/searchcore/util/log.cpp +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/searchcore/util/log.h> - -#include <vespa/log/log.h> -LOG_SETUP(".searchcore.util.log"); - -/** - * assert and abort functions. - */ -void __FastS_assert_fail(const char *assertion, - const char *file, - unsigned int line, - const char * function) -{ - const char *vtag = V_TAG; - if (function != NULL) { - LOG(error, "FATAL: %s:%d (%s) %s: Failed assertion: '%s'", - file, line, vtag, function, assertion); - fprintf(stderr, "%s:%d (%s) %s: Failed assertion: '%s'\n", - file, line, vtag, function, assertion); - } else { - LOG(error, "FATAL: %s:%d (%s): Failed assertion: '%s'", - file, line, vtag, assertion); - fprintf(stderr, "%s:%d (%s): Failed assertion: '%s'\n", - file, line, vtag, assertion); - } - EV_STOPPING("", "assert failed"); - abort(); -} - -void __FastS_abort(const char *message, - const char *file, - unsigned int line, - const char * function) -{ - const char *vtag = V_TAG; - if (function != NULL) { - LOG(error, "FATAL: %s:%d (%s) %s: Abort called. Reason: %s", - file, line, vtag, function, message); - fprintf(stderr, "%s:%d (%s) %s: Abort called. Reason: %s\n", - file, line, vtag, function, message); - } else { - LOG(error, "FATAL: %s:%d (%s): Abort called. Reason: %s", - file, line, vtag, message); - fprintf(stderr, "%s:%d (%s): Abort called. Reason: %s\n", - file, line, vtag, message); - } - EV_STOPPING("", "aborted"); - abort(); -} diff --git a/searchcore/src/vespa/searchcore/util/log.h b/searchcore/src/vespa/searchcore/util/log.h deleted file mode 100644 index 72aa525a421..00000000000 --- a/searchcore/src/vespa/searchcore/util/log.h +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - - -/* - * Define FastS_abort and FastS_assert macro's. - */ - -/** - * This logs an "assertion failed" message and aborts. - */ -extern void __FastS_assert_fail(const char *assertion, - const char *file, - unsigned int line, - const char * function); - -/** - * This logs an "abort" message and aborts. - */ -extern void __FastS_abort(const char *message, - const char *file, - unsigned int line, - const char * function); - -# ifndef __STRING -# define __STRING(x) #x -# endif - -# ifndef V_TAG -# define V_TAG "NOTAG" -# endif - -# ifndef __ASSERT_FUNCTION -# define __ASSERT_FUNCTION NULL -# endif - - -# define FastS_abort(msg) \ - (__FastS_abort(msg, __FILE__, __LINE__, __ASSERT_FUNCTION), abort()) - -# ifndef NDEBUG -# define FastS_assert(expr) \ - ((void) ((expr) ? 0 : \ - (__FastS_assert_fail (__STRING(expr), \ - __FILE__, __LINE__, \ - __ASSERT_FUNCTION), 0))) -# else -# define FastS_assert(expr) -# endif // #ifndef NDEBUG - diff --git a/searchcore/src/vespa/searchcore/util/stlishheap.h b/searchcore/src/vespa/searchcore/util/stlishheap.h deleted file mode 100644 index 3a04f8f2d39..00000000000 --- a/searchcore/src/vespa/searchcore/util/stlishheap.h +++ /dev/null @@ -1,396 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -/* These algorithms only work for standard C-type arrays, not generic - iterators the way STL works. To make them work for stl-type - iterators, we need to create a set of traits classes, but that is - kind of overkill until we need that functionality. */ - -/* You can use these functions to customize the heap */ - -template<typename T> -inline bool -FastS_min(T a, T b) -{ - return a < b; -} - -template<typename T> -inline bool -FastS_max(T a, T b) -{ - return b < a; -} - -/* Push obj onto the heap first of length len. len must be large - * enough to include the new object. For example if you have a heap - * with 3 elements and want to push a new element onto the heap, len - * should be 4. - */ - -template <typename T, typename Comp> -inline void -FastS_push_heap(T *first, int len, T obj, Comp comp) -{ - int x = len - 1; - int parent = (x - 1)/2; - while (x > 0 && comp(*(first + parent), obj)) { - *(first + x) = *(first + parent); - x = parent; - parent = (x - 1)/2; - } - *(first + x) = obj; -} - -/* Pop the largest element off the heap, reducing the size of the heap - * by 1. (Note: it is the responsibility of the caller to keep track - * of the size of the heap.) - */ - -template<typename T, typename Comp> -inline T -FastS_pop_heap(T *first, int len, Comp comp) -{ - /* The algorithm we use is a variation of the textbook algorithm. - We first remove the first element, then instead of putting the - last element at the top of the heap and heapify(), we propagate - the "hole" left by the removed first element to the bottom. Then - we copy the last element into the hole and push this element - upwards. Since the last element has a high probability of being - pushed down to the bottom anyways, this reduces the number of - comparisons we need to do. */ - T ret = *first; - /* right child */ - int topidx = 0; - int childidx = 2; - /* while both right and left child exist.. */ - while (childidx < len) { - /* compare right to left child */ - if (comp(*(first + childidx), *(first + childidx - 1))) - childidx--; - *(first + topidx) = *(first + childidx); - topidx = childidx; - childidx = 2 * (topidx + 1); - } - /* if only left child exists.. */ - if (childidx == len) { - *(first + topidx) = *(first + childidx - 1); - topidx = childidx - 1; - } - /* now topidx is the hole.. */ - FastS_push_heap(first, topidx + 1, *(first + len - 1), comp); - return ret; -} - -/* Pop the largest element off the heap, and push a new element on the - * heap in the same operation. - */ - -template<typename T, typename Comp> -inline T -FastS_pop_push_heap(T *first, int len, T obj, Comp comp) -{ - T ret = *first; - /* right child */ - int topidx = 0; - int childidx = 2; - /* while both right and left child exist.. */ - while (childidx < len) { - /* compare right to left child */ - if (comp(*(first + childidx), *(first + childidx - 1))) - childidx--; - *(first + topidx) = *(first + childidx); - topidx = childidx; - childidx = 2 * (topidx + 1); - } - /* if only left child exists.. */ - if (childidx == len) { - *(first + topidx) = *(first + childidx - 1); - topidx = childidx - 1; - } - /* now topidx is the hole.. */ - FastS_push_heap(first, topidx + 1, obj, comp); - return ret; -} - -/* Similar to FastS_pop_heap, this function, given a "hole" in the - * heap, heapify()es the heap downwards. It then inserts obj and - * adjusts the heap upwards. - */ - -template<typename T, typename Comp> -inline void -FastS__adjust_heap(T *first, int len, int hole, T obj, Comp comp) -{ - /* right child */ - int topidx = hole; - int childidx = 2 * (hole + 1); - /* while both right and left child exist.. */ - while (childidx < len) { - /* compare right to left child */ - if (comp(*(first + childidx), *(first + childidx - 1))) - childidx--; - *(first + topidx) = *(first + childidx); - topidx = childidx; - childidx = 2 * (topidx + 1); - } - /* if only left child exists.. */ - if (childidx == len) { - *(first + topidx) = *(first + childidx - 1); - topidx = childidx - 1; - } - /* now first[topidx] is the hole.. */ - FastS_push_heap(first, topidx + 1, obj, comp); -} - -template <typename T, typename Comp> -inline void -FastS_make_heap(T *first, int len, Comp comp) -{ - if (len < 2) - return; - int parent = (len - 2)/2; - for (/**/; parent >= 0; parent--) { - int holeidx = parent; - T obj = *(first + parent); - int childidx = 2 * (parent + 1); - while (childidx < len) { - if (comp(*(first + childidx), *(first + childidx - 1))) - childidx--; - if (comp(*(first + childidx), obj)) { - *(first + holeidx) = obj; - goto nextparent; - } else { - *(first + holeidx) = *(first + childidx); - holeidx = childidx; - childidx = 2* (holeidx + 1); - } - } - if (childidx == len) { - if (comp(*(first + childidx - 1), obj)) { - *(first + holeidx) = obj; - } else { - *(first + holeidx) = *(first + childidx - 1); - *(first + childidx - 1) = obj; - } - } else /* childidx > len */ - *(first + holeidx) = obj; - nextparent: - ; - } -} - -template <typename T, typename Comp> -inline void -FastS_sort_heap(T *first, int len, Comp comp) -{ - while (len > 0) { - *(first + len - 1) = FastS_pop_heap(first, len, comp); - len--; - } -} - -template <typename T, typename Comp> -inline bool -FastS_is_heap(T *first, int len, Comp comp) -{ - for (int i = 0; i < len; i++) { - int left = 2 * i + 1; - int right = 2 * i + 2; - if (left < len && comp(*(first + i), *(first + left))) - return false; - if (right < len && comp(*(first + i), *(first + right))) - return false; - } - return true; -} - -//////////////////////////////////////////////////////// -// Similar to the above, but without comparator support -//////////////////////////////////////////////////////// - -template <typename T> -inline void -FastS_push_heap(T *first, int len, T obj) -{ - int x = len - 1; - int parent = (x - 1)/2; - while (x > 0 && *(first + parent) < obj) { - *(first + x) = *(first + parent); - x = parent; - parent = (x - 1)/2; - } - *(first + x) = obj; -} - -/* Pop the largest element off the heap, reducing the size of the heap - * by 1. (Note: it is the responsibility of the caller to keep track - * of the size of the heap.) - */ - -template<typename T> -inline T -FastS_pop_heap(T *first, int len) -{ - /* The algorithm we use is a variation of the textbook algorithm. - We first remove the first element, then instead of putting the - last element at the top of the heap and heapify(), we propagate - the "hole" left by the removed first element to the bottom. Then - we copy the last element into the hole and push this element - upwards. Since the last element has a high probability of being - pushed down to the bottom anyways, this reduces the number of - comparisons we need to do. */ - T ret = *first; - /* right child */ - int topidx = 0; - int childidx = 2; - /* while both right and left child exist.. */ - while (childidx < len) { - /* compare right to left child */ - if (*(first + childidx) < *(first + childidx - 1)) - childidx--; - *(first + topidx) = *(first + childidx); - topidx = childidx; - childidx = 2 * (topidx + 1); - } - /* if only left child exists.. */ - if (childidx == len) { - *(first + topidx) = *(first + childidx - 1); - topidx = childidx - 1; - } - /* now topidx is the hole.. */ - FastS_push_heap(first, topidx + 1, *(first + len - 1)); - return ret; -} - - -/* Pop the largest element off the heap, and push a new element on the - * heap in the same operation. - */ - -template<typename T> -inline T -FastS_pop_push_heap(T *first, int len, T obj) -{ - T ret = *first; - /* right child */ - int topidx = 0; - int childidx = 2; - /* while both right and left child exist.. */ - while (childidx < len) { - /* compare right to left child */ - if (*(first + childidx) < *(first + childidx - 1)) - childidx--; - *(first + topidx) = *(first + childidx); - topidx = childidx; - childidx = 2 * (topidx + 1); - } - /* if only left child exists.. */ - if (childidx == len) { - *(first + topidx) = *(first + childidx - 1); - topidx = childidx - 1; - } - /* now topidx is the hole.. */ - FastS_push_heap(first, topidx + 1, obj); - return ret; -} - - -/* Similar to FastS_pop_heap, this function, given a "hole" in the - * heap, heapify()es the heap downwards. It then inserts obj and - * adjusts the heap upwards. - */ - -template<typename T> -inline void -FastS__adjust_heap(T *first, int len, int hole, T obj) -{ - /* right child */ - int topidx = hole; - int childidx = 2 * (hole + 1); - /* while both right and left child exist.. */ - while (childidx < len) { - /* compare right to left child */ - if (*(first + childidx) < *(first + childidx - 1)) - childidx--; - *(first + topidx) = *(first + childidx); - topidx = childidx; - childidx = 2 * (topidx + 1); - } - /* if only left child exists.. */ - if (childidx == len) { - *(first + topidx) = *(first + childidx - 1); - topidx = childidx - 1; - } - /* now first[topidx] is the hole.. */ - FastS_push_heap(first, topidx + 1, obj); -} - -template <typename T> -inline void -FastS_make_heap(T *first, int len) -{ - if (len < 2) - return; - int parent = (len - 2)/2; - for (/**/; parent >= 0; parent--) { - int holeidx = parent; - T obj = *(first + parent); - int childidx = 2 * (parent + 1); - while (childidx < len) { - // Find largest of left, right child of holeidx, and object. - if (*(first + childidx) < *(first + childidx - 1)) - childidx--; - if (*(first + childidx) < obj) { - *(first + holeidx) = obj; - goto nextparent; - } else { - // If child is largest, put it at holeidx, and - // look further down. - *(first + holeidx) = *(first + childidx); - holeidx = childidx; - childidx = 2* (holeidx + 1); - } - } - if (childidx == len) { - // Only left child exists - if (*(first + childidx - 1) < obj) { - *(first + holeidx) = obj; - } else { - *(first + holeidx) = *(first + childidx - 1); - *(first + childidx - 1) = obj; - } - } else /* childidx > len */ - *(first + holeidx) = obj; - nextparent: - ; - } -} - -template <typename T> -inline void -FastS_sort_heap(T *first, int len) -{ - while (len > 0) { - *(first + len - 1) = FastS_pop_heap(first, len); - len--; - } -} - -template <typename T> -inline bool -FastS_is_heap(T *first, int len) -{ - for (int i = 0; i < len; i++) { - int left = 2 * i + 1; - int right = 2 * i + 2; - if (left < len && *(first + i) < *(first + left)) - return false; - if (right < len && *(first + i) < *(first + right)) - return false; - } - return true; -} - - diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 449580e577b..b78f4fc89ac 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -101,7 +101,6 @@ vespa_define_module( src/tests/common/foregroundtaskexecutor src/tests/common/location src/tests/common/matching_elements - src/tests/common/packets src/tests/common/resultset src/tests/common/sequencedtaskexecutor src/tests/common/struct_field_mapper @@ -119,12 +118,8 @@ vespa_define_module( src/tests/docstore/lid_info src/tests/docstore/logdatastore src/tests/docstore/store_by_bucket - src/tests/engine/docsumapi - src/tests/engine/monitorapi src/tests/engine/proto_converter src/tests/engine/proto_rpc_adapter - src/tests/engine/searchapi - src/tests/engine/transportserver src/tests/expression/attributenode src/tests/features src/tests/features/beta diff --git a/searchlib/src/tests/common/packets/.gitignore b/searchlib/src/tests/common/packets/.gitignore deleted file mode 100644 index e3dcf5376d5..00000000000 --- a/searchlib/src/tests/common/packets/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -packets_test -searchlib_packets_test_app diff --git a/searchlib/src/tests/common/packets/CMakeLists.txt b/searchlib/src/tests/common/packets/CMakeLists.txt deleted file mode 100644 index 12c4b2cd6f5..00000000000 --- a/searchlib/src/tests/common/packets/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_packets_test_app TEST - SOURCES - packets_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_packets_test_app COMMAND searchlib_packets_test_app) diff --git a/searchlib/src/tests/common/packets/packets_test.cpp b/searchlib/src/tests/common/packets/packets_test.cpp deleted file mode 100644 index 35ca7bc1dd9..00000000000 --- a/searchlib/src/tests/common/packets/packets_test.cpp +++ /dev/null @@ -1,663 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/searchlib/common/mapnames.h> -#include <vespa/searchlib/common/packets.h> -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/fnet/controlpacket.h> - -using namespace search::fs4transport; -using vespalib::compression::CompressionConfig; - -#define PCODE_BEGIN PCODE_EOL -#define PCODE_END PCODE_LastCode - -class MyPersistentPacketStreamer : public FS4PersistentPacketStreamer { -public: - MyPersistentPacketStreamer() : - FS4PersistentPacketStreamer(FS4PacketFactory::CreateFS4Packet) { - // empty - } - - uint32_t getChannelId(uint32_t pcode, uint32_t chid) { - return HasChannelID(pcode) ? chid : -1u; - } -}; - -FNET_Packet * -testEncodeDecode(FS4PersistentPacketStreamer &streamer, FNET_Packet &packet) -{ - FNET_Context ctx; - FNET_DataBuffer buf; - buf.WriteInt32(0xdeadbeef); // buffers can have extra data at the front. - streamer.Encode(&packet, 1u, &buf); - buf.DataToDead(sizeof(uint32_t)); - - FNET_DataBuffer lhs; - lhs.WriteBytes(buf.GetData(), buf.GetDataLen()); - - buf.WriteInt32(0xdeadbeef); // buffers can have extra data at the end. - - bool broken; - uint32_t plen, pcode, chid; - MyPersistentPacketStreamer myStreamer; - EXPECT_TRUE(streamer.GetPacketInfo(&buf, &plen, &pcode, &chid, &broken)); - if ((pcode & ~PCODE_MASK) == 0) { - EXPECT_EQUAL(packet.GetLength(), plen); - } - EXPECT_EQUAL(packet.GetPCODE() & PCODE_MASK, pcode & PCODE_MASK); - EXPECT_EQUAL(myStreamer.getChannelId(pcode, 1u), chid); - - FNET_Packet *ret = streamer.Decode(&buf, plen, pcode, ctx); - assert(ret); - if (ret->GetPCODE() == (pcode & PCODE_MASK)) { - FNET_DataBuffer rhs; - streamer.Encode(ret, 1u, &rhs); - if (!EXPECT_TRUE(lhs.Equals(&rhs))) { - lhs.HexDump(); - rhs.HexDump(); - } - } else { - // Packet was transcoded. - } - return ret; -} - -FNET_Packet * -testEncodeDecode(FNET_Packet &packet) -{ - return testEncodeDecode(FS4PersistentPacketStreamer::Instance, packet); -} - -void fillProperties(FS4Properties &props, const std::string &name, - uint32_t len) { - props.setName(name); - props.allocEntries(len); - for (uint32_t i = 0; i < len; ++i) { - std::string key = vespalib::make_string("key%d", i); - props.setKey(i, key); - - std::string val = vespalib::make_string("val%d", i); - props.setValue(i, val); - } -} - -void testProperties(FS4Properties &props, const std::string &name, - uint32_t len) { - EXPECT_EQUAL(name, props.getName()); - EXPECT_EQUAL(name.size(), props.getNameLen()); - for (uint32_t i = 0; i < len; ++i) { - std::string key = vespalib::make_string("key%d", i); - EXPECT_EQUAL(key, std::string(props.getKey(i), props.getKeyLen(i))); - - std::string val = vespalib::make_string("val%d", i); - EXPECT_EQUAL(val, - std::string(props.getValue(i), props.getValueLen(i))); - } -} - - -// ---------------------------------------------------------------------------- -// -// Tests -// -// ---------------------------------------------------------------------------- - -document::GlobalId gid0("aaaaaaaaaaaa"); -document::GlobalId gid1("bbbbbbbbbbbb"); - -TEST("testPacketArray") { - PacketArray arr; - for (uint32_t i = 0; i < 32; ++i) { - EXPECT_EQUAL(i, arr.Length()); - arr.Add(new FNET_ControlPacket(i)); - EXPECT_EQUAL(i, static_cast<FNET_ControlPacket&>(*arr.Array()[i]).GetCommand()); - } - for (uint32_t i = 0; i < arr.Length(); ++i) { - delete static_cast<FNET_ControlPacket *>(arr.Array()[i]); - } -} - -TEST("testPacketFactory") { - ASSERT_TRUE(FS4PacketFactory::CreateFS4Packet(PCODE_BEGIN - 1) == NULL); - - ASSERT_TRUE(FS4PacketFactory::CreateFS4Packet(PCODE_END) == NULL); - - for (uint32_t pcode = PCODE_BEGIN; pcode < PCODE_END; ++pcode) { - if ((pcode != PCODE_MLD_QUERYRESULT2_NOTUSED) && - (pcode != PCODE_QUERY_NOTUSED) && - (pcode != PCODE_MONITORQUERY_NOTUSED) && - (pcode != PCODE_GETDOCSUMS_NOTUSED) && - (pcode != PCODE_MLD_GETDOCSUMS_NOTUSED) && - (pcode != PCODE_QUERYRESULT_NOTUSED) && - (pcode != PCODE_MLD_QUERYRESULT_NOTUSED) && - (pcode != PCODE_MONITORRESULT_NOTUSED) && - (pcode != PCODE_MLD_MONITORRESULT_NOTUSED) && - (pcode != PCODE_CLEARCACHES_NOTUSED) && - (pcode != PCODE_PARSEDQUERY2_NOTUSED) && - (pcode != PCODE_QUEUELEN_NOTUSED) && - (pcode != PCODE_QUERY2_NOTUSED) && - (pcode != PCODE_MLD_GETDOCSUMS2_NOTUSED)) - { - std::unique_ptr<FNET_Packet> aptr(FS4PacketFactory::CreateFS4Packet(pcode)); - ASSERT_TRUE(aptr.get() != NULL); - EXPECT_EQUAL(pcode, aptr->GetPCODE()); - } - } -} - -TEST("testPersistentPacketStreamer") { - for (uint32_t pcode = PCODE_BEGIN; pcode < PCODE_END; ++pcode) { - if ((pcode == PCODE_QUERYX) || - (pcode != PCODE_MLD_QUERYRESULT2_NOTUSED) || - (pcode != PCODE_MLD_GETDOCSUMS2_NOTUSED)) - { - continue; - } - std::unique_ptr<FNET_Packet> arg(FS4PacketFactory::CreateFS4Packet(pcode)); - std::unique_ptr<FNET_Packet> ret(testEncodeDecode(FS4PersistentPacketStreamer::Instance, *arg)); - EXPECT_TRUE(ret.get() != NULL); - - FNET_Packet *raw = testEncodeDecode(FS4PersistentPacketStreamer::Instance, - *FS4PacketFactory::CreateFS4Packet(pcode)); - EXPECT_TRUE(raw != NULL); - } -} - -TEST("testProperties") { - FS4Properties src; - fillProperties(src, "foo", 32u); - testProperties(src, "foo", 32u); - - FNET_DataBuffer buf; - src.encode(buf); - FNET_DataBuffer lhs; - lhs.WriteBytes(buf.GetData(), buf.GetDataLen()); - - uint32_t len = buf.GetDataLen(); - FS4Properties dst; - dst.decode(buf, len); - EXPECT_EQUAL(src.getLength(), dst.getLength()); - - testProperties(dst, "foo", 32u); - - FNET_DataBuffer rhs; - dst.encode(rhs); - EXPECT_TRUE(lhs.Equals(&rhs)); -} - -TEST("testEol") { - FS4Packet_EOL *src = dynamic_cast<FS4Packet_EOL*>(FS4PacketFactory::CreateFS4Packet(PCODE_EOL)); - ASSERT_TRUE(src != NULL); - - std::vector<FNET_Packet*> lst { src, testEncodeDecode(*src) }; - - for (FNET_Packet * packet : lst) { - FS4Packet_EOL *ptr = dynamic_cast<FS4Packet_EOL*>(packet); - ASSERT_TRUE(ptr != NULL); - EXPECT_EQUAL((uint32_t)PCODE_EOL, ptr->GetPCODE()); - EXPECT_EQUAL(0u, ptr->GetLength()); - - delete ptr; - } -} - -TEST("testError") { - FS4Packet_ERROR *src = dynamic_cast<FS4Packet_ERROR*>(FS4PacketFactory::CreateFS4Packet(PCODE_ERROR)); - ASSERT_TRUE(src != NULL); - src->_errorCode = 1u; - src->setErrorMessage("foo"); - - std::vector<FNET_Packet*> lst { src, testEncodeDecode(*src) }; - - for (FNET_Packet * packet : lst) { - FS4Packet_ERROR *ptr = dynamic_cast<FS4Packet_ERROR*>(packet); - ASSERT_TRUE(ptr != NULL); - EXPECT_EQUAL((uint32_t)PCODE_ERROR, ptr->GetPCODE()); - EXPECT_EQUAL(11u, ptr->GetLength()); - EXPECT_EQUAL(1u, ptr->_errorCode); - EXPECT_EQUAL("foo", ptr->_message); - - delete ptr; - } -} - -TEST("testDocsum") { - FS4Packet_DOCSUM *src = dynamic_cast<FS4Packet_DOCSUM*>(FS4PacketFactory::CreateFS4Packet(PCODE_DOCSUM)); - ASSERT_TRUE(src != NULL); - src->setGid(gid0); - src->SetBuf("foo", 3u); - - std::vector<FNET_Packet*> lst { src, testEncodeDecode(*src) }; - - for (FNET_Packet * packet : lst) { - FS4Packet_DOCSUM *ptr = dynamic_cast<FS4Packet_DOCSUM*>(packet); - ASSERT_TRUE(ptr != NULL); - EXPECT_EQUAL((uint32_t)PCODE_DOCSUM, ptr->GetPCODE()); - EXPECT_EQUAL(3u + 12u, ptr->GetLength()); - EXPECT_EQUAL(gid0, ptr->getGid()); - EXPECT_EQUAL("foo", std::string(ptr->getBuf().c_str(), ptr->getBuf().size())); - - delete ptr; - } -} - -TEST("testMonitorQueryX") { - FS4Packet_MONITORQUERYX *src = dynamic_cast<FS4Packet_MONITORQUERYX*>(FS4PacketFactory::CreateFS4Packet(PCODE_MONITORQUERYX)); - ASSERT_TRUE(src != NULL); - src->_qflags = 1u; - - std::vector<FNET_Packet*> lst; - for (uint32_t i = MQF_QFLAGS, len = (uint32_t)(MQF_QFLAGS << 1); i < len; ++i) { - if (i & ~FNET_MQF_SUPPORTED_MASK) { - continue; // not supported; - } - src->_features = i; - lst.push_back(testEncodeDecode(*src)); - } - src->_features = (uint32_t)-1; - lst.push_back(src); - - for (FNET_Packet * packet : lst) { - FS4Packet_MONITORQUERYX *ptr = dynamic_cast<FS4Packet_MONITORQUERYX*>(packet); - ASSERT_TRUE(ptr != NULL); - EXPECT_EQUAL((uint32_t)PCODE_MONITORQUERYX, ptr->GetPCODE()); - EXPECT_EQUAL(ptr->_features & MQF_QFLAGS ? 1u : 0u, ptr->_qflags); - - delete ptr; - } -} - -TEST("testMonitorResultX") { - FS4Packet_MONITORRESULTX *src = dynamic_cast<FS4Packet_MONITORRESULTX*>(FS4PacketFactory::CreateFS4Packet(PCODE_MONITORRESULTX)); - ASSERT_TRUE(src != NULL); - src->_partid = 1u; - src->_timestamp = 2u; - src->_totalNodes = 3u; - src->_activeNodes = 4u; - src->_totalParts = 5u; - src->_activeParts = 6u; - src->_rflags = 7u; - - std::vector<FNET_Packet*> lst; - for (uint32_t i = MRF_MLD, len = (uint32_t)(MRF_RFLAGS << 1); i < len; ++i) { - if (i & ~FNET_MRF_SUPPORTED_MASK) { - continue; // not supported; - } - src->_features = i; - lst.push_back(testEncodeDecode(*src)); - } - src->_features = (uint32_t)-1; - lst.push_back(src); - - for (FNET_Packet * packet : lst) { - FS4Packet_MONITORRESULTX *ptr = dynamic_cast<FS4Packet_MONITORRESULTX*>(packet); - ASSERT_TRUE(ptr != NULL); - EXPECT_EQUAL((uint32_t)PCODE_MONITORRESULTX, ptr->GetPCODE()); - EXPECT_EQUAL(1u, ptr->_partid); - EXPECT_EQUAL(2u, ptr->_timestamp); - EXPECT_EQUAL(ptr->_features & MRF_MLD ? 3u : 0u, ptr->_totalNodes); - EXPECT_EQUAL(ptr->_features & MRF_MLD ? 4u : 0u, ptr->_activeNodes); - EXPECT_EQUAL(ptr->_features & MRF_MLD ? 5u : 0u, ptr->_totalParts); - EXPECT_EQUAL(ptr->_features & MRF_MLD ? 6u : 0u, ptr->_activeParts); - EXPECT_EQUAL(ptr->_features & MRF_RFLAGS ? 7u : 0u, ptr->_rflags); - - delete ptr; - } -} - -TEST("testQueryResultX") { - FS4Packet_QUERYRESULTX *src = dynamic_cast<FS4Packet_QUERYRESULTX*>(FS4PacketFactory::CreateFS4Packet(PCODE_QUERYRESULTX)); - ASSERT_TRUE(src != NULL); - src->_offset = 1u; - src->_totNumDocs = 2u; - src->_maxRank = (search::HitRank)3; - src->setDistributionKey(4u); - src->_coverageDocs = 6u; - src->_activeDocs = 7u; - src->_soonActiveDocs = 8; - src->_coverageDegradeReason = 0x17; - src->setNodesQueried(12); - src->setNodesReplied(11); - uint32_t sortIndex[3] = { 0u, 1u, 3u /* size of data */}; // numDocs + 1 - src->SetSortDataRef(2, sortIndex, "foo"); - src->SetGroupDataRef("baz", 3u); - src->AllocateHits(2); - src->_hits[0]._gid = gid0; - src->_hits[0]._metric = (search::HitRank)2; - src->_hits[0]._partid = 3u; - src->_hits[0].setDistributionKey(4u); - src->_hits[1]._gid = gid1; - src->_hits[1]._metric = (search::HitRank)3; - src->_hits[1]._partid = 4u; - src->_hits[1].setDistributionKey(5u); - - std::vector<FNET_Packet*> lst; - for (uint32_t i = QRF_MLD, len = (uint32_t)(QRF_GROUPDATA << 1); i < len; ++i) { - if (i & ~FNET_QRF_SUPPORTED_MASK) { - continue; // not supported; - } - src->_features = i; - lst.push_back(testEncodeDecode(*src)); - } - src->_features = (uint32_t)-1; - lst.push_back(src); - - for (FNET_Packet * packet : lst) { - FS4Packet_QUERYRESULTX *ptr = dynamic_cast<FS4Packet_QUERYRESULTX*>(packet); - ASSERT_TRUE(ptr != NULL); - EXPECT_EQUAL((uint32_t)PCODE_QUERYRESULTX, ptr->GetPCODE()); - - EXPECT_EQUAL(1u, ptr->_offset); - EXPECT_EQUAL(2u, ptr->_totNumDocs); - EXPECT_EQUAL((search::HitRank)3, ptr->_maxRank); - EXPECT_EQUAL(4u, ptr->getDistributionKey()); - EXPECT_EQUAL(ptr->_features & QRF_COVERAGE_NODES ? 12 : 1u, ptr->getNodesQueried()); - EXPECT_EQUAL(ptr->_features & QRF_COVERAGE_NODES ? 11 : 1u, ptr->getNodesReplied()); - EXPECT_EQUAL(6u, ptr->_coverageDocs); - EXPECT_EQUAL(7u, ptr->_activeDocs); - EXPECT_EQUAL(8u, ptr->_soonActiveDocs); - EXPECT_EQUAL(0x17u, ptr->_coverageDegradeReason); - if (ptr->_features & QRF_SORTDATA) { - EXPECT_EQUAL(0u, ptr->_sortIndex[0]); - EXPECT_EQUAL(1u, ptr->_sortIndex[1]); - EXPECT_EQUAL(3u, ptr->_sortIndex[2]); - EXPECT_EQUAL("foo", std::string(ptr->_sortData, ptr->_sortIndex[2])); - } else { - EXPECT_EQUAL((void*)NULL, ptr->_sortIndex); - EXPECT_EQUAL((void*)NULL, ptr->_sortData); - } - if (ptr->_features & QRF_GROUPDATA) { - EXPECT_EQUAL("baz", std::string(ptr->_groupData, ptr->_groupDataLen)); - } else { - EXPECT_EQUAL(0u, ptr->_groupDataLen); - EXPECT_EQUAL((void*)NULL, ptr->_groupData); - } - EXPECT_EQUAL(2u, ptr->_numDocs); - for (uint32_t i = 0; i < ptr->_numDocs; ++i) { - EXPECT_EQUAL(i == 0 ? gid0 : gid1, ptr->_hits[i]._gid); - EXPECT_EQUAL((search::HitRank)2 + i, ptr->_hits[i]._metric); - EXPECT_EQUAL(ptr->_features & QRF_MLD ? 3u + i : 0u, ptr->_hits[i]._partid); - EXPECT_EQUAL(ptr->_features & QRF_MLD ? 4u + i : ptr->getDistributionKey(), ptr->_hits[i].getDistributionKey()); - } - - delete ptr; - } -} - -FS4Packet_QUERYX * -createAndFill_QUERYX() -{ - FS4Packet_QUERYX *src = dynamic_cast<FS4Packet_QUERYX*>(FS4PacketFactory::CreateFS4Packet(PCODE_QUERYX)); - ASSERT_TRUE(src != NULL); - src->_offset = 2u; - src->_maxhits = 3u; - src->setTimeout(fastos::TimeStamp(4*fastos::TimeStamp::MS)); - EXPECT_EQUAL(fastos::TimeStamp(4*fastos::TimeStamp::MS), src->getTimeout()); - src->setTimeout(fastos::TimeStamp(-4*fastos::TimeStamp::MS)); - EXPECT_EQUAL(0l, src->getTimeout()); - src->setTimeout(fastos::TimeStamp(4*fastos::TimeStamp::MS)); - EXPECT_EQUAL(fastos::TimeStamp(4*fastos::TimeStamp::MS), src->getTimeout()); - src->setQueryFlags(5u); - src->setRanking("seven"); - src->_numStackItems = 14u; - src->_propsVector.resize(2); - fillProperties(src->_propsVector[0], "foo", 8); - fillProperties(src->_propsVector[1], "bar", 16); - src->setSortSpec("sortspec"); - src->setGroupSpec("groupspec"); - src->setLocation("location"); - src->setStackDump("stackdump"); - return src; -} - -void -verifyQueryX(FS4Packet_QUERYX & queryX, uint32_t features) -{ - EXPECT_EQUAL((uint32_t)PCODE_QUERYX, queryX.GetPCODE()); - EXPECT_EQUAL(features, queryX._features); - EXPECT_EQUAL(2u, queryX._offset); - EXPECT_EQUAL(3u, queryX._maxhits); - EXPECT_EQUAL(fastos::TimeStamp(4*fastos::TimeStamp::MS), queryX.getTimeout()); - EXPECT_EQUAL(0x1u, queryX.getQueryFlags()); //Filtered - if (queryX._features & QF_RANKP) { - EXPECT_EQUAL("seven", queryX._ranking); - } else { - EXPECT_EQUAL("", queryX._ranking); - } - EXPECT_EQUAL(queryX._features & QF_PARSEDQUERY ? 14u : 0u, queryX._numStackItems); - if (queryX._features & QF_PROPERTIES) { - EXPECT_EQUAL(2u, queryX._propsVector.size()); - testProperties(queryX._propsVector[0], "foo", 8); - testProperties(queryX._propsVector[1], "bar", 16); - } else { - EXPECT_EQUAL(0u, queryX._propsVector.size()); - } - if (queryX._features & QF_SORTSPEC) { - EXPECT_EQUAL("sortspec", queryX._sortSpec); - } else { - EXPECT_EQUAL(0u, queryX._sortSpec.size()); - } - if (queryX._features & QF_GROUPSPEC) { - EXPECT_EQUAL("groupspec", queryX._groupSpec); - } else { - EXPECT_EQUAL(0u, queryX._groupSpec.size()); - } - if (queryX._features & QF_LOCATION) { - EXPECT_EQUAL("location", queryX._location); - } else { - EXPECT_EQUAL(0u, queryX._location.size()); - } - if (queryX._features & QF_PARSEDQUERY) { - EXPECT_EQUAL("stackdump", queryX._stackDump); - } else { - EXPECT_EQUAL(0u, queryX._stackDump.size()); - } -} - -TEST("testQueryX") { - FS4Packet_QUERYX *src = createAndFill_QUERYX(); - std::vector<std::pair<FNET_Packet*, uint32_t>> lst; - for (uint32_t i = QF_PARSEDQUERY, len = (uint32_t)(QF_GROUPSPEC << 1), skip = 0; i < len; ++i) { - if (!(i & QF_PARSEDQUERY)) { - continue; // skip most - } - if (i & ~FNET_QF_SUPPORTED_MASK) { - continue; // not supported - } - if (++skip % 10) { - continue; // skip most - } - src->_features = i; - lst.emplace_back(testEncodeDecode(*src), i); - } - src->_features = uint32_t(-1); - lst.emplace_back(src, -1); - - for (const auto & pfPair : lst) { - FS4Packet_QUERYX *ptr = dynamic_cast<FS4Packet_QUERYX*>(pfPair.first); - ASSERT_TRUE(ptr != NULL); - verifyQueryX(*ptr, pfPair.second); - - delete ptr; - } -} - -TEST("testSharedPacket") { - FNET_Packet::SP src(createAndFill_QUERYX()); - static_cast<FS4Packet_QUERYX *>(src.get())->_features=FNET_QF_SUPPORTED_MASK; - FNET_Packet::SP decoded(testEncodeDecode(*src)); - verifyQueryX(*static_cast<FS4Packet_QUERYX *>(decoded.get()), FNET_QF_SUPPORTED_MASK); - EXPECT_TRUE(decoded.get() != nullptr); - FS4Packet_Shared shared(decoded); - FNET_Packet::UP decoded2(testEncodeDecode(shared)); - EXPECT_TRUE(decoded2.get() != nullptr); - EXPECT_TRUE(nullptr == dynamic_cast<const FS4Packet_Shared *>(decoded2.get())); - EXPECT_TRUE(nullptr != dynamic_cast<const FS4Packet_QUERYX *>(decoded2.get())); - EXPECT_EQUAL(src->GetLength(), decoded2->GetLength()); - verifyQueryX(*static_cast<FS4Packet_QUERYX *>(decoded2.get()), FNET_QF_SUPPORTED_MASK); -} - -TEST("test pre serializing packets no compression") { - FNET_Packet::UP src(createAndFill_QUERYX()); - FS4Packet_QUERYX * queryX = static_cast<FS4Packet_QUERYX *>(src.get()); - queryX->_features=FNET_QF_SUPPORTED_MASK; - FNET_Packet::UP decoded(testEncodeDecode(*src)); - verifyQueryX(*static_cast<FS4Packet_QUERYX *>(decoded.get()), FNET_QF_SUPPORTED_MASK); - EXPECT_EQUAL(500u, src->GetLength()); - EXPECT_EQUAL(src->GetLength(), decoded->GetLength()); - FS4Packet_PreSerialized serialized(*src); - EXPECT_EQUAL(218u, serialized.GetPCODE()); - EXPECT_EQUAL(500u, serialized.GetLength()); - FNET_Packet::UP decoded2(testEncodeDecode(serialized)); - EXPECT_EQUAL(500u, decoded2->GetLength()); - verifyQueryX(*static_cast<FS4Packet_QUERYX *>(decoded2.get()), FNET_QF_SUPPORTED_MASK); -} - -TEST("test pre serializing packets with compression") { - FNET_Packet::UP src(createAndFill_QUERYX()); - FS4Packet_QUERYX * queryX = static_cast<FS4Packet_QUERYX *>(src.get()); - queryX->_features=FNET_QF_SUPPORTED_MASK; - FNET_Packet::UP decoded(testEncodeDecode(*src)); - verifyQueryX(*static_cast<FS4Packet_QUERYX *>(decoded.get()), FNET_QF_SUPPORTED_MASK); - EXPECT_EQUAL(500u, src->GetLength()); - EXPECT_EQUAL(src->GetLength(), decoded->GetLength()); - FS4PersistentPacketStreamer::Instance.SetCompressionLimit(100); - FS4Packet_PreSerialized serialized(*src); - EXPECT_EQUAL(218u | (CompressionConfig::LZ4 << 24), serialized.GetPCODE()); - EXPECT_GREATER_EQUAL(321u, serialized.GetLength()); - FNET_Packet::UP decoded2(testEncodeDecode(serialized)); - EXPECT_EQUAL(500u, decoded2->GetLength()); - verifyQueryX(*static_cast<FS4Packet_QUERYX *>(decoded2.get()), FNET_QF_SUPPORTED_MASK); -} - - -TEST("testGetDocsumsX") { - FS4Packet_GETDOCSUMSX *src = dynamic_cast<FS4Packet_GETDOCSUMSX*>(FS4PacketFactory::CreateFS4Packet(PCODE_GETDOCSUMSX)); - ASSERT_TRUE(src != NULL); - src->setTimeout(fastos::TimeStamp(2*fastos::TimeStamp::MS)); - src->setRanking("four"); - src->_qflags = 5u; - src->_stackItems = 7u; - src->_propsVector.resize(2); - fillProperties(src->_propsVector[0], "foo", 8); - fillProperties(src->_propsVector[1], "bar", 16); - src->setResultClassName("resultclassname"); - src->setStackDump("stackdump"); - src->setLocation("location"); - src->_flags = GDFLAG_IGNORE_ROW; - src->AllocateDocIDs(2); - src->_docid[0]._gid = gid0; - src->_docid[0]._partid = 2u; - src->_docid[1]._gid = gid1; - src->_docid[1]._partid = 3u; - - std::vector<std::pair<FNET_Packet*, uint32_t>> lst; - for (uint32_t i = GDF_MLD, len = (uint32_t)(GDF_FLAGS << 1); i < len; ++i) { - if (i & ~FNET_GDF_SUPPORTED_MASK) { - continue; // not supported - } - src->_features = i; - lst.emplace_back(testEncodeDecode(*src), i); - } - src->_features = uint32_t(-1); - lst.emplace_back(src, uint32_t(-1)); - - for (const auto & pfPair : lst) { - FS4Packet_GETDOCSUMSX *ptr = dynamic_cast<FS4Packet_GETDOCSUMSX*>(pfPair.first); - ASSERT_TRUE(ptr != NULL); - EXPECT_EQUAL((uint32_t)PCODE_GETDOCSUMSX, ptr->GetPCODE()); - EXPECT_EQUAL(pfPair.second, ptr->_features); - EXPECT_EQUAL(fastos::TimeStamp(2*fastos::TimeStamp::MS), ptr->getTimeout()); - if (ptr->_features & GDF_RANKP_QFLAGS) { - EXPECT_EQUAL("four", ptr->_ranking); - } else { - EXPECT_EQUAL("", ptr->_ranking); - } - EXPECT_EQUAL(ptr->_features & GDF_RANKP_QFLAGS ? 5u : 0u, ptr->_qflags); - EXPECT_EQUAL(ptr->_features & GDF_QUERYSTACK ? 7u : 0u, ptr->_stackItems); - if (ptr->_features & GDF_PROPERTIES) { - EXPECT_EQUAL(2u, ptr->_propsVector.size()); - testProperties(ptr->_propsVector[0], "foo", 8); - testProperties(ptr->_propsVector[1], "bar", 16); - } else { - EXPECT_EQUAL(0u, ptr->_propsVector.size()); - } - if (ptr->_features & GDF_RESCLASSNAME) { - EXPECT_EQUAL("resultclassname", ptr->_resultClassName); - } else { - EXPECT_EQUAL(0u, ptr->_resultClassName.size()); - } - if (ptr->_features & GDF_QUERYSTACK) { - EXPECT_EQUAL("stackdump", ptr->_stackDump); - } else { - EXPECT_EQUAL(0u, ptr->_stackDump.size()); - } - if (ptr->_features & GDF_LOCATION) { - EXPECT_EQUAL("location", ptr->_location); - } else { - EXPECT_EQUAL(0u, ptr->_location.size()); - } - if (ptr->_features & GDF_FLAGS) { - EXPECT_EQUAL(static_cast<uint32_t>(GDFLAG_IGNORE_ROW), - ptr->_flags); - } else { - EXPECT_EQUAL(0u, ptr->_flags); - } - EXPECT_EQUAL(2u, ptr->_docid.size()); - for (uint32_t i = 0; i < ptr->_docid.size(); ++i) { - EXPECT_EQUAL(i == 0u ? gid0 : gid1, ptr->_docid[i]._gid); - EXPECT_EQUAL(ptr->_features & GDF_MLD ? 2u + i : 0u, ptr->_docid[i]._partid); - } - - delete ptr; - } -} - -TEST("require that FS4PersistentPacketStreamer can compress packets") { - FS4Packet_ERROR *packet = static_cast<FS4Packet_ERROR*>(FS4PacketFactory::CreateFS4Packet(PCODE_ERROR)); - packet->_errorCode = 1u; - packet->setErrorMessage(string(1000, 'a')); - - FS4PersistentPacketStreamer streamer(FS4PacketFactory::CreateFS4Packet); - - FNET_DataBuffer buf1; - streamer.Encode(packet, 1u, &buf1); - EXPECT_EQUAL(1020u, buf1.GetDataLen()); - - streamer.SetCompressionLimit(100); - FNET_DataBuffer buf2; - streamer.Encode(packet, 1u, &buf2); - EXPECT_EQUAL(38u, buf2.GetDataLen()); - - std::vector<FNET_Packet*> lst{ packet, testEncodeDecode(streamer, *packet) }; - - for (FNET_Packet * fnetPacket : lst) { - FS4Packet_ERROR *ptr = dynamic_cast<FS4Packet_ERROR*>(fnetPacket); - ASSERT_TRUE(ptr != NULL); - EXPECT_EQUAL((uint32_t)PCODE_ERROR, ptr->GetPCODE()); - EXPECT_EQUAL(1008u, ptr->GetLength()); - delete ptr; - } -} - -TEST("require that FS4PersistentPacketStreamer can avoid compressing small packets") { - FS4Packet_ERROR *packet = static_cast<FS4Packet_ERROR*>(FS4PacketFactory::CreateFS4Packet(PCODE_ERROR)); - packet->_errorCode = 1u; - packet->setErrorMessage("a"); - - FS4PersistentPacketStreamer streamer(FS4PacketFactory::CreateFS4Packet); - - FNET_DataBuffer buf1; - streamer.Encode(packet, 1u, &buf1); - EXPECT_EQUAL(21u, buf1.GetDataLen()); - - streamer.SetCompressionLimit(10); - FNET_DataBuffer buf2; - streamer.Encode(packet, 1u, &buf2); - EXPECT_EQUAL(21u, buf2.GetDataLen()); - - delete packet; -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp index d636de46426..63c1b320fb8 100644 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp @@ -131,6 +131,7 @@ checkStats(IDataStore &store, } +#ifdef __linux__ TEST("test that DirectIOPadding works accordng to spec") { constexpr ssize_t FILE_SIZE = 4096*3; FastOS_File file("directio.test"); @@ -199,6 +200,7 @@ TEST("test that DirectIOPadding works accordng to spec") { EXPECT_TRUE(file.Close()); FastOS_File::Delete(file.GetFileName()); } +#endif TEST("testGrowing") { FastOS_File::EmptyAndRemoveDirectory("growing"); @@ -287,6 +289,7 @@ TEST("testTruncatedIdxFile"){ } const char * magic = "mumbo jumbo"; { + truncate("bug-7257706-truncated/1422358701368384000.idx", 3830); LogDataStore datastore(executor, "bug-7257706-truncated", config, GrowStrategy(), TuneFileSummary(), fileHeaderContext, tlSyncer, nullptr); EXPECT_EQUAL(331ul, datastore.lastSyncToken()); @@ -843,7 +846,7 @@ struct Fixture { Fixture(const vespalib::string &dirName = "tmp", bool dirCleanup = true, size_t maxFileSize = 4096 * 2) - : executor(1, 0x10000), + : executor(1, 0x20000), dir(dirName), serialNum(0), fileHeaderCtx(), diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.sh b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.sh index 45b3c804014..4aeea02efe2 100755 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.sh +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.sh @@ -23,7 +23,6 @@ cp -a $SOURCE_DIRECTORY/bug-7257706/1422358701368384000.idx incompletecompact-te cp -a $SOURCE_DIRECTORY/bug-7257706/1422358701368384000.dat incompletecompact-test/2422358701368384000.dat cp -a $SOURCE_DIRECTORY/bug-7257706/1422358701368384000.idx incompletecompact-test/2422358701368384000.idx -truncate --size 3830 bug-7257706-truncated/1422358701368384000.idx fail=0 VESPA_LOG_TARGET=file:vlog2.txt $VALGRIND ./searchlib_logdatastore_test_app || fail=1 rm -rf bug-7257706-truncated dangling-test incompletecompact-test diff --git a/searchlib/src/tests/engine/docsumapi/.gitignore b/searchlib/src/tests/engine/docsumapi/.gitignore deleted file mode 100644 index 1b38a4ff745..00000000000 --- a/searchlib/src/tests/engine/docsumapi/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -docsumapi_test -searchlib_docsumapi_test_app diff --git a/searchlib/src/tests/engine/docsumapi/CMakeLists.txt b/searchlib/src/tests/engine/docsumapi/CMakeLists.txt deleted file mode 100644 index d1acc996848..00000000000 --- a/searchlib/src/tests/engine/docsumapi/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_docsumapi_test_app TEST - SOURCES - docsumapi_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_docsumapi_test_app COMMAND searchlib_docsumapi_test_app) diff --git a/searchlib/src/tests/engine/docsumapi/docsumapi_test.cpp b/searchlib/src/tests/engine/docsumapi/docsumapi_test.cpp deleted file mode 100644 index 612e4622373..00000000000 --- a/searchlib/src/tests/engine/docsumapi/docsumapi_test.cpp +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("docsumapi_test"); -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/common/packets.h> -#include <vespa/searchlib/engine/docsumapi.h> -#include <vespa/searchlib/engine/packetconverter.h> - -using namespace search::engine; -using namespace search::fs4transport; - -namespace { - -// light-weight network hop simulation -template <typename T> void copyPacket(T &src, T &dst) { - FNET_DataBuffer buf; - src.Encode(&buf); - dst.Decode(&buf, buf.GetDataLen()); -} - -} // namespace <unnamed> - -class Test : public vespalib::TestApp -{ -public: - void convertToRequest(); - void convertFromReply(); - int Main() override; -}; - -document::GlobalId gid0("aaaaaaaaaaaa"); -document::GlobalId gid1("bbbbbbbbbbbb"); - -void -Test::convertToRequest() -{ - const string sessionId("qrserver.0.XXXXXXXXXXXXX.0"); - - FS4Packet_GETDOCSUMSX src; - src.setTimeout(fastos::TimeStamp(4*fastos::TimeStamp::MS)); - src._features |= GDF_RANKP_QFLAGS; - src.setRanking("seven"); - src._qflags = 5u; - src._features |= GDF_RESCLASSNAME; - src.setResultClassName("resclass"); - src._features |= GDF_PROPERTIES; - src._propsVector.resize(3); - src._propsVector[0].allocEntries(2); - src._propsVector[0].setName("feature", strlen("feature")); - src._propsVector[0].setKey(0, "p1k1", strlen("p1k1")); - src._propsVector[0].setValue(0, "p1v1", strlen("p1v1")); - src._propsVector[0].setKey(1, "p1k2", strlen("p1k2")); - src._propsVector[0].setValue(1, "p1v2", strlen("p1v2")); - src._propsVector[1].allocEntries(2); - src._propsVector[1].setName("caches", strlen("caches")); - src._propsVector[1].setKey(0, "p2k1", strlen("p2k1")); - src._propsVector[1].setValue(0, "p2v1", strlen("p2v1")); - src._propsVector[1].setKey(1, "p2k2", strlen("p2k2")); - src._propsVector[1].setValue(1, "p2v2", strlen("p2v2")); - src._propsVector[2].allocEntries(1); - src._propsVector[2].setName("rank", strlen("rank")); - src._propsVector[2].setKey(0, "sessionId", strlen("sessionId")); - src._propsVector[2].setValue(0, sessionId.c_str(), sessionId.size()); - src._features |= GDF_QUERYSTACK; - src._stackItems = 14u; - src.setStackDump("stackdump"); - src._features |= GDF_LOCATION; - src.setLocation("location"); - src._features |= GDF_MLD; - src.AllocateDocIDs(2); - src._docid[0]._gid = gid0; - src._docid[0]._partid = 5; - src._docid[1]._gid = gid1; - src._docid[1]._partid = 6; - - { // full copy - FS4Packet_GETDOCSUMSX cpy; - copyPacket(src, cpy); - - DocsumRequest dst; - PacketConverter::toDocsumRequest(cpy, dst); - EXPECT_EQUAL((dst.getTimeOfDoom() - dst.getStartTime()).ms(), 4u); - EXPECT_EQUAL(dst.ranking, "seven"); - EXPECT_EQUAL(dst.queryFlags, 5u); - EXPECT_EQUAL(dst.resultClassName, "resclass"); - EXPECT_EQUAL(dst.propertiesMap.size(), 3u); - EXPECT_EQUAL(dst.propertiesMap.featureOverrides().lookup("p1k1").get(), std::string("p1v1")); - EXPECT_EQUAL(dst.propertiesMap.featureOverrides().lookup("p1k2").get(), std::string("p1v2")); - EXPECT_EQUAL(dst.propertiesMap.cacheProperties().lookup("p2k1").get(), std::string("p2v1")); - EXPECT_EQUAL(dst.propertiesMap.cacheProperties().lookup("p2k2").get(), std::string("p2v2")); - EXPECT_EQUAL(dst.propertiesMap.matchProperties().lookup("p3k1").get(), std::string("")); - EXPECT_EQUAL(std::string(&dst.stackDump[0], dst.stackDump.size()), "stackdump"); - EXPECT_EQUAL(dst.location, "location"); - EXPECT_EQUAL(dst._flags, 0u); - EXPECT_EQUAL(dst.hits.size(), 2u); - EXPECT_EQUAL(dst.hits[0].docid, 0u); - EXPECT_TRUE(dst.hits[0].gid == gid0); - EXPECT_EQUAL(dst.hits[0].path, 5u); - EXPECT_EQUAL(dst.hits[1].docid, 0u); - EXPECT_TRUE(dst.hits[1].gid == gid1); - EXPECT_EQUAL(dst.hits[1].path, 6u); - EXPECT_EQUAL(sessionId, - string(&dst.sessionId[0], dst.sessionId.size())); - } - { // without datetime - FS4Packet_GETDOCSUMSX cpy; - copyPacket(src, cpy); - - DocsumRequest dst; - PacketConverter::toDocsumRequest(cpy, dst); - } - { // without mld - FS4Packet_GETDOCSUMSX cpy; - copyPacket(src, cpy); - cpy._features &= ~GDF_MLD; - - DocsumRequest dst; - PacketConverter::toDocsumRequest(cpy, dst); - EXPECT_EQUAL(dst.useWideHits, false); - EXPECT_EQUAL(dst.hits.size(), 2u); - EXPECT_EQUAL(dst.hits[0].docid, 0u); - EXPECT_TRUE(dst.hits[0].gid == gid0); - EXPECT_EQUAL(dst.hits[1].docid, 0u); - EXPECT_TRUE(dst.hits[1].gid == gid1); - } - { // with ignore row flag - FS4Packet_GETDOCSUMSX tcpy; - copyPacket(src, tcpy); - tcpy._features |= GDF_FLAGS; - tcpy._flags = GDFLAG_IGNORE_ROW; - FS4Packet_GETDOCSUMSX cpy; - copyPacket(tcpy, cpy); - DocsumRequest dst; - PacketConverter::toDocsumRequest(cpy, dst); - EXPECT_EQUAL(dst._flags, static_cast<uint32_t>(GDFLAG_IGNORE_ROW)); - } -} - -void -Test::convertFromReply() -{ - DocsumReply src; - src.docsums.resize(2); - src.docsums[0].docid = 1; - src.docsums[0].gid = gid0; - src.docsums[0].data.resize(2); - src.docsums[0].data.str()[0] = 5; - src.docsums[0].data.str()[1] = 6; - src.docsums[1].docid = 2; - src.docsums[1].gid = gid1; - src.docsums[1].data.resize(3); - src.docsums[1].data.str()[0] = 7; - src.docsums[1].data.str()[1] = 8; - src.docsums[1].data.str()[2] = 9; - - { // test first - FS4Packet_DOCSUM dst; - PacketConverter::fromDocsumReplyElement(src.docsums[0], dst); - EXPECT_EQUAL(dst.getGid(), gid0); - EXPECT_EQUAL(dst.getBuf().size(), 2u); - EXPECT_EQUAL(dst.getBuf().c_str()[0], 5); - EXPECT_EQUAL(dst.getBuf().c_str()[1], 6); - } - { // test second - FS4Packet_DOCSUM dst; - PacketConverter::fromDocsumReplyElement(src.docsums[1], dst); - EXPECT_EQUAL(dst.getGid(), gid1); - EXPECT_EQUAL(dst.getBuf().size(), 3u); - EXPECT_EQUAL(dst.getBuf().c_str()[0], 7); - EXPECT_EQUAL(dst.getBuf().c_str()[1], 8); - EXPECT_EQUAL(dst.getBuf().c_str()[2], 9); - } -} - -int -Test::Main() -{ - TEST_INIT("docsumapi_test"); - convertToRequest(); - convertFromReply(); - TEST_DONE(); -} - -TEST_APPHOOK(Test); diff --git a/searchlib/src/tests/engine/monitorapi/.gitignore b/searchlib/src/tests/engine/monitorapi/.gitignore deleted file mode 100644 index 66fc005087f..00000000000 --- a/searchlib/src/tests/engine/monitorapi/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -monitorapi_test -searchlib_monitorapi_test_app diff --git a/searchlib/src/tests/engine/monitorapi/CMakeLists.txt b/searchlib/src/tests/engine/monitorapi/CMakeLists.txt deleted file mode 100644 index 3fea05f4841..00000000000 --- a/searchlib/src/tests/engine/monitorapi/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_monitorapi_test_app TEST - SOURCES - monitorapi_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_monitorapi_test_app COMMAND searchlib_monitorapi_test_app) diff --git a/searchlib/src/tests/engine/monitorapi/monitorapi_test.cpp b/searchlib/src/tests/engine/monitorapi/monitorapi_test.cpp deleted file mode 100644 index 27b00b8a074..00000000000 --- a/searchlib/src/tests/engine/monitorapi/monitorapi_test.cpp +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("monitorapi_test"); -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/common/packets.h> -#include <vespa/searchlib/engine/monitorapi.h> -#include <vespa/searchlib/engine/packetconverter.h> - -using namespace search::engine; -using namespace search::fs4transport; - -namespace { - -bool checkFeature(uint32_t features, uint32_t mask) { - return ((features & mask) != 0); -} - -bool checkNotFeature(uint32_t features, uint32_t mask) { - return !checkFeature(features, mask); -} - -// light-weight network hop simulation -template <typename T> void copyPacket(T &src, T &dst) { - FNET_DataBuffer buf; - src.Encode(&buf); - dst.Decode(&buf, buf.GetDataLen()); -} - -} // namespace <unnamed> - -class Test : public vespalib::TestApp -{ -public: - void convertToRequest(); - void convertFromReply(); - int Main() override; -}; - -void -Test::convertToRequest() -{ - FS4Packet_MONITORQUERYX src; - src._features |= MQF_QFLAGS; - src._qflags = 1u; - - { // copy all - FS4Packet_MONITORQUERYX cpy; - copyPacket(src, cpy); - - MonitorRequest dst; - PacketConverter::toMonitorRequest(cpy, dst); - EXPECT_EQUAL(dst.flags, 1u); - } -} - -void -Test::convertFromReply() -{ - MonitorReply src; - src.mld = true; - src.partid = 1u; - src.timestamp = 2u; - src.totalNodes = 3u; - src.activeNodes = 4u; - src.totalParts = 5u; - src.activeParts = 6u; - src.flags = 7u; - src.activeDocs = 8u; - src.activeDocsRequested = true; - - { // full copy - MonitorReply cpy = src; - - FS4Packet_MONITORRESULTX dst; - PacketConverter::fromMonitorReply(cpy, dst); - EXPECT_EQUAL(dst._partid, 1u); - EXPECT_EQUAL(dst._timestamp, 2u); - EXPECT_TRUE(checkFeature(dst._features, MRF_MLD)); - EXPECT_EQUAL(dst._totalNodes, 3u); - EXPECT_EQUAL(dst._activeNodes, 4u); - EXPECT_EQUAL(dst._totalParts, 5u); - EXPECT_EQUAL(dst._activeParts, 6u); - EXPECT_TRUE(checkFeature(dst._features, MRF_RFLAGS)); - EXPECT_EQUAL(dst._rflags, 7u); - EXPECT_EQUAL(dst._activeDocs, 8u); - EXPECT_TRUE(checkFeature(dst._features, MRF_ACTIVEDOCS)); - } - { // non-mld - MonitorReply cpy = src; - cpy.mld = false; - - FS4Packet_MONITORRESULTX dst; - PacketConverter::fromMonitorReply(cpy, dst); - EXPECT_TRUE(checkNotFeature(dst._features, MRF_MLD)); - } - { // without flags - MonitorReply cpy = src; - cpy.flags = 0; - - FS4Packet_MONITORRESULTX dst; - PacketConverter::fromMonitorReply(cpy, dst); - EXPECT_TRUE(checkNotFeature(dst._features, MRF_RFLAGS)); - EXPECT_EQUAL(dst._rflags, 0u); - } - { // without activedocs - MonitorReply cpy = src; - cpy.activeDocsRequested = false; - - FS4Packet_MONITORRESULTX dst; - PacketConverter::fromMonitorReply(cpy, dst); - EXPECT_TRUE(checkNotFeature(dst._features, MRF_ACTIVEDOCS)); - EXPECT_EQUAL(dst._activeDocs, 0u); - } -} - -int -Test::Main() -{ - TEST_INIT("monitorapi_test"); - convertToRequest(); - convertFromReply(); - TEST_DONE(); -} - -TEST_APPHOOK(Test); diff --git a/searchlib/src/tests/engine/proto_converter/proto_converter_test.cpp b/searchlib/src/tests/engine/proto_converter/proto_converter_test.cpp index e38820b6e8b..7526326b6ca 100644 --- a/searchlib/src/tests/engine/proto_converter/proto_converter_test.cpp +++ b/searchlib/src/tests/engine/proto_converter/proto_converter_test.cpp @@ -2,7 +2,6 @@ #include <vespa/vespalib/gtest/gtest.h> #include <vespa/searchlib/engine/proto_converter.h> -#include <vespa/searchlib/common/transport.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/slime/binary_format.h> @@ -343,14 +342,14 @@ TEST_F(DocsumRequestTest, require_that_cache_query_is_converted) { proto.set_cache_query(true); convert(); EXPECT_TRUE(request.propertiesMap.cacheProperties().lookup("query").found()); - EXPECT_FALSE((request.queryFlags & search::fs4transport::QFLAG_DUMP_FEATURES) != 0); + EXPECT_FALSE(request.dumpFeatures); } TEST_F(DocsumRequestTest, require_that_dump_features_is_converted) { proto.set_dump_features(true); convert(); EXPECT_FALSE(request.propertiesMap.cacheProperties().lookup("query").found()); - EXPECT_TRUE((request.queryFlags & search::fs4transport::QFLAG_DUMP_FEATURES) != 0); + EXPECT_TRUE(request.dumpFeatures); } TEST_F(DocsumRequestTest, require_that_rank_profile_is_converted) { diff --git a/searchlib/src/tests/engine/searchapi/.gitignore b/searchlib/src/tests/engine/searchapi/.gitignore deleted file mode 100644 index 92089e63cdd..00000000000 --- a/searchlib/src/tests/engine/searchapi/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -searchapi_test -searchlib_searchapi_test_app diff --git a/searchlib/src/tests/engine/searchapi/CMakeLists.txt b/searchlib/src/tests/engine/searchapi/CMakeLists.txt deleted file mode 100644 index c4e3de1280a..00000000000 --- a/searchlib/src/tests/engine/searchapi/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_searchapi_test_app TEST - SOURCES - searchapi_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_searchapi_test_app COMMAND searchlib_searchapi_test_app) diff --git a/searchlib/src/tests/engine/searchapi/searchapi_test.cpp b/searchlib/src/tests/engine/searchapi/searchapi_test.cpp deleted file mode 100644 index 99d1967b0df..00000000000 --- a/searchlib/src/tests/engine/searchapi/searchapi_test.cpp +++ /dev/null @@ -1,276 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/common/packets.h> -#include <vespa/searchlib/engine/searchapi.h> -#include <vespa/searchlib/engine/packetconverter.h> -#include <vespa/vespalib/data/slime/slime.h> -#include <vespa/log/log.h> -LOG_SETUP("searchapi_test"); - -using namespace search::engine; -using namespace search::fs4transport; - -namespace { - -bool checkFeature(uint32_t features, uint32_t mask) { - return ((features & mask) != 0); -} - -bool checkNotFeature(uint32_t features, uint32_t mask) { - return !checkFeature(features, mask); -} - -// light-weight network hop simulation -template <typename T> void copyPacket(T &src, T &dst) { - FNET_DataBuffer buf; - src.Encode(&buf); - dst.Decode(&buf, buf.GetDataLen()); -} - -} // namespace <unnamed> - - -TEST("propertyNames") { - EXPECT_EQUAL(search::MapNames::RANK, "rank"); - EXPECT_EQUAL(search::MapNames::FEATURE, "feature"); - EXPECT_EQUAL(search::MapNames::HIGHLIGHTTERMS, "highlightterms"); - EXPECT_EQUAL(search::MapNames::MATCH, "match"); - EXPECT_EQUAL(search::MapNames::CACHES, "caches"); - EXPECT_EQUAL(search::MapNames::TRACE, "trace"); -} - -TEST("convertToReques") { - FS4Packet_QUERYX src; - src._offset = 2u; - src._maxhits = 3u; - src.setTimeout(fastos::TimeStamp(4*fastos::TimeStamp::MS)); - src.setQueryFlags(5u); - src._features |= QF_RANKP; - src.setRanking("seven"); - src._features |= QF_PROPERTIES; - src._propsVector.resize(2); - src._propsVector[0].allocEntries(2); - src._propsVector[0].setName("feature", strlen("feature")); - src._propsVector[0].setKey(0, "p1k1", strlen("p1k1")); - src._propsVector[0].setValue(0, "p1v1", strlen("p1v1")); - src._propsVector[0].setKey(1, "p1k2", strlen("p1k2")); - src._propsVector[0].setValue(1, "p1v2", strlen("p1v2")); - src._propsVector[1].allocEntries(2); - src._propsVector[1].setName("caches", strlen("caches")); - src._propsVector[1].setKey(0, "p2k1", strlen("p2k1")); - src._propsVector[1].setValue(0, "p2v1", strlen("p2v1")); - src._propsVector[1].setKey(1, "p2k2", strlen("p2k2")); - src._propsVector[1].setValue(1, "p2v2", strlen("p2v2")); - src._features |= QF_SORTSPEC; - src.setSortSpec("sortspec"); - src._features |= QF_GROUPSPEC; - src.setGroupSpec("groupspec"); - src._features |= QF_SESSIONID; - src.setSessionId("sessionid"); - src._features |= QF_LOCATION; - src.setLocation("location"); - src._features |= QF_PARSEDQUERY; - src._numStackItems = 14u; - src.setStackDump("stackdump"); - - { // full copy - FS4Packet_QUERYX cpy; - copyPacket(src, cpy); - - SearchRequest dst; - PacketConverter::toSearchRequest(cpy, dst); - EXPECT_EQUAL(dst.offset, 2u); - EXPECT_EQUAL(dst.maxhits, 3u); - EXPECT_EQUAL((dst.getTimeOfDoom() - dst.getStartTime()).ms(), 4u); - EXPECT_EQUAL(dst.queryFlags, 1u); //Filtered - EXPECT_EQUAL(vespalib::string("seven"), dst.ranking); - EXPECT_EQUAL(dst.propertiesMap.size(), 2u); - EXPECT_EQUAL(dst.propertiesMap.featureOverrides().lookup("p1k1").get(), std::string("p1v1")); - EXPECT_EQUAL(dst.propertiesMap.featureOverrides().lookup("p1k2").get(), std::string("p1v2")); - EXPECT_EQUAL(dst.propertiesMap.cacheProperties().lookup("p2k1").get(), std::string("p2v1")); - EXPECT_EQUAL(dst.propertiesMap.cacheProperties().lookup("p2k2").get(), std::string("p2v2")); - EXPECT_EQUAL(dst.propertiesMap.matchProperties().lookup("p3k1").get(), std::string("")); - EXPECT_EQUAL(dst.sortSpec, "sortspec"); - EXPECT_EQUAL(std::string(&dst.groupSpec[0], dst.groupSpec.size()), "groupspec"); - EXPECT_EQUAL(std::string(&dst.sessionId[0], dst.sessionId.size()), "sessionid"); - EXPECT_EQUAL(dst.location, "location"); - EXPECT_EQUAL(dst.stackItems, 14u); - EXPECT_EQUAL(std::string(&dst.stackDump[0], dst.stackDump.size()), "stackdump"); - } - { // without datetime - FS4Packet_QUERYX cpy; - copyPacket(src, cpy); - - SearchRequest dst; - PacketConverter::toSearchRequest(cpy, dst); - } -} - -TEST("convertFromReply") { - SearchReply src; - src.offset = 1u; - src.totalHitCount = 2u; - src.maxRank = 3; - src.setDistributionKey(4u); - src.sortIndex.push_back(0); - src.sortIndex.push_back(1); - src.sortIndex.push_back(2); - src.sortData.push_back(11); - src.sortData.push_back(22); - src.groupResult.push_back(2); - src.coverage = SearchReply::Coverage(5, 3); - src.useWideHits = true; - src.hits.resize(2); - document::GlobalId gid0("aaaaaaaaaaaa"); - document::GlobalId gid1("bbbbbbbbbbbb"); - src.hits[0].gid = gid0; - src.hits[0].metric = 5; - src.hits[0].path = 11; - src.hits[0].setDistributionKey(100); - src.hits[1].gid = gid1; - src.hits[1].metric = 4; - src.hits[1].path = 10; - src.hits[1].setDistributionKey(105); - - { // full copy - SearchReply cpy = src; - - FS4Packet_QUERYRESULTX dst0; - PacketConverter::fromSearchReply(cpy, dst0); - FS4Packet_QUERYRESULTX dst; - copyPacket(dst0, dst); - EXPECT_EQUAL(dst._offset, 1u); - EXPECT_EQUAL(dst._numDocs, 2u); - EXPECT_EQUAL(dst._totNumDocs, 2u); - EXPECT_EQUAL(dst._maxRank, 3); - EXPECT_EQUAL(4u, dst.getDistributionKey()); - EXPECT_TRUE(checkFeature(dst._features, QRF_SORTDATA)); - EXPECT_EQUAL(dst._sortIndex[0], 0u); - EXPECT_EQUAL(dst._sortIndex[1], 1u); - EXPECT_EQUAL(dst._sortIndex[2], 2u); - EXPECT_EQUAL(dst._sortData[0], 11); - EXPECT_EQUAL(dst._sortData[1], 22); - EXPECT_TRUE(checkFeature(dst._features, QRF_GROUPDATA)); - EXPECT_EQUAL(dst._groupDataLen, 1u); - EXPECT_EQUAL(dst._groupData[0], 2); - EXPECT_TRUE(checkFeature(dst._features, QRF_COVERAGE)); - EXPECT_EQUAL(dst._coverageDocs, 3u); - EXPECT_EQUAL(dst._activeDocs, 5u); - EXPECT_TRUE(checkFeature(dst._features, QRF_MLD)); - EXPECT_TRUE(dst._hits[0]._gid == gid0); - EXPECT_EQUAL(dst._hits[0]._metric, 5); - EXPECT_EQUAL(dst._hits[0]._partid, 11u); - EXPECT_EQUAL(dst._hits[0].getDistributionKey(), 100u); - EXPECT_TRUE(dst._hits[1]._gid == gid1); - EXPECT_EQUAL(dst._hits[1]._metric, 4); - EXPECT_EQUAL(dst._hits[1]._partid, 10u); - EXPECT_EQUAL(dst._hits[1].getDistributionKey(), 105u); - } - { // not sortdata - SearchReply cpy = src; - cpy.sortIndex.clear(); - cpy.sortData.clear(); - - FS4Packet_QUERYRESULTX dst0; - PacketConverter::fromSearchReply(cpy, dst0); - FS4Packet_QUERYRESULTX dst; - copyPacket(dst0, dst); - EXPECT_TRUE(checkNotFeature(dst._features, QRF_SORTDATA)); - } - { // not groupdata - SearchReply cpy = src; - cpy.groupResult.clear(); - - FS4Packet_QUERYRESULTX dst0; - PacketConverter::fromSearchReply(cpy, dst0); - FS4Packet_QUERYRESULTX dst; - copyPacket(dst0, dst); - EXPECT_TRUE(checkNotFeature(dst._features, QRF_GROUPDATA)); - } - { // non-full coverage - SearchReply cpy = src; - - FS4Packet_QUERYRESULTX dst0; - PacketConverter::fromSearchReply(cpy, dst0); - FS4Packet_QUERYRESULTX dst; - copyPacket(dst0, dst); - EXPECT_TRUE(checkFeature(dst._features, QRF_COVERAGE)); - EXPECT_EQUAL(dst._coverageDocs, 3u); - EXPECT_EQUAL(dst._activeDocs, 5u); - } - { // non-mld - SearchReply cpy = src; - cpy.useWideHits = false; - - FS4Packet_QUERYRESULTX dst0; - PacketConverter::fromSearchReply(cpy, dst0); - FS4Packet_QUERYRESULTX dst; - copyPacket(dst0, dst); - EXPECT_TRUE(checkNotFeature(dst._features, QRF_MLD)); - EXPECT_TRUE(dst._hits[0]._gid == gid0); - EXPECT_EQUAL(dst._hits[0]._metric, 5); - EXPECT_TRUE(dst._hits[1]._gid == gid1); - EXPECT_EQUAL(dst._hits[1]._metric, 4); - } - { // non-mld not siteid - SearchReply cpy = src; - cpy.useWideHits = false; - - FS4Packet_QUERYRESULTX dst0; - PacketConverter::fromSearchReply(cpy, dst0); - FS4Packet_QUERYRESULTX dst; - copyPacket(dst0, dst); - EXPECT_TRUE(checkNotFeature(dst._features, QRF_MLD)); - EXPECT_TRUE(dst._hits[0]._gid == gid0); - EXPECT_EQUAL(dst._hits[0]._metric, 5); - EXPECT_TRUE(dst._hits[1]._gid == gid1); - EXPECT_EQUAL(dst._hits[1]._metric, 4); - } -} - -void verify(vespalib::stringref expected, const vespalib::Slime & slime) { - vespalib::Slime expectedSlime; - vespalib::slime::JsonFormat::decode(expected, expectedSlime); - EXPECT_EQUAL(expectedSlime, slime); -} - -TEST("verify trace") { - RelativeTime clock(std::make_unique<CountingClock>(fastos::TimeStamp::fromSec(1500000000), 1700000L)); - Trace t(clock); - EXPECT_FALSE(t.hasTrace()); - t.start(0); - EXPECT_TRUE(t.hasTrace()); - t.createCursor("tag_a"); - verify("{" - " start_time_utc: '2017-07-14 02:40:00.000 UTC'," - " traces: [" - " {" - " tag: 'tag_a'," - " timestamp_ms: 1.7" - " }" - " ]" - "}", - t.getSlime()); - Trace::Cursor & tagB = t.createCursor("tag_b"); - tagB.setLong("long", 19); - t.done(); - verify("{" - " start_time_utc: '2017-07-14 02:40:00.000 UTC'," - " traces: [" - " {" - " tag: 'tag_a'," - " timestamp_ms: 1.7" - " }," - " {" - " tag: 'tag_b'," - " timestamp_ms: 3.4," - " long: 19" - " }" - " ]," - " duration_ms: 5.1" - "}", - t.getSlime()); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/engine/transportserver/.gitignore b/searchlib/src/tests/engine/transportserver/.gitignore deleted file mode 100644 index 09d836e0004..00000000000 --- a/searchlib/src/tests/engine/transportserver/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -.depend -Makefile -transportserver_test -vlog.txt -searchlib_transportserver_test_app diff --git a/searchlib/src/tests/engine/transportserver/CMakeLists.txt b/searchlib/src/tests/engine/transportserver/CMakeLists.txt deleted file mode 100644 index b4f94884ce4..00000000000 --- a/searchlib/src/tests/engine/transportserver/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_transportserver_test_app TEST - SOURCES - transportserver_test.cpp - DEPENDS - searchlib -) -vespa_add_test( - NAME searchlib_transportserver_test_app - COMMAND searchlib_transportserver_test_app - ENVIRONMENT "VESPA_LOG_TARGET=file:vlog.txt;VESPA_LOG_LEVEL=\"all -spam\"" -) diff --git a/searchlib/src/tests/engine/transportserver/transportserver_test.cpp b/searchlib/src/tests/engine/transportserver/transportserver_test.cpp deleted file mode 100644 index baa581c65f9..00000000000 --- a/searchlib/src/tests/engine/transportserver/transportserver_test.cpp +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/document/base/documentid.h> -#include <vespa/searchlib/common/packets.h> -#include <vespa/searchlib/engine/transportserver.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/fnet/fnet.h> -#include <vespa/searchlib/engine/errorcodes.h> -#include <thread> -#include <chrono> -#include <vespa/log/log.h> -LOG_SETUP("transportserver_test"); - -using namespace document; -using namespace vespalib; -using namespace search::engine; -using namespace search::fs4transport; - -class SyncServer : public search::engine::SearchServer, - public search::engine::DocsumServer, - public search::engine::MonitorServer -{ -private: - virtual SearchReply::UP search(SearchRequest::Source request, SearchClient &client) override; - virtual DocsumReply::UP getDocsums(DocsumRequest::Source request, DocsumClient &client) override; - virtual MonitorReply::UP ping(MonitorRequest::UP request, MonitorClient &client) override; - - SyncServer(const SyncServer &); - SyncServer &operator=(const SyncServer &); -public: - SyncServer() {} - virtual ~SyncServer() {} -}; - -SearchReply::UP -SyncServer::search(SearchRequest::Source request, SearchClient &) -{ - // fastos should use steady clock (this may fail) - std::this_thread::sleep_for(std::chrono::milliseconds(20)); - fastos::TimeStamp my_time = fastos::ClockSystem::now(); - std::this_thread::sleep_for(std::chrono::milliseconds(20)); - EXPECT_GREATER_EQUAL(my_time.val(), request->getStartTime()); - const SearchRequest &req = *request.get(); - SearchReply::UP reply(new SearchReply()); - SearchReply &ret = *reply; - ret.request = request.release(); - LOG(info, "responding to search request..."); - ret.offset = req.offset; - return reply; -} - -DocsumReply::UP -SyncServer::getDocsums(DocsumRequest::Source request, DocsumClient &) -{ - // fastos should use steady clock (this may fail) - std::this_thread::sleep_for(std::chrono::milliseconds(20)); - fastos::TimeStamp my_time = fastos::ClockSystem::now(); - std::this_thread::sleep_for(std::chrono::milliseconds(20)); - EXPECT_GREATER_EQUAL(my_time.val(), request->getStartTime()); - DocsumReply::UP reply(new DocsumReply()); - DocsumReply &ret = *reply; - ret.request = request.release(); - LOG(info, "responding to docsum request..."); - ret.docsums.resize(1); - ret.docsums[0].setData("data", strlen("data")); - ret.docsums[0].gid = DocumentId(vespalib::make_string("id:ns:type::100")).getGlobalId(); - return reply; -} - -MonitorReply::UP -SyncServer::ping(MonitorRequest::UP request, MonitorClient &) -{ - MonitorRequest &req = *request; - MonitorReply::UP reply(new MonitorReply()); - MonitorReply &ret = *reply; - LOG(info, "responding to monitor request..."); - ret.timestamp = req.flags; - return reply; -} - -TEST("transportserver") { - { - SyncServer server; - TransportServer transport(server, server, server, 0, - TransportServer::DEBUG_ALL); - ASSERT_TRUE(transport.start()); - int port = transport.getListenPort(); - ASSERT_TRUE(port > 0); - { - FNET_Context ctx; - FastOS_ThreadPool pool(128 * 1024); - FNET_Transport client; - ASSERT_TRUE(client.Start(&pool)); - - FNET_PacketQueue adminQ; - FNET_Connection *conn = client.Connect(make_string("tcp/localhost:%d", port).c_str(), - &FS4PersistentPacketStreamer::Instance, &adminQ); - ASSERT_TRUE(conn != 0); - { - FS4Packet_MONITORQUERYX *mq = new FS4Packet_MONITORQUERYX(); - mq->_qflags = 30; - mq->_features |= MQF_QFLAGS; - conn->PostPacket(mq, FNET_NOID); - FNET_Packet *p = adminQ.DequeuePacket(60000, &ctx); - ASSERT_TRUE(p != 0); - ASSERT_TRUE(p->GetPCODE() == PCODE_MONITORRESULTX); - FS4Packet_MONITORRESULTX *r = (FS4Packet_MONITORRESULTX*)p; - EXPECT_EQUAL(r->_timestamp, 30u); - p->Free(); - } - { - FNET_PacketQueue q; - FNET_Channel *ch = conn->OpenChannel(&q, FNET_Context()); - FS4Packet_QUERYX *qx = new FS4Packet_QUERYX(); - qx->_features |= QF_PARSEDQUERY; - qx->_offset = 100; - ch->Send(qx); - FNET_Packet *p = q.DequeuePacket(60000, &ctx); - ASSERT_TRUE(p != 0); - ASSERT_TRUE(p->GetPCODE() == PCODE_QUERYRESULTX); - FS4Packet_QUERYRESULTX *r = (FS4Packet_QUERYRESULTX*)p; - EXPECT_EQUAL(r->_offset, 100u); - p->Free(); - ch->CloseAndFree(); - } - { - FS4Packet_MONITORQUERYX *mq = new FS4Packet_MONITORQUERYX(); - mq->_qflags = 40; - mq->_features |= MQF_QFLAGS; - conn->PostPacket(mq, FNET_NOID); - FNET_Packet *p = adminQ.DequeuePacket(60000, &ctx); - ASSERT_TRUE(p != 0); - ASSERT_TRUE(p->GetPCODE() == PCODE_MONITORRESULTX); - FS4Packet_MONITORRESULTX *r = (FS4Packet_MONITORRESULTX*)p; - EXPECT_EQUAL(r->_timestamp, 40u); - p->Free(); - } - { - FNET_PacketQueue q; - FNET_Channel *ch = conn->OpenChannel(&q, FNET_Context()); - FS4Packet_GETDOCSUMSX *qdx = new FS4Packet_GETDOCSUMSX(); - ch->Send(qdx); - FNET_Packet *p = q.DequeuePacket(60000, &ctx); - ASSERT_TRUE(p != 0); - ASSERT_TRUE(p->GetPCODE() == PCODE_DOCSUM); - FS4Packet_DOCSUM *r = (FS4Packet_DOCSUM*)p; - EXPECT_EQUAL(r->getGid(), DocumentId("id:ns:type::100").getGlobalId()); - p->Free(); - p = q.DequeuePacket(60000, &ctx); - ASSERT_TRUE(p != 0); - ASSERT_TRUE(p->GetPCODE() == PCODE_EOL); - p->Free(); - ch->CloseAndFree(); - } - { - FS4Packet_MONITORQUERYX *mq = new FS4Packet_MONITORQUERYX(); - mq->_qflags = 50; - mq->_features |= MQF_QFLAGS; - conn->PostPacket(mq, FNET_NOID); - FNET_Packet *p = adminQ.DequeuePacket(60000, &ctx); - ASSERT_TRUE(p != 0); - ASSERT_TRUE(p->GetPCODE() == PCODE_MONITORRESULTX); - FS4Packet_MONITORRESULTX *r = (FS4Packet_MONITORRESULTX*)p; - EXPECT_EQUAL(r->_timestamp, 50u); - p->Free(); - } - // shut down client - conn->CloseAdminChannel(); - client.Close(conn); - conn->SubRef(); - client.sync(); - client.ShutDown(true); - pool.Close(); - } - - } -} - -void printError(ErrorCode ecode) { - fprintf(stderr, "error code %u: '%s'\n", ecode, getStringFromErrorCode(ecode)); -} - -TEST("print errors") { - printError(ECODE_NO_ERROR); - printError(ECODE_GENERAL_ERROR); - printError(ECODE_QUERY_PARSE_ERROR); - printError(ECODE_ALL_PARTITIONS_DOWN); - printError(ECODE_ILLEGAL_DATASET); - printError(ECODE_OVERLOADED); - printError(ECODE_NOT_IMPLEMENTED); - printError(ECODE_QUERY_NOT_ALLOWED); - printError(ECODE_TIMEOUT); -} - -TEST("test SearchReply::Coverage") { - SearchReply::Coverage c; - EXPECT_EQUAL(0u, c.getActive()); - EXPECT_EQUAL(0u, c.getSoonActive()); - EXPECT_EQUAL(0u, c.getCovered()); - EXPECT_EQUAL(0u, c.getDegradeReason()); -} - -TEST("test SearchReply::Coverage(7)") { - SearchReply::Coverage c(7); - EXPECT_EQUAL(7u, c.getActive()); - EXPECT_EQUAL(7u, c.getSoonActive()); - EXPECT_EQUAL(7u, c.getCovered()); - EXPECT_EQUAL(0u, c.getDegradeReason()); -} - -TEST("test SearchReply::Coverage(7, 19)") { - SearchReply::Coverage c(19, 7); - EXPECT_EQUAL(19u, c.getActive()); - EXPECT_EQUAL(19u, c.getSoonActive()); - EXPECT_EQUAL(7u, c.getCovered()); - EXPECT_EQUAL(0u, c.getDegradeReason()); -} - -TEST("test SearchReply::Coverage set and get") { - SearchReply::Coverage c; - EXPECT_EQUAL(7u, c.setActive(7).getActive()); - EXPECT_EQUAL(9u, c.setSoonActive(9).getSoonActive()); - EXPECT_EQUAL(19u, c.setCovered(19).getCovered()); - EXPECT_EQUAL(5u, c.setDegradeReason(5).getDegradeReason()); - EXPECT_EQUAL(1u, SearchReply::Coverage().degradeMatchPhase().getDegradeReason()); - EXPECT_EQUAL(2u, SearchReply::Coverage().degradeTimeout().getDegradeReason()); - EXPECT_EQUAL(4u, SearchReply::Coverage().degradeAdaptiveTimeout().getDegradeReason()); - EXPECT_EQUAL(7u, SearchReply::Coverage().degradeAdaptiveTimeout().degradeTimeout().degradeMatchPhase().getDegradeReason()); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/queryeval/same_element/same_element_test.cpp b/searchlib/src/tests/queryeval/same_element/same_element_test.cpp index c24a11833e7..622c8077c14 100644 --- a/searchlib/src/tests/queryeval/same_element/same_element_test.cpp +++ b/searchlib/src/tests/queryeval/same_element/same_element_test.cpp @@ -9,6 +9,7 @@ #include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchcommon/attribute/i_search_context.h> #include <vespa/searchlib/attribute/elementiterator.h> +#include <vespa/vespalib/test/insertion_operators.h> using namespace search::fef; using namespace search::queryeval; diff --git a/searchlib/src/vespa/searchlib/common/packets.cpp b/searchlib/src/vespa/searchlib/common/packets.cpp index 9862e221610..4b20513c66d 100644 --- a/searchlib/src/vespa/searchlib/common/packets.cpp +++ b/searchlib/src/vespa/searchlib/common/packets.cpp @@ -2,17 +2,12 @@ #include "mapnames.h" #include "packets.h" -#include "sortdata.h" -#include <vespa/searchlib/util/rawbuf.h> -#include <vespa/vespalib/util/compressor.h> -#include <vespa/vespalib/util/exceptions.h> -#include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/data/databuffer.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/fnet/databuffer.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.common.fs4packets"); -using vespalib::ConstBufferRef; using vespalib::make_string; using vespalib::stringref; @@ -21,192 +16,17 @@ namespace search::fs4transport { /** * Persistent packet streamer. **/ -FS4PersistentPacketStreamer FS4PersistentPacketStreamer:: -Instance(FS4PacketFactory::CreateFS4Packet); +FS4PersistentPacketStreamer FS4PersistentPacketStreamer::Instance; //============================================================ -bool -FS4PersistentPacketStreamer::HasChannelID(uint32_t pcode) -{ - switch(pcode & PCODE_MASK) { - case PCODE_EOL: - case PCODE_ERROR: - case PCODE_DOCSUM: - case PCODE_QUERYRESULTX: - case PCODE_QUERYX: - case PCODE_GETDOCSUMSX: - case PCODE_TRACEREPLY: - return true; - default: - return false; - } -} - FS4PersistentPacketStreamer:: -FS4PersistentPacketStreamer(FS4PacketFactory::CreatePacket_t cp) +FS4PersistentPacketStreamer() : _compressionLimit(0), _compressionLevel(9), - _compressionType(CompressionConfig::LZ4), - _conservative(false), - _createPacket(cp) + _compressionType(CompressionConfig::LZ4) { } -bool -FS4PersistentPacketStreamer::GetPacketInfo(FNET_DataBuffer *src, - uint32_t *plen, uint32_t *pcode, - uint32_t *chid, bool *broken) -{ - uint32_t tmpVal; - bool hasCHID; - - if (src->GetDataLen() < 2 * sizeof(uint32_t) || - ((hasCHID = HasChannelID(src->PeekInt32(sizeof(uint32_t)))) && - src->GetDataLen() < 3 * sizeof(uint32_t))) - return false; - - if (hasCHID) { - tmpVal = src->ReadInt32(); - if (tmpVal < 2 * sizeof(uint32_t)) { - // This is not a valid packet length. We might - // be out of sync. - *broken = _conservative; - if (*broken) { - LOG(warning, "Out of sync! Invalid packet length %u\n", tmpVal); - } - return false; - } else { - *plen = tmpVal - 2 * sizeof(uint32_t); - } - tmpVal = src->ReadInt32(); - if (!ValidPCode(tmpVal)) { - // Out of sync? - *broken = _conservative; - if (*broken) { - LOG(warning, "Out of sync! Invalid pcode %u (%u)\n", tmpVal, *plen); - } - return false; - } else { - *pcode = tmpVal; - } - *chid = src->ReadInt32(); - } else { - tmpVal = src->ReadInt32(); - if (tmpVal < sizeof(uint32_t)) { - // This is not a valid packet length. We might - // be out of sync. - *broken = _conservative; - if (*broken) { - LOG(warning, "Out of sync! Invalid length (noch) %u\n", tmpVal); - } - return false; - } else { - *plen = tmpVal - sizeof(uint32_t); - } - tmpVal = src->ReadInt32(); - if (!ValidPCode(tmpVal)) { - // Out of sync? - *broken = _conservative; - if (*broken) { - LOG(warning, "Out of sync! Invalid pcode (noch) %u (%u)\n", tmpVal, *plen); - } - return false; - } else { - *pcode = tmpVal; - } - *chid = FNET_NOID; - } - return true; -} - -namespace { - -void decodePacket(FNET_Packet *&packet, FNET_DataBuffer &buf, uint32_t size, uint32_t pcode) { - try { - if (!packet->Decode(&buf, size)) { - LOG(error, "could not decode packet (pcode=%u); " - "this could be caused by a protocol and/or " - "version incompatibility\n", pcode); - packet->Free(); - packet = NULL; - } - } catch (const vespalib::Exception & e) { - packet->Free(); - packet = NULL; - LOG(error, "%s", e.toString().c_str()); - } -} - -} // namespace - -FNET_Packet* -FS4PersistentPacketStreamer::Decode(FNET_DataBuffer *src, uint32_t plen, uint32_t pcode, FNET_Context) -{ - FNET_Packet *packet(_createPacket(pcode & PCODE_MASK)); - if (packet != NULL) { - uint32_t compressionByte = (pcode & ~PCODE_MASK) >> 24; - CompressionConfig::Type compressionType(CompressionConfig::toType(compressionByte)); - if (compressionType != 0) { - uint32_t uncompressed_size = src->ReadInt32(); - ConstBufferRef org(src->GetData(), plen - sizeof(uint32_t)); - vespalib::DataBuffer uncompressed(uncompressed_size); - vespalib::compression::decompress(compressionType, uncompressed_size, org, uncompressed, false); - FNET_DataBuffer buf(uncompressed.getData(), uncompressed.getDataLen()); - decodePacket(packet, buf, uncompressed_size, pcode); - src->DataToDead(plen - sizeof(uint32_t)); - } else { - decodePacket(packet, *src, plen, pcode); - } - } else { - src->DataToDead(plen); - } - return packet; -} - - -void -FS4PersistentPacketStreamer::Encode(FNET_Packet *packet, uint32_t chid, FNET_DataBuffer *dst) -{ - uint32_t len = packet->GetLength(); - uint32_t pcode = packet->GetPCODE(); - - uint32_t packet_start = dst->GetDataLen(); - if (HasChannelID(pcode)) { - dst->EnsureFree(len + 3 * sizeof(uint32_t)); - dst->WriteInt32Fast(len + 2 * sizeof(uint32_t)); - dst->WriteInt32Fast(pcode); - dst->WriteInt32Fast(chid); - } else { - dst->EnsureFree(len + 2 * sizeof(uint32_t)); - dst->WriteInt32Fast(len + sizeof(uint32_t)); - dst->WriteInt32Fast(pcode); - } - uint32_t header_len = dst->GetDataLen() - packet_start; - packet->Encode(dst); - dst->AssertValid(); - uint32_t body_len = dst->GetDataLen() - packet_start - header_len; - bool isCompressable((pcode & ~PCODE_MASK) == 0); - - if (isCompressable && _compressionLimit && (body_len > _compressionLimit)) { - CompressionConfig config(_compressionType, _compressionLevel, 90); - ConstBufferRef org(dst->GetData() + packet_start + header_len, body_len); - vespalib::DataBuffer compressed(org.size()); - CompressionConfig::Type r = vespalib::compression::compress(config, org, compressed, false); - if (r != CompressionConfig::NONE) { - dst->DataToFree(body_len + header_len); - // sizeof(data + header + uncompressed_size) - sizeof(uint32_t) - dst->WriteInt32Fast(compressed.getDataLen() + header_len); - dst->WriteInt32Fast(pcode | (_compressionType << 24)); - if (HasChannelID(pcode)) { - dst->FreeToData(sizeof(uint32_t)); // channel - } - dst->WriteInt32Fast(body_len); - dst->WriteBytes(compressed.getData(), compressed.getDataLen()); - dst->AssertValid(); - } - } -} - //============================================================ FS4Properties::FS4Properties() @@ -230,14 +50,7 @@ FS4Properties::operator=(FS4Properties && rhs) return *this; } -FS4Properties::~FS4Properties() { } - -void -FS4Properties::allocEntries(uint32_t cnt) -{ - _entries.resize(cnt); - _backing.reserve(cnt*2*40); // Assume strings are average 40 bytes -} +FS4Properties::~FS4Properties() = default; void FS4Properties::set(StringRef & e, vespalib::stringref s) { @@ -267,26 +80,29 @@ FS4Properties::getLength() return len; } -void -FS4Properties::encode(FNET_DataBuffer &dst) +vespalib::string +FS4Properties::toString(uint32_t indent) const { - dst.WriteInt32Fast(_name.size()); - dst.WriteBytesFast(_name.c_str(), _name.size()); - dst.WriteInt32Fast(size()); + vespalib::string s; + s += make_string("%*sProperties {\n", indent, ""); + s += make_string("%*s name: ", indent, ""); + s += _name; + s += "\n"; for (uint32_t i = 0; i < size(); ++i) { - dst.WriteInt32Fast(getKeyLen(i)); - dst.WriteBytesFast(getKey(i), getKeyLen(i)); - dst.WriteInt32Fast(getValueLen(i)); - dst.WriteBytesFast(getValue(i), getValueLen(i)); + s += make_string("%*s Entry[%d] {\n", indent, "", i); + s += make_string("%*s key : %s\n", indent, "", vespalib::string(getKey(i), getKeyLen(i)).c_str()); + s += make_string("%*s value: %s\n", indent, "", vespalib::string(getValue(i), getValueLen(i)).c_str()); + s += make_string("%*s }\n", indent, ""); } + s += make_string("%*s}\n", indent, ""); + return s; } bool FS4Properties::decode(FNET_DataBuffer &src, uint32_t &len) { - uint32_t strLen; if (len < sizeof(uint32_t)) return false; - strLen = src.ReadInt32(); + uint32_t strLen = src.ReadInt32(); len -= sizeof(uint32_t); if (len < strLen) return false; setName(src.GetData(), strLen); @@ -315,1467 +131,11 @@ FS4Properties::decode(FNET_DataBuffer &src, uint32_t &len) return true; } -vespalib::string -FS4Properties::toString(uint32_t indent) const -{ - vespalib::string s; - s += make_string("%*sProperties {\n", indent, ""); - s += make_string("%*s name: ", indent, ""); - s += _name; - s += "\n"; - for (uint32_t i = 0; i < size(); ++i) { - s += make_string("%*s Entry[%d] {\n", indent, "", i); - s += make_string("%*s key : %s\n", indent, "", vespalib::string(getKey(i), getKeyLen(i)).c_str()); - s += make_string("%*s value: %s\n", indent, "", vespalib::string(getValue(i), getValueLen(i)).c_str()); - s += make_string("%*s }\n", indent, ""); - } - s += make_string("%*s}\n", indent, ""); - return s; -} - -//============================================================ - -/** - * Write a string in usual format to a buffer. Usual format is first - * a 32-bit integer holding the string length, then the bytes that the - * string contained. Skip checking for free space. - * - * @param buf buffer to write to - * @param str string to write, of any type that has c_str() and size() - **/ -template<typename STR> -void -writeLenString(FNET_DataBuffer *buf, const STR &str) -{ - buf->WriteInt32Fast(str.size()); - buf->WriteBytesFast(str.c_str(), str.size()); -} - -//============================================================ - -FS4Packet::FS4Packet() - : FNET_Packet() -{ } - -FS4Packet::~FS4Packet() { } - -void -FS4Packet::Free() { - delete this; -} - -vespalib::string -FS4Packet::Print(uint32_t indent) { - return toString(indent); -} - -//============================================================ - -FS4Packet_EOL::FS4Packet_EOL() - : FS4Packet() -{ } - -FS4Packet_EOL::~FS4Packet_EOL() { } - -uint32_t -FS4Packet_EOL::GetLength() { - return 0; -} - -void -FS4Packet_EOL::Encode(FNET_DataBuffer *dst) { - (void) dst; -} - -bool -FS4Packet_EOL::Decode(FNET_DataBuffer *src, uint32_t len) -{ - src->DataToDead(len); - return (len == 0); -} - -vespalib::string -FS4Packet_EOL::toString(uint32_t indent) const -{ - return make_string("%*sFS4Packet_EOL {}\n", indent, ""); -} - -//============================================================ - -FS4Packet_Shared::FS4Packet_Shared(FNET_Packet::SP packet) - : FS4Packet(), - _packet(std::move(packet)) -{ } - -FS4Packet_Shared::~FS4Packet_Shared() { } - -uint32_t -FS4Packet_Shared::GetPCODE() { - return _packet->GetPCODE(); -} - -uint32_t -FS4Packet_Shared::GetLength() { - return _packet->GetLength(); -} - -void -FS4Packet_Shared::Encode(FNET_DataBuffer *dst) { - _packet->Encode(dst); -} - -bool -FS4Packet_Shared::Decode(FNET_DataBuffer *, uint32_t ) { - LOG_ABORT("should not be reached"); -} - -vespalib::string -FS4Packet_Shared::toString(uint32_t indent) const -{ - return _packet->Print(indent); -} - -//============================================================ - -FS4Packet_PreSerialized::FS4Packet_PreSerialized(FNET_Packet & packet) - : FS4Packet(), - _pcode(packet.GetPCODE()), - _compressionType(CompressionConfig::NONE), - _data(packet.GetLength() + 1*sizeof(uint32_t)) -{ - const uint32_t body_len(packet.GetLength()); - const uint32_t compressionLimit=FS4PersistentPacketStreamer::Instance.getCompressionLimit(); - if (compressionLimit && (body_len > compressionLimit)) { - FNET_DataBuffer tmp(packet.GetLength()); - packet.Encode(&tmp); - tmp.AssertValid(); - CompressionConfig config(FS4PersistentPacketStreamer::Instance.getCompressionType(), - FS4PersistentPacketStreamer::Instance.getCompressionLevel(), - 90); - ConstBufferRef org(tmp.GetData(), tmp.GetDataLen()); - vespalib::DataBuffer compressed(org.size()); - _compressionType = vespalib::compression::compress(config, org, compressed, false); - if (_compressionType != CompressionConfig::NONE) { - _data.WriteInt32Fast(body_len); - _data.WriteBytes(compressed.getData(), compressed.getDataLen()); - _data.AssertValid(); - } else { - packet.Encode(&_data); - } - } else { - packet.Encode(&_data); - } -} - -FS4Packet_PreSerialized::~FS4Packet_PreSerialized() { } - -uint32_t -FS4Packet_PreSerialized::GetPCODE() -{ - return ((_compressionType == CompressionConfig::NONE) - ? _pcode - : (_pcode | (_compressionType << 24))); -} - -uint32_t -FS4Packet_PreSerialized::GetLength() -{ - return _data.GetDataLen(); -} - -void -FS4Packet_PreSerialized::Encode(FNET_DataBuffer *dst) -{ - dst->WriteBytes(_data.GetData(), _data.GetDataLen()); -} - -bool -FS4Packet_PreSerialized::Decode(FNET_DataBuffer *, uint32_t) -{ - LOG_ABORT("should not be reached"); -} - -vespalib::string -FS4Packet_PreSerialized::toString(uint32_t indent) const -{ - vespalib::string s; - s += make_string("%*sFS4Packet_PreSerialized {\n", indent, ""); - s += make_string("%*s length : %d\n", indent, "", _data.GetDataLen()); - s += make_string("%*s}\n", indent, ""); - return s; -} - -//============================================================ - -FS4Packet_ERROR::FS4Packet_ERROR() - : FS4Packet(), - _errorCode(0), - _message() -{ } - -FS4Packet_ERROR::~FS4Packet_ERROR() { } - -uint32_t -FS4Packet_ERROR::GetLength() -{ - return 2 * sizeof(uint32_t) + _message.size(); -} - -void -FS4Packet_ERROR::Encode(FNET_DataBuffer *dst) -{ - dst->WriteInt32Fast(_errorCode); - writeLenString(dst, _message); -} - -bool -FS4Packet_ERROR::Decode(FNET_DataBuffer *src, uint32_t len) -{ - if (len < sizeof(uint32_t) * 2) { - src->DataToDead(len); - return false; - } - _errorCode = src->ReadInt32(); - uint32_t messageLen = src->ReadInt32(); - len -= 2 * sizeof(uint32_t); - if (len != messageLen) { - src->DataToDead(len); - return false; - } - setErrorMessage(stringref(src->GetData(), messageLen)); - src->DataToDead(messageLen); - return true; -} - - -vespalib::string -FS4Packet_ERROR::toString(uint32_t indent) const -{ - vespalib::string s; - s += make_string("%*sFS4Packet_ERROR {\n", indent, ""); - s += make_string("%*s errorCode : %d\n", indent, "", _errorCode); - s += make_string("%*s message : %s\n", indent, "", _message.c_str()); - s += make_string("%*s}\n", indent, ""); - return s; -} - -//============================================================ - -void -FS4Packet_DOCSUM::SetBuf(const char *buf, uint32_t len) -{ - _buf.resize(len); - memcpy(_buf.str(), buf, len); -} - -FS4Packet_DOCSUM::FS4Packet_DOCSUM() - : FS4Packet(), - _gid(), - _buf() -{ } - -FS4Packet_DOCSUM::~FS4Packet_DOCSUM() { } - void -FS4Packet_DOCSUM::Encode(FNET_DataBuffer *dst) -{ - dst->WriteBytesFast(_gid.get(), document::GlobalId::LENGTH); - dst->WriteBytesFast(_buf.c_str(), _buf.size()); -} - -bool -FS4Packet_DOCSUM::Decode(FNET_DataBuffer *src, uint32_t len) -{ - if (len < document::GlobalId::LENGTH) { - src->DataToDead(len); - return false; - } - unsigned char rawGid[document::GlobalId::LENGTH]; - src->ReadBytes(rawGid, document::GlobalId::LENGTH); - _gid.set(rawGid); - len -= document::GlobalId::LENGTH; - SetBuf(src->GetData(), len); - src->DataToDead(len); - return true; -} - -vespalib::string -FS4Packet_DOCSUM::toString(uint32_t indent) const -{ - vespalib::string s; - s += make_string("%*sFS4Packet_DOCSUM {\n", indent, ""); - s += make_string("%*s gid : %s\n", indent, "", _gid.toString().c_str()); - - uint32_t magic = SLIME_MAGIC_ID; - if (_buf.size() >= sizeof(magic) && - memcmp(_buf.c_str(), &magic, sizeof(magic)) == 0) { - vespalib::Slime slime; - vespalib::Memory input(_buf.c_str() + sizeof(magic), - _buf.size() - sizeof(magic)); - vespalib::SimpleBuffer buf; - vespalib::slime::BinaryFormat::decode(input, slime); - vespalib::slime::JsonFormat::encode(slime, buf, false); - s += make_string("%*s json dump : ", indent, ""); - s += buf.get().make_string(); - } else { - s += make_string("%*s data dump :\n", indent, ""); - const char *pt = _buf.c_str(); - uint32_t i = 0; - if ( ! _buf.empty()) - s += make_string("%*s ", indent, ""); - while (i < _buf.size()) { - s += make_string("%x ", (unsigned char) pt[i]); - if ((++i % 16) == 0) - s += make_string("\n%*s ", indent, ""); - } - if ((i % 16) != 0) - s += make_string("\n"); - } - s += make_string("%*s}\n", indent, ""); - return s; -} - -//============================================================ - -FS4Packet_MONITORQUERYX::FS4Packet_MONITORQUERYX() - : FS4Packet(), - _features(0), - _qflags(0u) -{ -} - - -FS4Packet_MONITORQUERYX::~FS4Packet_MONITORQUERYX() -{ -} - - -uint32_t -FS4Packet_MONITORQUERYX::GetLength() -{ - uint32_t plen = 0; - - plen += sizeof(uint32_t); - if (_features & MQF_QFLAGS) - plen += sizeof(uint32_t); - return plen; -} - - -void -FS4Packet_MONITORQUERYX::Encode(FNET_DataBuffer *dst) -{ - dst->WriteInt32Fast(_features); - - if ((_features & MQF_QFLAGS) != 0) - dst->WriteInt32Fast(_qflags); -} - - -bool -FS4Packet_MONITORQUERYX::Decode(FNET_DataBuffer *src, uint32_t len) -{ - if (len < sizeof(uint32_t)) - goto error; - _features = src->ReadInt32(); - len -= sizeof(uint32_t); - if ((_features & ~FNET_MQF_SUPPORTED_MASK) != 0) - goto error; - - if ((_features & MQF_QFLAGS) != 0) { - if (len < sizeof(uint32_t)) - goto error; - _qflags = src->ReadInt32(); - len -= sizeof(uint32_t); - } - - if (len != 0) - goto error; - - return true; - error: - src->DataToDead(len); - return false; // FAIL -} - - -vespalib::string -FS4Packet_MONITORQUERYX::toString(uint32_t indent) const -{ - vespalib::string s; - s += make_string("%*sFS4Packet_MONITORQUERYX {\n", indent, ""); - s += make_string("%*s features : 0x%x\n", indent, "", _features); - s += make_string("%*s qflags : %d\n", indent, "", _qflags); - s += make_string("%*s}\n", indent, ""); - return s; -} - -//============================================================ - -FS4Packet_MONITORRESULTX::FS4Packet_MONITORRESULTX() - : FS4Packet(), - _features(0), - _partid(0), - _timestamp(0), - _totalNodes(0), - _activeNodes(0), - _totalParts(0), - _activeParts(0), - _rflags(0u), - _activeDocs(0) -{ } - -FS4Packet_MONITORRESULTX::~FS4Packet_MONITORRESULTX() { } - -uint32_t -FS4Packet_MONITORRESULTX::GetLength() -{ - uint32_t plen = 2 * sizeof(uint32_t); - - plen += sizeof(uint32_t); - if ((_features & MRF_MLD) != 0) - plen += 4 * sizeof(uint32_t); - if ((_features & MRF_RFLAGS) != 0) - plen += sizeof(uint32_t); - if ((_features & MRF_ACTIVEDOCS) != 0) - plen += sizeof(uint64_t); - - return plen; -} - - -void -FS4Packet_MONITORRESULTX::Encode(FNET_DataBuffer *dst) -{ - dst->WriteInt32Fast(_features); - - dst->WriteInt32Fast(_partid); - dst->WriteInt32Fast(_timestamp); - if ((_features & MRF_MLD) != 0) { - dst->WriteInt32Fast(_totalNodes); - dst->WriteInt32Fast(_activeNodes); - dst->WriteInt32Fast(_totalParts); - dst->WriteInt32Fast(_activeParts); - } - if ((_features & MRF_RFLAGS) != 0) { - dst->WriteInt32Fast(_rflags); - } - if ((_features & MRF_ACTIVEDOCS) != 0) { - dst->WriteInt64Fast(_activeDocs); - } -} - - -bool -FS4Packet_MONITORRESULTX::Decode(FNET_DataBuffer *src, uint32_t len) -{ - if (len < sizeof(uint32_t)) goto error; - _features = src->ReadInt32(); - len -= sizeof(uint32_t); - if ((_features & ~FNET_MRF_SUPPORTED_MASK) != 0) - goto error; - - if (len < 2 * sizeof(uint32_t)) - goto error; - _partid = src->ReadInt32(); - _timestamp = src->ReadInt32(); - len -= 2 * sizeof(uint32_t); - - if ((_features & MRF_MLD) != 0) { - if (len < 4 * sizeof(uint32_t)) - goto error; - _totalNodes = src->ReadInt32(); - _activeNodes = src->ReadInt32(); - _totalParts = src->ReadInt32(); - _activeParts = src->ReadInt32(); - len -= 4 * sizeof(uint32_t); - } - - if ((_features & MRF_RFLAGS) != 0) { - if (len < sizeof(uint32_t)) - goto error; - _rflags = src->ReadInt32(); - len -= sizeof(uint32_t); - } - - if ((_features & MRF_ACTIVEDOCS) != 0) { - if (len < sizeof(uint64_t)) - goto error; - _activeDocs = src->ReadInt64(); - len -= sizeof(uint64_t); - } - - if (len != 0) - goto error; - - return true; // OK - error: - src->DataToDead(len); - return false; // FAIL -} - - -vespalib::string -FS4Packet_MONITORRESULTX::toString(uint32_t indent) const -{ - vespalib::string s; - s += make_string("%*sFS4Packet_MONITORRESULTX {\n", indent, ""); - s += make_string("%*s features : 0x%x\n", indent, "", _features); - s += make_string("%*s partid : %d\n", indent, "", _partid); - s += make_string("%*s timestamp : %d\n", indent, "", _timestamp); - s += make_string("%*s totalnodes : %d\n", indent, "", _totalNodes); - s += make_string("%*s activenodes : %d\n", indent, "", _activeNodes); - s += make_string("%*s totalparts : %d\n", indent, "", _totalParts); - s += make_string("%*s activeparts : %d\n", indent, "", _activeParts); - s += make_string("%*s}\n", indent, ""); - return s; -} - -//============================================================ - -void -FS4Packet_QUERYRESULTX::AllocateSortIndex(uint32_t cnt) -{ - if (cnt == 0) - return; - - cnt++; // end of data index entry - _sortIndex = new uint32_t[cnt]; -} - - -void -FS4Packet_QUERYRESULTX::AllocateSortData(uint32_t len) -{ - if (len == 0) - return; - - _sortData = (char *) malloc(len); -} - - -void -FS4Packet_QUERYRESULTX::SetSortDataRef(uint32_t cnt, - uint32_t *sortIndex, - const char *sortData) -{ - if (cnt == 0) - return; - - AllocateSortIndex(cnt); - AllocateSortData(sortIndex[cnt] - sortIndex[0]); - _sortIndex[0] = 0; - search::common::SortData::Copy(cnt, _sortIndex, _sortData, sortIndex, sortData); -} - - -void -FS4Packet_QUERYRESULTX::AllocateGroupData(uint32_t len) -{ - if (len == 0) - return; - - _groupData = (char *) malloc(len); - _groupDataLen = len; -} - - -void -FS4Packet_QUERYRESULTX::SetGroupDataRef(const char *groupData, - uint32_t len) -{ - if (len == 0) - return; - - AllocateGroupData(len); - memcpy(_groupData, groupData, len); -} - - -void -FS4Packet_QUERYRESULTX::AllocateHits(uint32_t cnt) -{ - if (cnt == 0) - return; - - _hits = new FS4_hit[cnt]; - _numDocs = cnt; -} - - -FS4Packet_QUERYRESULTX::FS4Packet_QUERYRESULTX() - : FS4Packet(), - _distributionKey(0), - _nodesQueried(1), - _nodesReplied(1), - _features(QRF_COVERAGE | QRF_EXTENDED_COVERAGE), - _offset(0), - _numDocs(0), - _totNumDocs(0), - _maxRank(0), - _sortIndex(NULL), - _sortData(NULL), - _groupDataLen(0), - _groupData(NULL), - _coverageDocs(0), - _activeDocs(0), - _soonActiveDocs(0), - _coverageDegradeReason(0), - _hits(NULL), - _propsVector() -{ } - - -FS4Packet_QUERYRESULTX::~FS4Packet_QUERYRESULTX() -{ - if (_sortIndex) { delete [] _sortIndex; } - if (_sortData) { free(_sortData); } - if (_groupData) { free(_groupData); } - if (_hits) { delete [] _hits; } -} - - -uint32_t -FS4Packet_QUERYRESULTX::GetLength() -{ - uint32_t plen = 3 * sizeof(uint32_t) + - sizeof(uint64_t) + // hit count is now 64-bit - sizeof(search::HitRank) + - _numDocs * (sizeof(document::GlobalId) + sizeof(search::HitRank)); - - plen += sizeof(uint32_t); - plen += (_features & QRF_COVERAGE_NODES) ? (2 * sizeof(uint16_t)) : 0; - plen += (_features & QRF_MLD) ? (_numDocs * 2 * sizeof(uint32_t)) : 0; - plen += (_features & QRF_GROUPDATA) ? (sizeof(uint32_t) + _groupDataLen) : 0; - plen += 3 * sizeof(uint64_t) + sizeof(uint32_t); - - if (((_features & QRF_SORTDATA) != 0) && (_numDocs > 0)) - plen += _numDocs * sizeof(uint32_t) + (_sortIndex[_numDocs] - _sortIndex[0]); - - if ((_features & QRF_PROPERTIES) != 0) { - plen += sizeof(uint32_t); - for (uint32_t i = 0; i < _propsVector.size(); ++i) { - plen += _propsVector[i].getLength(); - } - } - - return plen; -} - - -void -FS4Packet_QUERYRESULTX::Encode(FNET_DataBuffer *dst) -{ - dst->WriteInt32Fast(_features); - dst->WriteInt32Fast(_offset); - dst->WriteInt32Fast(_numDocs); - dst->WriteInt64Fast(_totNumDocs); - union { uint64_t INT64; double DOUBLE; } mrval; - mrval.DOUBLE = _maxRank; - dst->WriteInt64Fast(mrval.INT64); - dst->WriteInt32Fast(_distributionKey); - - if (_features & QRF_COVERAGE_NODES) { - dst->WriteInt16Fast(_nodesQueried); - dst->WriteInt16Fast(_nodesReplied); - } - - if (((_features & QRF_SORTDATA) != 0) && - (_numDocs > 0)) - { - uint32_t idx0 = _sortIndex[0]; - // implicit: first index entry always 0 - for (uint32_t i = 1; i <= _numDocs; i++) { - dst->WriteInt32Fast(_sortIndex[i] - idx0); - } - dst->WriteBytesFast(_sortData + idx0, _sortIndex[_numDocs] - idx0); - } - - if ((_features & QRF_GROUPDATA) != 0) { - dst->WriteInt32Fast(_groupDataLen); - dst->WriteBytesFast(_groupData, _groupDataLen); - } - - dst->WriteInt64Fast(_coverageDocs); - dst->WriteInt64Fast(_activeDocs); - dst->WriteInt64Fast(_soonActiveDocs); - dst->WriteInt32Fast(_coverageDegradeReason); - - - for (uint32_t i = 0; i < _numDocs; i++) { - dst->WriteBytesFast(_hits[i]._gid.get(), document::GlobalId::LENGTH); - union { uint64_t INT64; double DOUBLE; } val; - val.DOUBLE = _hits[i]._metric; - dst->WriteInt64Fast(val.INT64); - if ((_features & QRF_MLD) != 0) { - dst->WriteInt32Fast(_hits[i]._partid); - dst->WriteInt32Fast(_hits[i].getDistributionKey()); - } - } - - if ((_features & QRF_PROPERTIES) != 0) { - dst->WriteInt32Fast(_propsVector.size()); - for (uint32_t i = 0; i < _propsVector.size(); ++i) { - _propsVector[i].encode(*dst); - } - } -} - - -bool -FS4Packet_QUERYRESULTX::Decode(FNET_DataBuffer *src, uint32_t len) -{ - uint32_t hitSize = sizeof(document::GlobalId); - - if (len < sizeof(uint32_t)) goto error; - _features = src->ReadInt32(); - len -= sizeof(uint32_t); - - if ((_features & ~FNET_QRF_SUPPORTED_MASK) != 0) { - throwUnsupportedFeatures(_features, FNET_QRF_SUPPORTED_MASK); - } - hitSize += sizeof(uint64_t); - - if (len < 3 * sizeof(uint32_t) + sizeof(uint64_t) + sizeof(search::HitRank)) goto error; - _offset = src->ReadInt32(); - _numDocs = src->ReadInt32(); - _totNumDocs = src->ReadInt64(); - union { uint64_t INT64; double DOUBLE; } mrval; - mrval.INT64 = src->ReadInt64(); - _maxRank = mrval.DOUBLE; - _distributionKey = src->ReadInt32(); - len -= 3 * sizeof(uint32_t) + sizeof(uint64_t) + sizeof(search::HitRank); - - if (_features & QRF_COVERAGE_NODES) { - if (len < 2* sizeof(uint16_t)) goto error; - _nodesQueried = src->ReadInt16(); - _nodesReplied = src->ReadInt16(); - len -= 2*sizeof(uint16_t); - } - if (((_features & QRF_SORTDATA) != 0) && (_numDocs > 0)) { - if (len < _numDocs * sizeof(uint32_t)) goto error; - AllocateSortIndex(_numDocs); - _sortIndex[0] = 0; // implicit - for (uint32_t i = 1; i <= _numDocs; i++) { - _sortIndex[i] = src->ReadInt32(); - } - len -= _numDocs * sizeof(uint32_t); - uint32_t sortDataLen = _sortIndex[_numDocs]; - - if (len < sortDataLen) goto error; - AllocateSortData(sortDataLen); - src->ReadBytes(_sortData, sortDataLen); - len -= sortDataLen; - } - - if ((_features & QRF_GROUPDATA) != 0) { - if (len < sizeof(uint32_t)) goto error; - _groupDataLen = src->ReadInt32(); - len -= sizeof(uint32_t); - - if (len < _groupDataLen) goto error; - AllocateGroupData(_groupDataLen); - src->ReadBytes(_groupData, _groupDataLen); - len -= _groupDataLen; - } - - if (len < 3 * sizeof(uint64_t) + sizeof(uint32_t)) goto error; - _coverageDocs = src->ReadInt64(); - _activeDocs = src->ReadInt64(); - _soonActiveDocs = src->ReadInt64(); - _coverageDegradeReason = src->ReadInt32(); - - len -= 3*sizeof(uint64_t) + sizeof(uint32_t); - - if ((_features & QRF_MLD) != 0) { - hitSize += 2 * sizeof(uint32_t); - } - - if (len < _numDocs * hitSize) goto error; - AllocateHits(_numDocs); - unsigned char rawGid[document::GlobalId::LENGTH]; - for (uint32_t i = 0; i < _numDocs; i++) { - src->ReadBytes(rawGid, document::GlobalId::LENGTH); - _hits[i]._gid.set(rawGid); - union { uint64_t INT64; double DOUBLE; } val; - val.INT64 = src->ReadInt64(); - _hits[i]._metric = val.DOUBLE; - if ((_features & QRF_MLD) != 0) { - _hits[i]._partid = src->ReadInt32(); - _hits[i].setDistributionKey(src->ReadInt32()); - } else { - _hits[i]._partid = 0; // partid not available - _hits[i].setDistributionKey(getDistributionKey()); - } - } - len -= _numDocs * hitSize; - - if ((_features & QRF_PROPERTIES) != 0) { - uint32_t sz = src->ReadInt32(); - _propsVector.resize(sz); - len -= sizeof(uint32_t); - for (uint32_t i = 0; i < sz; ++i) { - if (! _propsVector[i].decode(*src, len)) goto error; - } - } - - if (len != 0) goto error; - - return true; // OK - - error: - src->DataToDead(len); - return false; // FAIL -} - - -vespalib::string -FS4Packet_QUERYRESULTX::toString(uint32_t indent) const -{ - vespalib::string s; - uint32_t i; - - s += make_string("%*sFS4Packet_QUERYRESULTX {\n", indent, ""); - s += make_string("%*s features : 0x%x\n", indent, "", _features); - s += make_string("%*s offset : %d\n", indent, "", _offset); - s += make_string("%*s numDocs : %d\n", indent, "", _numDocs); - s += make_string("%*s totNumDocs : %" PRIu64 "\n", indent, "", _totNumDocs); - s += make_string("%*s maxRank : %f\n", indent, "", _maxRank); - s += make_string("%*s distrib key : %d\n", indent, "", getDistributionKey()); - if (_numDocs > 0 && _sortIndex != NULL) { - uint32_t offset = _sortIndex[0]; - for (i = 0; i < _numDocs; i++) { - uint32_t end = _sortIndex[i + 1]; - s += make_string("%*s sort[%d] = { 0x", indent, "", i); - for (; offset < end; offset++) - s += make_string("%02x", (unsigned char)*(_sortData + offset)); - s += make_string(" }\n"); - } - } - s += make_string("%*s groupData : %d bytes\n", indent, "", _groupDataLen); - s += make_string("%*s coverageDocs : %" PRIu64 "\n", indent, "", _coverageDocs); - s += make_string("%*s activeDocs : %" PRIu64 "\n", indent, "", _activeDocs); - for (i = 0; i < _numDocs; i++) { - s += make_string("%*s hit {", indent, ""); - s += make_string("gid=%s, ", _hits[i]._gid.toString().c_str()); - s += make_string("metric=%f, ", _hits[i]._metric); - s += make_string("partid=%d, ", _hits[i]._partid); - s += make_string("distribkey=%d, ", _hits[i].getDistributionKey()); - } - s += make_string("%*s}\n", indent, ""); - return s; -} - -//============================================================ - - -FS4Packet_QUERYX::FS4Packet_QUERYX() - : FS4Packet(), - _timeout(0), - _qflags(0), - _features(0), - _offset(0), - _maxhits(0), - _ranking(), - _propsVector(), - _sortSpec(), - _groupSpec(), - _sessionId(), - _location(), - _numStackItems(0), - _stackDump() -{ } - -FS4Packet_QUERYX::~FS4Packet_QUERYX() { } - -uint32_t -FS4Packet_QUERYX::GetLength() -{ - uint32_t plen = 2 * sizeof(uint32_t); - plen += FNET_DataBuffer::getCompressedPositiveLength(_offset); - plen += FNET_DataBuffer::getCompressedPositiveLength(_maxhits); - plen += sizeof(uint32_t); - - if ((_features & QF_PARSEDQUERY) != 0) { - plen += sizeof(uint32_t)*2; - plen += _stackDump.size(); - } - if ((_features & QF_RANKP) != 0) { - plen += FNET_DataBuffer::getCompressedPositiveLength(_ranking.size()); - plen += _ranking.size(); - } - if ((_features & QF_PROPERTIES) != 0) { - plen += sizeof(uint32_t); - for (uint32_t i = 0; i < _propsVector.size(); ++i) { - plen += _propsVector[i].getLength(); - } - } - - if ((_features & QF_SORTSPEC) != 0) - plen += sizeof(uint32_t) + _sortSpec.size(); - - if ((_features & QF_GROUPSPEC) != 0) - plen += sizeof(uint32_t) + _groupSpec.size(); - - if ((_features & QF_SESSIONID) != 0) - plen += sizeof(uint32_t) + _sessionId.size(); - - if ((_features & QF_LOCATION) != 0) - plen += sizeof(uint32_t) + _location.size(); - - return plen; -} - - -void -FS4Packet_QUERYX::Encode(FNET_DataBuffer *dst) -{ - dst->WriteInt32Fast(_features); - - dst->writeCompressedPositive(_offset); - dst->writeCompressedPositive(_maxhits); - dst->WriteInt32Fast(_timeout); - dst->WriteInt32Fast(_qflags); - - if ((_features & QF_RANKP) != 0) { - dst->writeCompressedPositive(_ranking.size()); - dst->WriteBytesFast(_ranking.c_str(), _ranking.size()); - } - - if ((_features & QF_PROPERTIES) != 0) { - dst->WriteInt32Fast(_propsVector.size()); - for (uint32_t i = 0; i < _propsVector.size(); ++i) { - _propsVector[i].encode(*dst); - } - } - - if ((_features & QF_SORTSPEC) != 0) { - dst->WriteInt32Fast(_sortSpec.size()); - dst->WriteBytesFast(_sortSpec.c_str(), _sortSpec.size()); - } - - if ((_features & QF_GROUPSPEC) != 0) { - dst->WriteInt32Fast(_groupSpec.size()); - dst->WriteBytesFast(_groupSpec.c_str(), _groupSpec.size()); - } - - if ((_features & QF_SESSIONID) != 0) { - dst->WriteInt32Fast(_sessionId.size()); - dst->WriteBytesFast(_sessionId.c_str(), _sessionId.size()); - } - - if ((_features & QF_LOCATION) != 0) { - dst->WriteInt32Fast(_location.size()); - dst->WriteBytesFast(_location.c_str(), _location.size()); - } - - if ((_features & QF_PARSEDQUERY) != 0) { - dst->WriteInt32Fast(_numStackItems); - dst->WriteInt32Fast(_stackDump.size()); - dst->WriteBytesFast(_stackDump.c_str(), _stackDump.size()); - } -} - -void FS4Packet::throwPropertieDecodeError(size_t i) -{ - throw vespalib::IllegalArgumentException(make_string("Failed decoding properties[%ld]", i)); -} - -void FS4Packet::throwUnsupportedFeatures(uint32_t features, uint32_t set) -{ - throw vespalib::UnderflowException(make_string("Unsupported features(%x), supported set(%x)", features, set)); -} - -void FS4Packet::throwNotEnoughData(FNET_DataBuffer & buf, uint32_t left, uint32_t needed, const char * text) -{ - (void) buf; - throw vespalib::UnderflowException(make_string("Failed decoding packet of type %d. Only %d bytes left, needed %d from '%s'", GetPCODE(), left, needed, text)); -} - -#define VERIFY_LEN(needed, text) \ - { \ - if (len < needed) { \ - throwNotEnoughData(*src, len, needed, text); \ - } \ - len -= needed; \ - } - -uint32_t FS4Packet::readUInt32(FNET_DataBuffer & buf, uint32_t & len, const char *text) -{ - if (len < sizeof(uint32_t)) { - throwNotEnoughData(buf, len, sizeof(uint32_t), text); \ - } - len -= sizeof(uint32_t); - return buf.ReadInt32(); -} - -void -FS4Packet_GETDOCSUMSX::setTimeout(const fastos::TimeStamp & timeout) -{ - _timeout = std::max(INT64_C(0), timeout.ms()); -} - -fastos::TimeStamp -FS4Packet_GETDOCSUMSX::getTimeout() const -{ - return fastos::TimeStamp(_timeout*fastos::TimeStamp::MS); -} - -void -FS4Packet_QUERYX::setTimeout(const fastos::TimeStamp & timeout) -{ - _timeout = std::max(INT64_C(0), timeout.ms()); -} - -fastos::TimeStamp -FS4Packet_QUERYX::getTimeout() const -{ - return fastos::TimeStamp(_timeout*fastos::TimeStamp::MS); -} - -bool -FS4Packet_QUERYX::Decode(FNET_DataBuffer *src, uint32_t len) -{ - _features = readUInt32(*src, len, "features"); - - if (((_features & ~FNET_QF_SUPPORTED_MASK) != 0)) { - throwUnsupportedFeatures(_features, FNET_QF_SUPPORTED_MASK); - } - _offset = src->readCompressedPositiveInteger(); - len -= FNET_DataBuffer::getCompressedPositiveLength(_offset); - _maxhits = src->readCompressedPositiveInteger(); - len -= FNET_DataBuffer::getCompressedPositiveLength(_maxhits); - VERIFY_LEN(2 * sizeof(uint32_t), "offset, maxhits, timeout and qflags"); - _timeout = src->ReadInt32(); - _qflags = src->ReadInt32(); - - if ((_features & QF_RANKP) != 0) { - uint32_t rankingLen = src->readCompressedPositiveInteger(); - len -= FNET_DataBuffer::getCompressedPositiveLength(rankingLen); - VERIFY_LEN(rankingLen, "ranking blob"); - setRanking(stringref(src->GetData(), rankingLen)); - src->DataToDead(rankingLen); - } - - if ((_features & QF_PROPERTIES) != 0) { - uint32_t cnt = readUInt32(*src, len, "#properties"); - _propsVector.resize(cnt); - for (uint32_t i = 0; i < cnt; ++i) { - if (!_propsVector[i].decode(*src, len)) { - throwPropertieDecodeError(i); - } - } - } - - if ((_features & QF_SORTSPEC) != 0) { - uint32_t sortSpecLen = readUInt32(*src, len, "sortspec length"); - - VERIFY_LEN(sortSpecLen, "sortspec string"); - setSortSpec(stringref(src->GetData(), sortSpecLen)); - src->DataToDead(sortSpecLen); - } - - if ((_features & QF_GROUPSPEC) != 0) { - uint32_t groupSpecLen = readUInt32(*src, len, "groupspec length"); - - VERIFY_LEN(groupSpecLen, "groupspec string"); - setGroupSpec(stringref(src->GetData(), groupSpecLen)); - src->DataToDead(groupSpecLen); - } - - if ((_features & QF_SESSIONID) != 0) { - uint32_t sessionIdLen = readUInt32(*src, len, "sessionid length"); - VERIFY_LEN(sessionIdLen, "sessionid string"); - setSessionId(stringref(src->GetData(), sessionIdLen)); - src->DataToDead(sessionIdLen); - } - - if ((_features & QF_LOCATION) != 0) { - uint32_t locationLen = readUInt32(*src, len, "location length"); - - VERIFY_LEN(locationLen, "location string"); - setLocation(stringref(src->GetData(), locationLen)); - src->DataToDead(locationLen); - } - - if ((_features & QF_PARSEDQUERY) != 0) { - _numStackItems = readUInt32(*src, len, "# querystack items"); - - uint32_t stackDumpLen = readUInt32(*src, len, "stackdump length"); - VERIFY_LEN(stackDumpLen, "stackdump"); - setStackDump(stringref(src->GetData(), stackDumpLen)); - src->DataToDead(stackDumpLen); - } - if (len != 0) { - throwNotEnoughData(*src, len, 0, "eof"); - } - - return true; -} - - -vespalib::string -FS4Packet_QUERYX::toString(uint32_t indent) const -{ - vespalib::string s; - s += make_string("%*sFS4Packet_QUERYX {\n", indent, ""); - s += make_string("%*s features : 0x%x\n", indent, "", _features); - s += make_string("%*s offset : %d\n", indent, "", _offset); - s += make_string("%*s maxhits : %d\n", indent, "", _maxhits); - s += make_string("%*s qflags : %x\n", indent, "", _qflags); - s += make_string("%*s ranking : %s\n", indent, "", _ranking.c_str()); - for (uint32_t i = 0; i < _propsVector.size(); ++i) { - s += _propsVector[i].toString(indent + 2); - } - s += make_string("%*s sortspec : %s\n", indent, "", _sortSpec.c_str()); - s += make_string("%*s groupspec : (%d bytes)\n", indent, "", (int)_groupSpec.size()); - s += make_string("%*s sessionId : (%d bytes) %s\n", indent, "", (int)_sessionId.size(), _sessionId.c_str()); - s += make_string("%*s location : %s\n", indent, "", _location.c_str()); - s += make_string("%*s timeout : %d\n", indent, "", _timeout); - s += make_string("%*s stackitems : %d\n", indent, "", _numStackItems); - s += make_string("%*s stack dump :\n", indent, ""); - if (_stackDump.size() > 0) { - const char *pt = _stackDump.c_str(); - s += make_string("%*s ", indent, ""); - uint32_t i = 0; - while (i < _stackDump.size()) { - s += make_string("%x ", (unsigned char) pt[i]); - if ((++i % 16) == 0 && i < _stackDump.size()) { - s += make_string("\n%*s ", indent, ""); - } - } - if ((i % 16) != 0) s += make_string("\n"); - } - s += make_string("%*s}\n", indent, ""); - return s; -} - -//============================================================ - - -void -FS4Packet_GETDOCSUMSX::AllocateDocIDs(uint32_t cnt) -{ - if (cnt == 0) - return; - - _docid.resize(cnt); -} - - -FS4Packet_GETDOCSUMSX::FS4Packet_GETDOCSUMSX() - : FS4Packet(), - _timeout(0), - _features(0), - _ranking(), - _qflags(0), - _resultClassName(), - _propsVector(), - _stackItems(0), - _stackDump(), - _location(), - _flags(0u), - _docid() -{ } - - -FS4Packet_GETDOCSUMSX::~FS4Packet_GETDOCSUMSX() { } - -uint32_t -FS4Packet_GETDOCSUMSX::GetLength() -{ - uint32_t plen = 2 * sizeof(uint32_t) + - + _docid.size() * (sizeof(document::GlobalId)); - - plen += sizeof(uint32_t); - - if ((_features & GDF_MLD) != 0) - plen += 2 * _docid.size() * sizeof(uint32_t); - - if ((_features & GDF_QUERYSTACK) != 0) - plen += 2 * sizeof(uint32_t) + _stackDump.size(); - - if ((_features & GDF_RESCLASSNAME) != 0) - plen += sizeof(uint32_t) + _resultClassName.size(); - - if ((_features & GDF_PROPERTIES) != 0) { - plen += sizeof(uint32_t); - for (uint32_t i = 0; i < _propsVector.size(); ++i) { - plen += _propsVector[i].getLength(); - } - } - - if ((_features & GDF_RANKP_QFLAGS) != 0) { - plen += FNET_DataBuffer::getCompressedPositiveLength(_ranking.size()); - plen += _ranking.size(); - plen += sizeof(uint32_t); - } - - if ((_features & GDF_LOCATION) != 0) - plen += sizeof(uint32_t) + _location.size(); - - if ((_features & GDF_FLAGS) != 0) - plen += sizeof(uint32_t); - - return plen; -} - - -void -FS4Packet_GETDOCSUMSX::Encode(FNET_DataBuffer *dst) -{ - dst->WriteInt32Fast(_features); - - dst->WriteInt32Fast(0); - dst->WriteInt32Fast(_timeout); - - if ((_features & GDF_RANKP_QFLAGS) != 0) { - dst->writeCompressedPositive(_ranking.size()); - dst->WriteBytesFast(_ranking.c_str(), _ranking.size()); - dst->WriteInt32Fast(_qflags); - } - - if ((_features & GDF_RESCLASSNAME) != 0) { - writeLenString(dst, _resultClassName); - } - - if ((_features & GDF_PROPERTIES) != 0) { - dst->WriteInt32Fast(_propsVector.size()); - for (uint32_t i = 0; i < _propsVector.size(); ++i) { - _propsVector[i].encode(*dst); - } - } - - if ((_features & GDF_QUERYSTACK) != 0) { - dst->WriteInt32Fast(_stackItems); - writeLenString(dst, _stackDump); - } - - if ((_features & GDF_LOCATION) != 0) { - writeLenString(dst, _location); - } - - if ((_features & GDF_FLAGS) != 0) { - dst->WriteInt32Fast(_flags); - } - - for (const auto & docid : _docid) { - dst->WriteBytesFast(docid._gid.get(), document::GlobalId::LENGTH); - - if ((_features & GDF_MLD) != 0) { - dst->WriteInt32Fast(docid._partid); - dst->WriteInt32Fast(0); - } - } -} - - -bool -FS4Packet_GETDOCSUMSX::Decode(FNET_DataBuffer *src, uint32_t len) -{ - uint32_t docidSize = sizeof(document::GlobalId); - - _features = readUInt32(*src, len, "features"); - - if ((_features & ~FNET_GDF_SUPPORTED_MASK) != 0) { - throwUnsupportedFeatures(_features, FNET_GDF_SUPPORTED_MASK); - } - - VERIFY_LEN(2*sizeof(uint32_t), "unused and timeout"); - src->ReadInt32(); // unused - _timeout = src->ReadInt32(); - - if ((_features & GDF_RANKP_QFLAGS) != 0) { - uint32_t rankingLen = src->readCompressedPositiveInteger(); - len -= FNET_DataBuffer::getCompressedPositiveLength(rankingLen); - - VERIFY_LEN(rankingLen, "ranking blob"); - setRanking(vespalib::stringref(src->GetData(), rankingLen)); - src->DataToDead(rankingLen); - - _qflags = readUInt32(*src, len, "qflags"); - } - - if ((_features & GDF_RESCLASSNAME) != 0) { - uint32_t resultClassNameLen = readUInt32(*src, len, "result class name length"); - - VERIFY_LEN(resultClassNameLen, "result class"); - setResultClassName(stringref(src->GetData(), resultClassNameLen)); - src->DataToDead(resultClassNameLen); - } - - if ((_features & GDF_PROPERTIES) != 0) { - uint32_t cnt = readUInt32(*src, len, "#properties"); - _propsVector.resize(cnt); - for (uint32_t i = 0; i < cnt; ++i) { - if (!_propsVector[i].decode(*src, len)) { - throwPropertieDecodeError(i); - } - } - } - - if ((_features & GDF_QUERYSTACK) != 0) { - _stackItems = readUInt32(*src, len, "num stack items"); - uint32_t stackDumpLen = readUInt32(*src, len, "stackdump length"); - VERIFY_LEN(stackDumpLen, "stackdump"); - setStackDump(stringref(src->GetData(), stackDumpLen)); - src->DataToDead(stackDumpLen); - } - - if ((_features & GDF_LOCATION) != 0) { - uint32_t locationLen = readUInt32(*src, len, "location length"); - VERIFY_LEN(locationLen, "location string"); - setLocation(stringref(src->GetData(), locationLen)); - src->DataToDead(locationLen); - } - - if ((_features & GDF_FLAGS) != 0) { - _flags = readUInt32(*src, len, "flags"); - } - - if ((_features & GDF_MLD) != 0) - docidSize += 2 * sizeof(uint32_t); - - AllocateDocIDs(len / docidSize); - - unsigned char rawGid[document::GlobalId::LENGTH]; - for (auto & docid : _docid) { - src->ReadBytes(rawGid, document::GlobalId::LENGTH); - docid._gid.set(rawGid); - - if ((_features & GDF_MLD) != 0) { - docid._partid = src->ReadInt32(); - src->ReadInt32(); // unused - } else { - docid._partid = 0; // partid not available - } - } - len -= _docid.size() * docidSize; - - if (len != 0) { - throwNotEnoughData(*src, len, 0, "eof"); - } - - return true; // OK -} - - -vespalib::string -FS4Packet_GETDOCSUMSX::toString(uint32_t indent) const -{ - vespalib::string s; - s += make_string("%*sFS4Packet_GETDOCSUMSX {\n", indent, ""); - s += make_string("%*s features : %d\n", indent, "", _features); - s += make_string("%*s ranking : %s\n", indent, "", _ranking.c_str()); - s += make_string("%*s qflags : %x\n", indent, "", _qflags); - s += make_string("%*s resClassName: %s\n", indent, "", _resultClassName.c_str()); - for (uint32_t i = 0; i < _propsVector.size(); ++i) { - s += _propsVector[i].toString(indent + 2); - } - s += make_string("%*s stackItems : %d\n", indent, "", _stackItems); - s += make_string("%*s stackDumpLen : %d\n", indent, "", (int)_stackDump.size()); - s += make_string("%*s stackDump :\n", indent, ""); - - uint32_t i = 0; - if (_stackDump.size() > 0) { - const char *pt = _stackDump.c_str(); - s += make_string("%*s ", indent, ""); - while (i < _stackDump.size()) { - s += make_string("%x ", (unsigned char) pt[i]); - if ((++i % 16) == 0) - s += make_string("\n%*s ", indent, ""); - } - if ((i % 16) != 0) s += make_string("\n"); - } - for (const auto & docId : _docid) { - s += make_string("%*s gid=%s, partid=%d\n", indent, "", - docId._gid.toString().c_str(), docId._partid); - } - s += make_string("%*s location : %s\n", indent, "", _location.c_str()); - s += make_string("%*s timeout : %d\n", indent, "", _timeout); - s += make_string("%*s flags : %d\n", indent, "", _flags); - s += make_string("%*s}\n", indent, ""); - return s; -} - -//============================================================ - -uint32_t -FS4Packet_TRACEREPLY::GetLength() -{ - uint32_t plen = sizeof(uint32_t); - for (uint32_t i = 0; i < _propsVector.size(); ++i) { - plen += _propsVector[i].getLength(); - } - return plen; -} - -void -FS4Packet_TRACEREPLY::Encode(FNET_DataBuffer *dst) -{ - dst->WriteInt32Fast(_propsVector.size()); - for (uint32_t i = 0; i < _propsVector.size(); ++i) { - _propsVector[i].encode(*dst); - } -} - -bool -FS4Packet_TRACEREPLY::Decode(FNET_DataBuffer *src, uint32_t len) -{ - uint32_t cnt = readUInt32(*src, len, "#properties"); - _propsVector.resize(cnt); - for (uint32_t i = 0; i < cnt; ++i) { - if (!_propsVector[i].decode(*src, len)) { - throwPropertieDecodeError(i); - } - } - if (len != 0) goto error; - return true; // OK - error: - src->DataToDead(len); - return false; // FAIL -} - -vespalib::string -FS4Packet_TRACEREPLY::toString(uint32_t indent) const -{ - vespalib::string s; - s += make_string("%*sFS4Packet_TRACEREPLY {\n", indent, ""); - for (uint32_t i = 0; i < _propsVector.size(); ++i) { - s += _propsVector[i].toString(indent + 2); - } - s += make_string("%*s}\n", indent, ""); - return s; -} - - -//============================================================ - -FNET_Packet* -FS4PacketFactory::CreateFS4Packet(uint32_t pcode) +FS4Properties::allocEntries(uint32_t cnt) { - switch(pcode) { - case PCODE_EOL: - return new FS4Packet_EOL; - case PCODE_ERROR: - return new FS4Packet_ERROR; - case PCODE_DOCSUM: - return new FS4Packet_DOCSUM; - case PCODE_QUERYRESULTX: - return new FS4Packet_QUERYRESULTX; - case PCODE_QUERYX: - return new FS4Packet_QUERYX; - case PCODE_GETDOCSUMSX: - return new FS4Packet_GETDOCSUMSX; - case PCODE_MONITORQUERYX: - return new FS4Packet_MONITORQUERYX; - case PCODE_MONITORRESULTX: - return new FS4Packet_MONITORRESULTX; - case PCODE_TRACEREPLY: - return new FS4Packet_TRACEREPLY; - default: - return NULL; - } + _entries.resize(cnt); + _backing.reserve(cnt*2*40); // Assume strings are average 40 bytes } } diff --git a/searchlib/src/vespa/searchlib/common/packets.h b/searchlib/src/vespa/searchlib/common/packets.h index 3fd87ccfa25..5e0072c27c0 100644 --- a/searchlib/src/vespa/searchlib/common/packets.h +++ b/searchlib/src/vespa/searchlib/common/packets.h @@ -2,148 +2,28 @@ #pragma once -#include "transport.h" -#include "hitrank.h" -#include <vespa/fnet/context.h> -#include <vespa/fnet/ipacketstreamer.h> -#include <vespa/fnet/packet.h> -#include <vespa/fnet/databuffer.h> -#include <vespa/document/base/globalid.h> #include <vespa/vespalib/util/compressionconfig.h> #include <vespa/vespalib/util/memory.h> -#include <vespa/fastos/timestamp.h> #include <vector> +class FNET_DataBuffer; + namespace search::fs4transport { using vespalib::string; -enum fnet_feature_masks { - FNET_QRF_SUPPORTED_MASK = (QRF_MLD | - QRF_SORTDATA | - QRF_COVERAGE_NODES | - QRF_EXTENDED_COVERAGE | - QRF_COVERAGE | - QRF_GROUPDATA | - QRF_PROPERTIES), - - FNET_QF_SUPPORTED_MASK = (QF_PARSEDQUERY | - QF_RANKP | - QF_SORTSPEC | - QF_LOCATION | - QF_PROPERTIES | - QF_GROUPSPEC | - QF_SESSIONID), - - FNET_GDF_SUPPORTED_MASK = (GDF_MLD | - GDF_QUERYSTACK | - GDF_RANKP_QFLAGS | - GDF_LOCATION | - GDF_RESCLASSNAME | - GDF_PROPERTIES | - GDF_FLAGS), - ACTIVE_QUERY_FLAGS = (QFLAG_EXTENDED_COVERAGE | - QFLAG_COVERAGE_NODES | - QFLAG_ESTIMATE | - QFLAG_DROP_SORTDATA | - QFLAG_NO_RESULTCACHE | - QFLAG_DUMP_FEATURES), - - FNET_MQF_SUPPORTED_MASK = (MQF_QFLAGS), - - FNET_MRF_SUPPORTED_MASK = (MRF_MLD | MRF_RFLAGS | MRF_ACTIVEDOCS) -}; - -enum pcode_mask { - PCODE_MASK = 0x00ffffff -}; - -//========================================================================== - -class PacketArray -{ -private: - PacketArray(const PacketArray &); - PacketArray& operator=(const PacketArray &); - - FNET_Packet **_extArray; - FNET_Packet **_array; - uint32_t _size; - uint32_t _used; - -public: - PacketArray(FNET_Packet **arr = nullptr, uint32_t size = 0) - : _extArray(arr), - _array(arr), - _size(size), - _used(0) {} - ~PacketArray() - { - if (_array != _extArray) - delete [] _array; - } - void Add(FNET_Packet *packet) - { - if (_used == _size) { - _size *= 2; - if (_size < 16) - _size = 16; - FNET_Packet **newArray = new FNET_Packet*[_size]; - for (uint32_t i = 0; i < _used; i++) - newArray[i] = _array[i]; - if (_array != _extArray) - delete [] _array; - _array = newArray; - } - _array[_used++] = packet; - } - FNET_Packet **Array() const { return _array; } - uint32_t Length() const { return _used; } -}; - -//========================================================================== - -class FS4PacketFactory -{ -public: - typedef FNET_Packet *(* CreatePacket_t)(uint32_t pcode); - - static FNET_Packet *CreateFS4Packet(uint32_t pcode); -}; - -//========================================================================== - -class FS4PersistentPacketStreamer : public FNET_IPacketStreamer { - FS4PersistentPacketStreamer(const FS4PersistentPacketStreamer &); - FS4PersistentPacketStreamer& operator=(const FS4PersistentPacketStreamer &); +class FS4PersistentPacketStreamer { using CompressionConfig = vespalib::compression::CompressionConfig; unsigned int _compressionLimit; unsigned int _compressionLevel; CompressionConfig::Type _compressionType; -protected: - bool _conservative; // Set to true if out of sync should mark the - // stream as broken. - FS4PacketFactory::CreatePacket_t _createPacket; - - bool HasChannelID(uint32_t pcode); - bool ValidPCode(uint32_t pcode) const { - return ((pcode & PCODE_MASK) >= PCODE_EOL) - && ((pcode & PCODE_MASK) < PCODE_LastCode); - } public: static FS4PersistentPacketStreamer Instance; - FS4PersistentPacketStreamer(FS4PacketFactory::CreatePacket_t cp); + FS4PersistentPacketStreamer(); - bool GetPacketInfo(FNET_DataBuffer *src, uint32_t *plen, - uint32_t *pcode, uint32_t *chid, bool *broken) override; - FNET_Packet *Decode(FNET_DataBuffer *src, uint32_t plen, - uint32_t pcode, FNET_Context context) override; - void Encode(FNET_Packet *packet, uint32_t chid, FNET_DataBuffer *dst) override; - - void SetConservativeMode(bool cons) { _conservative = cons; } void SetCompressionLimit(unsigned int limit) { _compressionLimit = limit; } void SetCompressionLevel(unsigned int level) { _compressionLevel = level; } void SetCompressionType(CompressionConfig::Type compressionType) { _compressionType = compressionType; } @@ -166,13 +46,13 @@ private: vespalib::string _backing; const char * c_str(size_t sz) const { return _backing.c_str() + sz; } void set(StringRef & e, vespalib::stringref s); + void allocEntries(uint32_t cnt); public: FS4Properties(FS4Properties &&); FS4Properties &operator=(FS4Properties &&); FS4Properties(); ~FS4Properties(); - void allocEntries(uint32_t cnt); void setName(const char *name, uint32_t nameSize) { _name.assign(name, nameSize); } void setName(vespalib::stringref val) { setName(val.data(), val.size()); @@ -195,355 +75,10 @@ public: // sub-packet methods below uint32_t getLength(); + void encode(FNET_DataBuffer &dst); bool decode(FNET_DataBuffer &src, uint32_t &len); vespalib::string toString(uint32_t indent = 0) const; }; -//========================================================================== - -typedef std::vector<FS4Properties> PropsVector; - -//========================================================================== - -class FS4Packet : public FNET_Packet -{ -private: - FS4Packet(const FS4Packet &); - FS4Packet& operator=(const FS4Packet &); - -public: - FS4Packet(); - ~FS4Packet(); - vespalib::string Print(uint32_t indent) override; - void Free() override; - virtual vespalib::string toString(uint32_t indent) const = 0; -protected: - uint32_t readUInt32(FNET_DataBuffer & buf, uint32_t & len, const char *text) __attribute__((noinline)); - void throwNotEnoughData(FNET_DataBuffer & buf, uint32_t left, uint32_t needed, const char * text) __attribute__((noinline)); - void throwUnsupportedFeatures(uint32_t features, uint32_t set) __attribute__((noinline)); - void throwPropertieDecodeError(size_t i) __attribute__((noinline)); -}; - -//========================================================================== - -class FS4Packet_EOL : public FS4Packet -{ -public: - FS4Packet_EOL(); - ~FS4Packet_EOL(); - uint32_t GetPCODE() override { return PCODE_EOL; } - uint32_t GetLength() override; - void Encode(FNET_DataBuffer *dst) override; - bool Decode(FNET_DataBuffer *src, uint32_t len) override; - vespalib::string toString(uint32_t indent) const override; -}; - -class FS4Packet_PreSerialized : public FS4Packet -{ -public: - FS4Packet_PreSerialized(FNET_Packet & packet); - ~FS4Packet_PreSerialized(); - uint32_t GetPCODE() override; - uint32_t GetLength() override; - void Encode(FNET_DataBuffer *dst) override; - bool Decode(FNET_DataBuffer *src, uint32_t len) override; - vespalib::string toString(uint32_t indent) const override; -private: - using CompressionConfig = vespalib::compression::CompressionConfig; - uint32_t _pcode; - CompressionConfig::Type _compressionType; - FNET_DataBuffer _data; -}; - -class FS4Packet_Shared : public FS4Packet -{ -public: - FS4Packet_Shared(FNET_Packet::SP packet); - ~FS4Packet_Shared(); - uint32_t GetPCODE() override; - uint32_t GetLength() override; - void Encode(FNET_DataBuffer *dst) override; - bool Decode(FNET_DataBuffer *, uint32_t ) override; - vespalib::string toString(uint32_t indent) const override; -private: - FNET_Packet::SP _packet; -}; - -//========================================================================== - -class FS4Packet_ERROR : public FS4Packet -{ -private: - FS4Packet_ERROR(const FS4Packet_ERROR &); - FS4Packet_ERROR& operator=(const FS4Packet_ERROR &); - -public: - uint32_t _errorCode; - string _message; - - void setErrorMessage(vespalib::stringref msg) { _message = msg; } - - FS4Packet_ERROR(); - ~FS4Packet_ERROR(); - uint32_t GetPCODE() override { return PCODE_ERROR; } - uint32_t GetLength() override; - void Encode(FNET_DataBuffer *dst) override; - bool Decode(FNET_DataBuffer *src, uint32_t len) override; - vespalib::string toString(uint32_t indent) const override; -}; - -//========================================================================== - -class FS4Packet_DOCSUM : public FS4Packet -{ -public: - typedef vespalib::MallocPtr Buf; -private: - FS4Packet_DOCSUM(const FS4Packet_DOCSUM &); - FS4Packet_DOCSUM& operator=(const FS4Packet_DOCSUM &); - - document::GlobalId _gid; - Buf _buf; -public: - FS4Packet_DOCSUM(); - ~FS4Packet_DOCSUM(); - const Buf & getBuf() const { return _buf; } - void swapBuf(Buf & other) { _buf.swap(other); } - void setGid(const document::GlobalId & gid) { _gid = gid; } - const document::GlobalId & getGid() const { return _gid; } - bool empty() const { return _buf.empty(); } - void SetBuf(const char *buf, uint32_t len); - uint32_t GetPCODE() override { return PCODE_DOCSUM; } - uint32_t GetLength() override { return sizeof(_gid) + _buf.size(); } - void Encode(FNET_DataBuffer *dst) override; - bool Decode(FNET_DataBuffer *src, uint32_t len) override; - vespalib::string toString(uint32_t indent) const override; -}; - -//========================================================================== - -class FS4Packet_MONITORQUERYX : public FS4Packet -{ - FS4Packet_MONITORQUERYX(const FS4Packet_MONITORQUERYX &); - FS4Packet_MONITORQUERYX& operator=(const FS4Packet_MONITORQUERYX &); - -public: - uint32_t _features; // see monitorquery_features - uint32_t _qflags; // if MQF_QFLAGS - - FS4Packet_MONITORQUERYX(); - ~FS4Packet_MONITORQUERYX(); - uint32_t GetPCODE() override { return PCODE_MONITORQUERYX; } - uint32_t GetLength() override; - void Encode(FNET_DataBuffer *dst) override; - bool Decode(FNET_DataBuffer *src, uint32_t len) override; - vespalib::string toString(uint32_t indent) const override; -}; - -//========================================================================== - -class FS4Packet_MONITORRESULTX : public FS4Packet -{ -private: - FS4Packet_MONITORRESULTX(const FS4Packet_MONITORRESULTX &); - FS4Packet_MONITORRESULTX& operator=(const FS4Packet_MONITORRESULTX &); - -public: - uint32_t _features; // see monitor - uint32_t _partid; - uint32_t _timestamp; - - uint32_t _totalNodes; // if MRF_MLD - uint32_t _activeNodes; // if MRF_MLD - uint32_t _totalParts; // if MRF_MLD - uint32_t _activeParts; // if MRF_MLD - - uint32_t _rflags; // if MRF_RFLAGS - uint64_t _activeDocs; // if MRF_ACTIVEDOCS - - FS4Packet_MONITORRESULTX(); - ~FS4Packet_MONITORRESULTX(); - uint32_t GetPCODE() override { return PCODE_MONITORRESULTX; } - uint32_t GetLength() override; - void Encode(FNET_DataBuffer *dst) override; - bool Decode(FNET_DataBuffer *src, uint32_t len) override; - vespalib::string toString(uint32_t indent) const override; -}; - -//========================================================================== - -class FS4Packet_QUERYRESULTX : public FS4Packet -{ -private: - FS4Packet_QUERYRESULTX(const FS4Packet_QUERYRESULTX &); - FS4Packet_QUERYRESULTX& operator=(const FS4Packet_QUERYRESULTX &); - - uint32_t _distributionKey; - uint16_t _nodesQueried; - uint16_t _nodesReplied; - -public: - uint32_t _features; // see queryresult_features - uint32_t _offset; - uint32_t _numDocs; - uint64_t _totNumDocs; - search::HitRank _maxRank; - uint32_t *_sortIndex; // if QRF_SORTDATA - char *_sortData; // if QRF_SORTDATA - uint32_t _groupDataLen; // if QRF_GROUPDATA - char *_groupData; // if QRF_GROUPDATA - uint64_t _coverageDocs; // if QRF_COVERAGE - uint64_t _activeDocs; // if QRF_COVERAGE - uint64_t _soonActiveDocs; // if QRF_EXTENDED_COVERAGE - uint32_t _coverageDegradeReason; // if QRF_EXTENDED_COVERAGE - class FS4_hit { - public: - FS4_hit() : _gid(), _metric(0), _partid(0), _distributionKey(0) { } - uint32_t getDistributionKey() const { return _distributionKey; } - void setDistributionKey(uint32_t key) { _distributionKey = key; } - const document::GlobalId & HT_GetGlobalID() const { return _gid; } - search::HitRank HT_GetMetric() const { return _metric; } - uint32_t HT_GetPartID() const { return _partid; } - - void HT_SetGlobalID(const document::GlobalId & val) { _gid = val; } - void HT_SetMetric(search::HitRank val) { _metric = val; } - void HT_SetPartID(uint32_t val) { _partid = val; } - document::GlobalId _gid; - search::HitRank _metric; - uint32_t _partid; // if QRF_MLD - private: - uint32_t _distributionKey; // if QRF_MLD - } *_hits; - PropsVector _propsVector; // if QRF_PROPERTIES - - void AllocateSortIndex(uint32_t cnt); - void AllocateSortData(uint32_t len); - void SetSortDataRef(uint32_t cnt, uint32_t *sortIndex, const char *sortData); - void AllocateGroupData(uint32_t len); - void SetGroupDataRef(const char *groupData, uint32_t len); - void AllocateHits(uint32_t cnt); - - FS4Packet_QUERYRESULTX(); - ~FS4Packet_QUERYRESULTX(); - uint32_t GetPCODE() override { return PCODE_QUERYRESULTX; } - uint32_t GetLength() override; - void Encode(FNET_DataBuffer *dst) override; - bool Decode(FNET_DataBuffer *src, uint32_t len) override ; - vespalib::string toString(uint32_t indent) const override ; - uint32_t getDistributionKey() const { return _distributionKey; } - void setDistributionKey(uint32_t key) { _distributionKey = key; } - uint16_t getNodesQueried() const { return _nodesQueried; } - void setNodesQueried(uint16_t key) { _nodesQueried = key; } - uint16_t getNodesReplied() const { return _nodesReplied; } - void setNodesReplied(uint16_t key) { _nodesReplied = key; } -}; -//========================================================================== - -class FS4Packet_QUERYX : public FS4Packet -{ -private: - FS4Packet_QUERYX(const FS4Packet_QUERYX &); - FS4Packet_QUERYX& operator=(const FS4Packet_QUERYX &); - - uint32_t _timeout; - uint32_t _qflags; - -public: - uint32_t _features; // see query_features - uint32_t _offset; - uint32_t _maxhits; - string _ranking; // if QF_RANKP - PropsVector _propsVector; // if QF_PROPERTIES - string _sortSpec; // if QF_SORTSPEC - string _groupSpec; // if QF_GROUPSPEC - string _sessionId; // if QF_SESSIONID - string _location; // if QF_LOCATION - - uint32_t _numStackItems; // if QF_PARSEDQUERY - string _stackDump; // if QF_PARSEDQUERY - - void setQueryFlags(uint32_t qflags) { _qflags = ACTIVE_QUERY_FLAGS & qflags; } - void setRanking(vespalib::stringref ranking) { _ranking = ranking; } - void setSortSpec(vespalib::stringref spec) { _sortSpec = spec; } - void setGroupSpec(vespalib::stringref spec) { _groupSpec = spec; } - void setSessionId(vespalib::stringref sid) { _sessionId = sid; } - void setLocation(vespalib::stringref loc) { _location = loc; } - void setStackDump(vespalib::stringref buf) { _stackDump = buf; } - void setTimeout(const fastos::TimeStamp & timeout); - fastos::TimeStamp getTimeout() const; - uint32_t getQueryFlags() const { return _qflags; } - - explicit FS4Packet_QUERYX(); - ~FS4Packet_QUERYX(); - uint32_t GetPCODE() override { return PCODE_QUERYX; } - uint32_t GetLength() override; - void Encode(FNET_DataBuffer *dst) override; - bool Decode(FNET_DataBuffer *src, uint32_t len) override; - vespalib::string toString(uint32_t indent) const override; -}; - -//========================================================================== - -class FS4Packet_GETDOCSUMSX : public FS4Packet -{ -private: - FS4Packet_GETDOCSUMSX(const FS4Packet_GETDOCSUMSX &); - FS4Packet_GETDOCSUMSX& operator=(const FS4Packet_GETDOCSUMSX &); - - uint32_t _timeout; -public: - uint32_t _features; // see getdocsums_features - string _ranking; // if GDF_RANKP_QFLAGS - uint32_t _qflags; // if GDF_RANKP_QFLAGS - string _resultClassName; // if GDF_RESCLASSNAME - PropsVector _propsVector; // if GDF_PROPERTIES - uint32_t _stackItems; // if GDF_QUERYSTACK - string _stackDump; // if GDF_QUERYSTACK - string _location; // if GDF_LOCATION - uint32_t _flags; // if GDF_FLAGS - class FS4_docid { - public: - FS4_docid() : _gid(), _partid(0) { } - document::GlobalId _gid; - uint32_t _partid; // if GDF_MLD - }; - std::vector<FS4_docid> _docid; - - void AllocateDocIDs(uint32_t cnt); - - void setResultClassName(vespalib::stringref name) { _resultClassName = name; } - void setStackDump(vespalib::stringref buf) { _stackDump = buf; } - void setRanking(vespalib::stringref ranking) { _ranking = ranking; } - void setLocation(vespalib::stringref loc) { _location = loc; } - void setTimeout(const fastos::TimeStamp & timeout); - fastos::TimeStamp getTimeout() const; - - FS4Packet_GETDOCSUMSX(); - ~FS4Packet_GETDOCSUMSX(); - uint32_t GetPCODE() override { return PCODE_GETDOCSUMSX; } - uint32_t GetLength() override; - void Encode(FNET_DataBuffer *dst) override; - bool Decode(FNET_DataBuffer *src, uint32_t len) override; - vespalib::string toString(uint32_t indent) const override; -}; - -//========================================================================== - -class FS4Packet_TRACEREPLY : public FS4Packet -{ -public: - FS4Packet_TRACEREPLY() {} - ~FS4Packet_TRACEREPLY() {} - uint32_t GetPCODE() override { return PCODE_TRACEREPLY; } - uint32_t GetLength() override; - void Encode(FNET_DataBuffer *dst) override; - bool Decode(FNET_DataBuffer *src, uint32_t len) override; - vespalib::string toString(uint32_t indent) const override; - - PropsVector _propsVector; -}; - -//========================================================================== - } diff --git a/searchlib/src/vespa/searchlib/common/transport.h b/searchlib/src/vespa/searchlib/common/transport.h deleted file mode 100644 index 51098112476..00000000000 --- a/searchlib/src/vespa/searchlib/common/transport.h +++ /dev/null @@ -1,286 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <cstdint> - -namespace search::fs4transport { - -/** - * Instead of using a 32-bit number to send the 'usehardware' flag, we - * now use this 32-bit number to send 32 flags. The currently defined flags - * are as follows: - * <ul> - * <li><b>QFLAG_EXTENDED_COVERAGE</b>: Indicates that the it is able to receive extended coverage information.</li> - * <li><b>QFLAG_COVERAGE_NODES</b>: Indicate that it is able to handle nodes information.</li> - * <li><b>QFLAG_ESTIMATE</b>: Indicates that the query is performed to get - * an estimate of the total number of hits</li> - * <li><b>QFLAG_DUMP_FEATURES</b>: Dump detailed ranking information. Note that - * this flag will only be considered when sent in a - * GETDOCSUMSX packet. Is is put here to avoid having - * 2 separate query related flag spaces</li> - * <li><b>QFLAG_DROP_SORTDATA</b>: Don't return any sort data even if sortspec - * is used.</li> - * <li><b>QFLAG_NO_RESULTCACHE</b>: Do not use any result cache. Perform query no matter what.</li> - * </ul> - **/ -enum queryflags { - QFLAG_EXTENDED_COVERAGE = 0x00000001, - QFLAG_COVERAGE_NODES = 0x00000002, - QFLAG_ESTIMATE = 0x00000080, - QFLAG_DROP_SORTDATA = 0x00004000, - QFLAG_NO_RESULTCACHE = 0x00010000, - QFLAG_DUMP_FEATURES = 0x00040000 -}; - - -/** - * The new PCODE_QUERYRESULTX packet contains a 32-bit field called - * 'featureflags'. Each bit in that field denotes a separate feature - * that may be present in the query result packet or not. The comment - * describing the packet format indicates what data fields depend on - * what features. The features present in the 'old' query result packets - * are defined in this enum along with the Query Result Features - * themselves. The value called QRF_SUPPORTED_MASK denotes which - * features are supported by the current version. If a packet with - * unknown features is received on the network is is discarded (as it - * would be if it had an illegal PCODE). - **/ -enum queryresult_features { - QRF_MLD = 0x00000001, - QRF_COVERAGE_NODES = 0x00000002, - QRF_SORTDATA = 0x00000010, - QRF_EXTENDED_COVERAGE = 0x00000020, - QRF_COVERAGE = 0x00000040, - QRF_GROUPDATA = 0x00000200, - QRF_PROPERTIES = 0x00000400 -}; - - -/** - * The new PCODE_QUERYX packet contains a 32-bit field called - * 'featureflags'. Each bit in that field denotes a separate feature - * that may be present in the query packet or not. The comment - * describing the packet format indicates what data fields depend on - * what features. The features present in the - * 'old' query packets are defined in this enum along with the Query - * Features themselves. The values called - * QF_SUPPORTED_[FSEARCH/FDISPATCH]_MASK denotes which features are - * supported by the current version. If a packet with unknown features - * is received on the network is is discarded (as it would be if it - * had an illegal PCODE). - **/ -enum query_features { - QF_PARSEDQUERY = 0x00000002, - QF_RANKP = 0x00000004, - QF_SORTSPEC = 0x00000080, - QF_LOCATION = 0x00000800, - QF_PROPERTIES = 0x00100000, - QF_GROUPSPEC = 0x00400000, - QF_SESSIONID = 0x00800000 -}; - - -/** - * The new PCODE_GETDOCSUMSX packet contains a 32-bit field called - * 'featureflags'. Each bit in that field denotes a separate feature - * that may be present in the getdocsums packet or not. The comment - * describing the packet format indicates what data fields depend on - * what features. The features present in the 'old' getdocsums packets are - * defined in this enum along with the GetDocsums Features - * themselves. The values called - * GDF_SUPPORTED_[FSEARCH/FDISPATCH]_MASK denotes which features are - * supported by the current version. If a packet with unknown features - * is received on the network is is discarded (as it would be if it - * had an illegal PCODE). - **/ -enum getdocsums_features { - GDF_MLD = 0x00000001, - GDF_QUERYSTACK = 0x00000004, - GDF_RANKP_QFLAGS = 0x00000010, - GDF_LOCATION = 0x00000080, - GDF_RESCLASSNAME = 0x00000800, - GDF_PROPERTIES = 0x00001000, - GDF_FLAGS = 0x00002000 -}; - - -enum getdocsums_flags { - GDFLAG_IGNORE_ROW = 0x00000001 -}; - -// docsum class for slime tunneling -const uint32_t SLIME_MAGIC_ID = 0x55555555; - -enum monitorquery_features { - MQF_QFLAGS = 0x00000002, -}; - -enum monitorquery_flags { - MQFLAG_REPORT_ACTIVEDOCS = 0x00000020 -}; - -enum monitorresult_features { - MRF_MLD = 0x00000001, - MRF_RFLAGS = 0x00000008, - MRF_ACTIVEDOCS = 0x00000010, -}; - -/** - * Codes for packets between dispatch nodes and search nodes. - * general packet (i.e. message) format: - * uint32_t packetLength- length in bytes, EXCLUDING this length field - * packetcode pCode - see the enum below; same length as uint32_t - * packetData - variable length - */ -enum packetcode { - PCODE_EOL = 200, /* ..fdispatch <-> ..fsearch. PacketData: - *0 {uint32_t queryId,} - only in new format!*/ - PCODE_QUERY_NOTUSED = 201, - PCODE_QUERYRESULT_NOTUSED = 202, - PCODE_ERROR = 203, /* ..fdispatch <- ..fsearch/..fdispatch - * {uint32_t queryId,} - only in new format! - * uint32_t error_code [see common/errorcodes.h] - * uint32_t message_len - * char[] message (UTF-8) */ - PCODE_GETDOCSUMS_NOTUSED = 204, - PCODE_DOCSUM = 205, /* ..fdispatch <- ..fsearch. - *0 {uint32_t queryId,} - only in new format! - *1 uint32_t location - *2 char[] <title, incipit, URL, ...> - */ - PCODE_MONITORQUERY_NOTUSED = 206, - PCODE_MONITORRESULT_NOTUSED = 207, - PCODE_MLD_QUERYRESULT_NOTUSED = 208, - PCODE_MLD_GETDOCSUMS_NOTUSED = 209, - PCODE_MLD_MONITORRESULT_NOTUSED = 210, - PCODE_CLEARCACHES_NOTUSED = 211, - PCODE_QUERY2_NOTUSED = 212, - PCODE_PARSEDQUERY2_NOTUSED = 213, - PCODE_MLD_QUERYRESULT2_NOTUSED = 214, - PCODE_MLD_GETDOCSUMS2_NOTUSED = 215, - PCODE_QUEUELEN_NOTUSED = 216, - - PCODE_QUERYRESULTX = 217, /* - * {uint32_t queryId,} - only if persistent - * uint32_t featureflags, - see 'queryresult_features' - * uint32_t offset, - * uint32_t numDocs, - * uint32_t totNumDocs, - * search::HitRank maxRank, - * uint32_t docstamp, - * uint32_t[numDocs] sortIndex - if QRF_SORTDATA - * char[sidx[n - 1]] sortData - if QRF_SORTDATA - * uint32_t groupDataLen - if QRF_GROUPDATA - * char[groupDataLen] groupData - if QRF_GROUPDATA - * uint64_t coverageDocs - if QRF_COVERAGE - * uint32_t coverageNodes - if QRF_COVERAGE - * uint32_t coverageFull - if QRF_COVERAGE - * numDocs * hit { - * uint32_t docid, - * search::HitRank metric, - * uint32_t partid, - if QRF_MLD - * uint32_t docstamp, - if QRF_MLD - * } */ - PCODE_QUERYX = 218, /* - * {uint32_t queryId,} - only if persistent - * uint32_t featureflags, - see 'query_features' - * uint32_t querytype - * uint32_t offset, - * uint32_t maxhits, - * uint32_t qflags, - * uint32_t minhits, - if QF_MINHITS - * uint32_t numProperties - if QF_PROPERTIES - * numProperties * props { - if QF_PROPERTIES - * uint32_t nameLen - * char[nameLen] name - * uint32_t numEntries - * numentries * entry { - * uint32_t keyLen - * char[keyLen] key - * uint32_t valueLen - * char[valueLen] value - * } - * } - * uint32_t sortSpecLen - if QF_SORTSPEC - * char[sortSpecLen] sortSpec - if QF_SORTSPEC - * uint32_t groupSpecLen - if QF_GROUPSPEC - * char[groupSpecLen] groupSpec - if QF_GROUPSPEC - * uint32_t locationLen - if QF_LOCATION - * char[locationLen] location - if QF_LOCATION - * uint32_t numStackItems, - if QF_PARSEDQUERY - * multiple encoded stackitems: - if QF_PARSEDQUERY - - uint32_t OR|AND|NOT|RANK - uint32_t arity - - uint32_t PHRASE - uint32_t arity - uint32_t indexNameLen - char[] indexName - - uint32_t TERM - uint32_t indexNameLen - char[] indexName - uint32_t termLen - char[] term - */ - PCODE_GETDOCSUMSX = 219, /* - * {uint32_t queryId,} - only if persistent - * uint32_t featureflags, - see 'getdocsums_features' - * uint32_t docstamp, - * uint32_t rankprofile, - if GDF_RANKP_QFLAGS - * uint32_t qflags, - if GDF_RANKP_QFLAGS - * uint32_t resClassNameLen - if GDF_RESCLASSNAME - * char [] resClassName - if GDF_RESCLASSNAME - * uint32_t numProperties - if GDF_PROPERTIES - * numProperties * props { - if GDF_PROPERTIES - * uint32_t nameLen - * char[nameLen] name - * uint32_t numEntries - * numentries * entry { - * uint32_t keyLen - * char[keyLen] key - * uint32_t valueLen - * char[valueLen] value - * } - * } - * uint32_t stackItems, - if GDF_STACKDUMP - * uint32_t stackDumpLen, - if GDF_STACKDUMP - * char[stackDumpLen] stackDump, - if GDF_STACKDUMP - * uint32_t locationLen - if GDF_LOCATION - * char[locationLen] location - if GDF_LOCATION - * N * doc { - * uint32_t docid, - * uint32_t partid, - if GDF_MLD - * uint32_t docstamp, - if GDF_MLD - * } - */ - PCODE_MONITORQUERYX = 220, /* - * uint32_t featureFlags; - * - see monitorquery_features - */ - PCODE_MONITORRESULTX = 221, /* - * uint32_t featureFlags; - * - see monitorresult_features - * uint32_t partitionId; - * uint32_t timestamp; - * uint32_t totalNodes; - if MRF_MLD - * uint32_t activeNodes; - if MRF_MLD - * uint32_t totalParts; - if MRF_MLD - * uint32_t activeParts; - if MRF_MLD - */ - PCODE_TRACEREPLY = 222, /* - * numProperties * props { - * uint32_t nameLen - * char[nameLen] name - * uint32_t numEntries - * numentries * entry { - * uint32_t keyLen - * char[keyLen] key - * uint32_t valueLen - * char[valueLen] value - * } - * } - */ - PCODE_LastCode = 223 // Used for consistency checking only, must be last. -}; - -} diff --git a/searchlib/src/vespa/searchlib/engine/CMakeLists.txt b/searchlib/src/vespa/searchlib/engine/CMakeLists.txt index 36ff4e12108..21a5b232ae0 100644 --- a/searchlib/src/vespa/searchlib/engine/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/engine/CMakeLists.txt @@ -14,11 +14,9 @@ vespa_add_library(searchlib_engine OBJECT docsumapi.cpp docsumreply.cpp docsumrequest.cpp - errorcodes.cpp lazy_source.cpp monitorreply.cpp monitorrequest.cpp - packetconverter.cpp propertiesmap.cpp proto_converter.cpp proto_rpc_adapter.cpp @@ -27,8 +25,6 @@ vespa_add_library(searchlib_engine OBJECT searchreply.cpp searchrequest.cpp trace.cpp - transport_metrics.cpp - transportserver.cpp ${searchlib_engine_PROTOBUF_SRCS} DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/engine/docsumrequest.cpp b/searchlib/src/vespa/searchlib/engine/docsumrequest.cpp index e2dffb54e56..33a2c437a80 100644 --- a/searchlib/src/vespa/searchlib/engine/docsumrequest.cpp +++ b/searchlib/src/vespa/searchlib/engine/docsumrequest.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "docsumrequest.h" -#include "packetconverter.h" namespace search::engine { diff --git a/searchlib/src/vespa/searchlib/engine/errorcodes.cpp b/searchlib/src/vespa/searchlib/engine/errorcodes.cpp deleted file mode 100644 index 0182d85177a..00000000000 --- a/searchlib/src/vespa/searchlib/engine/errorcodes.cpp +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "errorcodes.h" - -namespace search::engine { - -const char * -getStringFromErrorCode(ErrorCode ecode) -{ - switch (ecode) { - case ECODE_NO_ERROR: - return "No error has occurred"; - case ECODE_GENERAL_ERROR: - return "General error"; - case ECODE_QUERY_PARSE_ERROR: - return "Error parsing query"; - case ECODE_ALL_PARTITIONS_DOWN: - return "All searchnodes are down. This might indicate that no index is available yet."; - case ECODE_ILLEGAL_DATASET: - return "No such dataset"; - case ECODE_OVERLOADED: - return "System is overloaded"; - case ECODE_NOT_IMPLEMENTED: - return "The requested functionality is not implemented"; - case ECODE_QUERY_NOT_ALLOWED: - return "Query not allowed to run"; - case ECODE_TIMEOUT: - return "Query timed out"; - } - return "Unknown error"; -} - -} - diff --git a/searchlib/src/vespa/searchlib/engine/errorcodes.h b/searchlib/src/vespa/searchlib/engine/errorcodes.h deleted file mode 100644 index d0812ad9d6c..00000000000 --- a/searchlib/src/vespa/searchlib/engine/errorcodes.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -namespace search::engine { - -/** - * Enum defining global error codes. - * Used in error_code field in search::fs4transport::PCODE_ERROR packets. - **/ -enum ErrorCode { - ECODE_NO_ERROR = 0, - ECODE_GENERAL_ERROR = 1, - ECODE_QUERY_PARSE_ERROR = 2, - ECODE_ALL_PARTITIONS_DOWN = 3, - ECODE_ILLEGAL_DATASET = 4, - ECODE_OVERLOADED = 5, - ECODE_NOT_IMPLEMENTED = 6, - ECODE_QUERY_NOT_ALLOWED = 7, - ECODE_TIMEOUT = 8 -}; - -/** - * Normally error codes should be accompanied by an error message - * describing the error. If no such message is present, this method - * may be used to obtain the default description of an error code. - * - * @param error the error code we want info about. - * @return the default error message for the given error code. - **/ -const char* getStringFromErrorCode(ErrorCode error); - -} - diff --git a/searchlib/src/vespa/searchlib/engine/packetconverter.cpp b/searchlib/src/vespa/searchlib/engine/packetconverter.cpp deleted file mode 100644 index 863c204f26c..00000000000 --- a/searchlib/src/vespa/searchlib/engine/packetconverter.cpp +++ /dev/null @@ -1,264 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "packetconverter.h" - -#include <vespa/log/log.h> -LOG_SETUP(".engine.packetconverter"); - -using search::fef::Property; -using search::fef::Properties; - -namespace { - -bool checkFeature(uint32_t features, uint32_t mask) { - return ((features & mask) != 0); -} - -struct FS4PropertiesBuilder : public search::fef::IPropertiesVisitor { - uint32_t idx; - search::fs4transport::FS4Properties &props; - FS4PropertiesBuilder(search::fs4transport::FS4Properties &p) : idx(0), props(p) {} - void visitProperty(const Property::Value &key, const Property &values) override { - for (uint32_t i = 0; i < values.size(); ++i) { - props.setKey(idx, key.data(), key.size()); - props.setValue(idx, values.getAt(i).data(), values.getAt(i).size()); - ++idx; - } - } -}; - -} // namespace <unnamed> - -namespace search::engine { - -using namespace search::fs4transport; - -void -PacketConverter::fillPacketProperties(const PropertiesMap &source, PropsVector& target) -{ - target.resize(source.size()); - PropertiesMap::ITR itr = source.begin(); - PropertiesMap::ITR end = source.end(); - for (uint32_t i = 0; itr != end; ++itr, ++i) { - const vespalib::string &name = itr->first; - const Properties &values = itr->second; - target[i].setName(name.c_str(), name.size()); - target[i].allocEntries(values.numValues()); - FS4PropertiesBuilder builder(target[i]); - values.visitProperties(builder); - LOG_ASSERT(builder.idx == target[i].size()); - LOG_ASSERT(builder.idx == values.numValues()); - } -} - -void -PacketConverter::toSearchRequest(const QUERYX &packet, SearchRequest &request) -{ - request.offset = packet._offset; - request.maxhits = packet._maxhits; - request.setTimeout(packet.getTimeout()); - request.queryFlags = packet.getQueryFlags(); - request.ranking = packet._ranking; - - for (const FS4Properties &src : packet._propsVector) { - Properties &dst = request.propertiesMap.lookupCreate(src.getName()); - for (uint32_t e = 0; e < src.size(); ++e) { - dst.add(vespalib::stringref(src.getKey(e), src.getKeyLen(e)), - vespalib::stringref(src.getValue(e), src.getValueLen(e))); - } - } - request.sortSpec = packet._sortSpec; - request.groupSpec.assign( packet._groupSpec.begin(), packet._groupSpec.end()); - request.sessionId.assign( packet._sessionId.begin(), packet._sessionId.end()); - request.location = packet._location; - request.stackItems = packet._numStackItems; - request.stackDump.assign( packet._stackDump.begin(), packet._stackDump.end()); -} - -void -PacketConverter::fromSearchRequest(const SearchRequest &request, QUERYX &packet) -{ - // not needed yet - (void) packet; - (void) request; - LOG_ABORT("not implemented"); -} - -void -PacketConverter::toSearchReply(const QUERYRESULTX &packet, SearchReply &reply) -{ - // not needed yet - (void) packet; - (void) reply; - LOG_ABORT("not implemented"); -} - -void -PacketConverter::fromSearchReply(const SearchReply &reply, QUERYRESULTX &packet) -{ - packet._offset = reply.offset; - packet._numDocs = reply.hits.size(); - packet._totNumDocs = reply.totalHitCount; - packet._maxRank = reply.maxRank; - packet.setDistributionKey(reply.getDistributionKey()); - if ( ! reply.sortIndex.empty()) { - packet._features |= QRF_SORTDATA; - uint32_t idxCnt = reply.sortIndex.size(); - LOG_ASSERT(reply.sortIndex.size() == reply.hits.size()+1); - // allocate for N hits (will make space for N+1 indexes) - packet.AllocateSortIndex(reply.hits.size()); - packet.AllocateSortData(reply.sortData.size()); - for (uint32_t i = 0; i < idxCnt; ++i) { - packet._sortIndex[i] = reply.sortIndex[i]; - } - memcpy(packet._sortData, &(reply.sortData[0]), reply.sortData.size()); - } - if ( ! reply.groupResult.empty()) { - packet._features |= QRF_GROUPDATA; - packet.AllocateGroupData(reply.groupResult.size()); - memcpy(packet._groupData, &(reply.groupResult[0]), reply.groupResult.size()); - } - packet._coverageDocs = reply.coverage.getCovered(); - packet._activeDocs = reply.coverage.getActive(); - packet._soonActiveDocs = reply.coverage.getSoonActive(); - packet._coverageDegradeReason = reply.coverage.getDegradeReason(); - packet.setNodesQueried(reply.coverage.getNodesQueried()); - packet.setNodesReplied(reply.coverage.getNodesReplied()); - if (reply.request) { - if (reply.request->queryFlags & QFLAG_COVERAGE_NODES) { - packet._features |= QRF_COVERAGE_NODES; - } - } - if (reply.useWideHits) { - packet._features |= QRF_MLD; - } - if (reply.propertiesMap.size() > 0) { - fillPacketProperties(reply.propertiesMap, packet._propsVector); - packet._features |= QRF_PROPERTIES; - } - uint32_t hitCnt = reply.hits.size(); - packet.AllocateHits(hitCnt); - for (uint32_t i = 0; i < hitCnt; ++i) { - packet._hits[i]._gid = reply.hits[i].gid; - packet._hits[i]._metric = reply.hits[i].metric; - packet._hits[i]._partid = reply.hits[i].path; - packet._hits[i].setDistributionKey(reply.hits[i].getDistributionKey()); - } -} - -void -PacketConverter::toDocsumRequest(const GETDOCSUMSX &packet, DocsumRequest &request) -{ - request.setTimeout(packet.getTimeout()); - request.ranking = packet._ranking; - request.queryFlags = packet._qflags; - request.resultClassName = packet._resultClassName; - for (const FS4Properties &src : packet._propsVector) { - Properties &dst = request.propertiesMap.lookupCreate(src.getName()); - for (uint32_t e = 0; e < src.size(); ++e) { - dst.add(vespalib::stringref(src.getKey(e), src.getKeyLen(e)), - vespalib::stringref(src.getValue(e), src.getValueLen(e))); - } - } - request.stackItems = packet._stackItems; - request.stackDump.assign(packet._stackDump.begin(), packet._stackDump.end()); - request.location = packet._location; - request._flags = packet._flags; - request.useWideHits = checkFeature(packet._features, GDF_MLD); - uint32_t hitCnt = packet._docid.size(); - request.hits.resize(hitCnt); - for (uint32_t i = 0; i < hitCnt; ++i) { - request.hits[i].gid = packet._docid[i]._gid; - request.hits[i].path = packet._docid[i]._partid; - } - Property sessionId = request.propertiesMap.rankProperties().lookup("sessionId"); - if (sessionId.found()) { - vespalib::string id = sessionId.get(); - request.sessionId.assign(id.begin(), id.end()); - } -} - -void -PacketConverter::fromDocsumRequest(const DocsumRequest &request, GETDOCSUMSX &packet) -{ - // not needed yet - (void) packet; - (void) request; - LOG_ABORT("not implemented"); -} - -void -PacketConverter::toDocsumReplyElement(const DOCSUM &packet, DocsumReply::Docsum &docsum) -{ - // not needed yet - (void) packet; - (void) docsum; - LOG_ABORT("not implemented"); -} - -void -PacketConverter::fromDocsumReplyElement(const DocsumReply::Docsum &docsum, DOCSUM &packet) -{ - if (docsum.data.get() != nullptr) { - packet.SetBuf(docsum.data.c_str(), docsum.data.size()); - } - packet.setGid(docsum.gid); -} - -void -PacketConverter::toMonitorRequest(const MONITORQUERYX &packet, MonitorRequest &request) -{ - request.flags = packet._qflags; - if ((packet._qflags & MQFLAG_REPORT_ACTIVEDOCS) != 0) { - request.reportActiveDocs = true; - } -} - -void -PacketConverter::fromMonitorRequest(const MonitorRequest &request, MONITORQUERYX &packet) -{ - // not needed yet - (void) packet; - (void) request; - LOG_ABORT("not implemented"); -} - -void -PacketConverter::toMonitorReply(const MONITORRESULTX &packet, MonitorReply &reply) -{ - // not needed yet - (void) packet; - (void) reply; - LOG_ABORT("not implemented"); -} - -void -PacketConverter::fromMonitorReply(const MonitorReply &reply, MONITORRESULTX &packet) -{ - if (reply.mld) { - packet._features |= MRF_MLD; - } - if (reply.activeDocsRequested) { - packet._features |= MRF_ACTIVEDOCS; - packet._activeDocs = reply.activeDocs; - } - packet._partid = reply.partid; - packet._timestamp = reply.timestamp; - packet._totalNodes = reply.totalNodes; - packet._activeNodes = reply.activeNodes; - packet._totalParts = reply.totalParts; - packet._activeParts = reply.activeParts; - packet._rflags = reply.flags; - if (packet._rflags != 0) { - packet._features |= MRF_RFLAGS; - } -} - -void -PacketConverter::fromTraceReply(const TraceReply &reply, TRACEREPLY &packet) -{ - fillPacketProperties(reply.propertiesMap, packet._propsVector); -} - -} - diff --git a/searchlib/src/vespa/searchlib/engine/packetconverter.h b/searchlib/src/vespa/searchlib/engine/packetconverter.h deleted file mode 100644 index 90fc27c8c97..00000000000 --- a/searchlib/src/vespa/searchlib/engine/packetconverter.h +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "searchrequest.h" -#include "searchreply.h" -#include "docsumrequest.h" -#include "docsumreply.h" -#include "monitorrequest.h" -#include "monitorreply.h" -#include "tracereply.h" -#include <vespa/searchlib/common/packets.h> - -namespace search::engine { - - -/** - * This class helps convert data back and forth between transport - * packets and engine api request/reply objects. All converting - * methods expect the const object to be fully filled out and the - * non-const object to be newly created and thus empty. Half of the - * methods are left unimplemented for now as they would only be needed - * if we also were to use the api to wrap remote engines. However, if - * such a time comes, we will probably not be using the packet - * protocol anymore anyways. - **/ -class PacketConverter -{ -private: - PacketConverter(); // can not be instantiated - PacketConverter(const PacketConverter &); - PacketConverter &operator=(const PacketConverter &); - -public: - typedef search::fs4transport::FS4Packet_QUERYX QUERYX; - typedef search::fs4transport::FS4Packet_QUERYRESULTX QUERYRESULTX; - typedef search::fs4transport::FS4Packet_ERROR ERROR; - typedef search::fs4transport::FS4Packet_GETDOCSUMSX GETDOCSUMSX; - typedef search::fs4transport::FS4Packet_DOCSUM DOCSUM; - typedef search::fs4transport::FS4Packet_EOL EOL; - typedef search::fs4transport::FS4Packet_MONITORQUERYX MONITORQUERYX; - typedef search::fs4transport::FS4Packet_MONITORRESULTX MONITORRESULTX; - typedef search::fs4transport::FS4Packet_TRACEREPLY TRACEREPLY; - - /** - * Utility conversion from a "fef" set of propertymaps to an array of FS4Properties. - * @return false if no properties were converted. - **/ - static void - fillPacketProperties(const PropertiesMap &source, search::fs4transport::PropsVector& target); - - /** - * Convert from a QUERYX packet to a SearchRequest object. - * - * @param packet transport packet - * @param request api request object - **/ - static void toSearchRequest(const QUERYX &packet, SearchRequest &request); - - /** - * Convert from a SearchRequest object to a QUERYX packet. - * - * (NOT YET IMPLEMENTED) - * - * @param request api request object - * @param packet transport packet - **/ - static void fromSearchRequest(const SearchRequest &request, QUERYX &packet); - - /** - * Convert from a QUERYRESULTX packet to a SearchReply object. - * - * (NOT YET IMPLEMENTED) - * - * @param packet transport packet - * @param reply api reply object - **/ - static void toSearchReply(const QUERYRESULTX &packet, SearchReply &reply); - - /** - * Convert from a SearchReply object to a QUERYRESULTX - * packet. Note that this method only handles the query result - * aspect of the reply, errors and queue length reporting still - * needs to be handled separately by the code using this utility - * method. - * - * @param reply api reply object - * @param packet transport packet - **/ - static void fromSearchReply(const SearchReply &reply, QUERYRESULTX &packet); - - /** - * Convert from a GETDOCSUMSX packet to a DocsumRequest object. - * - * @param packet transport packet - * @param request api request object - **/ - static void toDocsumRequest(const GETDOCSUMSX &packet, DocsumRequest &request); - - /** - * Convert from a DocsumRequest object to a GETDOCSUMSX packet. - * - * (NOT YET IMPLEMENTED) - * - * @param packet transport packet - * @param request api request object - **/ - static void fromDocsumRequest(const DocsumRequest &request, GETDOCSUMSX &packet); - - /** - * Convert from a DOCSUM packet to an entry in a DocsumReply object - * - * (NOT YET IMPLEMENTED) - * - * @param packet transport packet - * @param docsum api reply object element - **/ - static void toDocsumReplyElement(const DOCSUM &packet, DocsumReply::Docsum &docsum); - - /** - * Convert from an entry in a DocsumReply object to a DOCSUM packet. - * - * @param docsum api reply object element - * @param packet transport packet - **/ - static void fromDocsumReplyElement(const DocsumReply::Docsum &docsum, DOCSUM &packet); - - /** - * Convert a MONITORQUERYX packet to a MonitorRequest object. - * - * @param packet transport packet - * @param request api request object - **/ - static void toMonitorRequest(const MONITORQUERYX &packet, MonitorRequest &request); - - /** - * Convert from a MonitorRequest object to a MONITORQUERYX packet - * - * (NOT YET IMPLEMENTED) - * - * @param request api request object - * @param packet transport packet - **/ - static void fromMonitorRequest(const MonitorRequest &request, MONITORQUERYX &packet); - - /** - * Convert from a MONITORRESULTX packet to a MonitorReply object. - * - * (NOT YET IMPLEMENTED) - * - * @param packet transport packet - * @param reply api reply object - **/ - static void toMonitorReply(const MONITORRESULTX &packet, MonitorReply &reply); - - /** - * Convert from a MonitorReply object to a MONITORRESULTX packet. - * - * @param reply api reply object - * @param packet transport packet - **/ - static void fromMonitorReply(const MonitorReply &reply, MONITORRESULTX &packet); - - /** - * Convert from a TraceReply object to a TRACE packet. - * - * @param reply api reply object - * @param packet transport packet - **/ - static void fromTraceReply(const TraceReply &reply, TRACEREPLY &packet); -}; - -} - diff --git a/searchlib/src/vespa/searchlib/engine/proto_converter.cpp b/searchlib/src/vespa/searchlib/engine/proto_converter.cpp index 1736cf1f72a..2495a6e12bd 100644 --- a/searchlib/src/vespa/searchlib/engine/proto_converter.cpp +++ b/searchlib/src/vespa/searchlib/engine/proto_converter.cpp @@ -5,7 +5,6 @@ #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/slime/binary_format.h> #include <vespa/vespalib/data/smart_buffer.h> -#include <vespa/searchlib/common/transport.h> namespace search::engine { @@ -119,9 +118,7 @@ ProtoConverter::docsum_request_from_proto(const ProtoDocsumRequest &proto, Docsu if (proto.cache_query()) { request.propertiesMap.lookupCreate(MapNames::CACHES).add("query", "true"); } - if (proto.dump_features()) { - request.queryFlags |= fs4transport::QFLAG_DUMP_FEATURES; - } + request.dumpFeatures = proto.dump_features(); request.ranking = proto.rank_profile(); if ((proto.feature_overrides_size() + proto.tensor_feature_overrides_size()) > 0) { auto &feature_overrides = request.propertiesMap.lookupCreate(MapNames::FEATURE); diff --git a/searchlib/src/vespa/searchlib/engine/request.cpp b/searchlib/src/vespa/searchlib/engine/request.cpp index 956653d5269..84615105579 100644 --- a/searchlib/src/vespa/searchlib/engine/request.cpp +++ b/searchlib/src/vespa/searchlib/engine/request.cpp @@ -1,14 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "request.h" -#include <vespa/searchlib/common/transport.h> namespace search::engine { Request::Request(RelativeTime relativeTime) : _relativeTime(std::move(relativeTime)), _timeOfDoom(fastos::TimeStamp(fastos::TimeStamp::FUTURE)), - queryFlags(0), + dumpFeatures(false), ranking(), location(), propertiesMap(), @@ -35,10 +34,4 @@ fastos::TimeStamp Request::getTimeLeft() const return _timeOfDoom - _relativeTime.now(); } -bool -Request::should_drop_sort_data() const -{ - return ((queryFlags & fs4transport::QFLAG_DROP_SORTDATA) != 0); -} - } diff --git a/searchlib/src/vespa/searchlib/engine/request.h b/searchlib/src/vespa/searchlib/engine/request.h index f5f24b6743f..a021ec6bfaa 100644 --- a/searchlib/src/vespa/searchlib/engine/request.h +++ b/searchlib/src/vespa/searchlib/engine/request.h @@ -28,8 +28,6 @@ public: return vespalib::stringref(&stackDump[0], stackDump.size()); } - bool should_drop_sort_data() const; - void setTraceLevel(uint32_t level, uint32_t minLevel) const { _trace.setLevel(level); _trace.start(minLevel); @@ -43,7 +41,7 @@ private: fastos::TimeStamp _timeOfDoom; public: /// Everything here should move up to private section and have accessors - uint32_t queryFlags; + bool dumpFeatures; vespalib::string ranking; vespalib::string location; PropertiesMap propertiesMap; diff --git a/searchlib/src/vespa/searchlib/engine/searchreply.cpp b/searchlib/src/vespa/searchlib/engine/searchreply.cpp index ef39e4da027..7fc0ac81289 100644 --- a/searchlib/src/vespa/searchlib/engine/searchreply.cpp +++ b/searchlib/src/vespa/searchlib/engine/searchreply.cpp @@ -16,8 +16,6 @@ SearchReply::SearchReply() coverage(), useWideHits(false), hits(), - errorCode(0), - errorMessage(), request() { } @@ -35,8 +33,6 @@ SearchReply::SearchReply(const SearchReply &rhs) : coverage (rhs.coverage), useWideHits (rhs.useWideHits), hits (rhs.hits), - errorCode (rhs.errorCode), - errorMessage (rhs.errorMessage), request() // NB not copied { } diff --git a/searchlib/src/vespa/searchlib/engine/searchreply.h b/searchlib/src/vespa/searchlib/engine/searchreply.h index d2645bbb2f6..d9468216a90 100644 --- a/searchlib/src/vespa/searchlib/engine/searchreply.h +++ b/searchlib/src/vespa/searchlib/engine/searchreply.h @@ -83,10 +83,6 @@ public: std::vector<Hit> hits; PropertiesMap propertiesMap; - // in case of error - uint32_t errorCode; - vespalib::string errorMessage; - SearchRequest::UP request; SearchReply(); diff --git a/searchlib/src/vespa/searchlib/engine/searchrequest.cpp b/searchlib/src/vespa/searchlib/engine/searchrequest.cpp index 53467e5c56c..9ca89a8636f 100644 --- a/searchlib/src/vespa/searchlib/engine/searchrequest.cpp +++ b/searchlib/src/vespa/searchlib/engine/searchrequest.cpp @@ -1,9 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "searchrequest.h" -#include "packetconverter.h" -#include "proto_converter.h" -#include <vespa/fnet/frt/rpcrequest.h> namespace search::engine { diff --git a/searchlib/src/vespa/searchlib/engine/transport_metrics.cpp b/searchlib/src/vespa/searchlib/engine/transport_metrics.cpp deleted file mode 100644 index 62d2b9d5489..00000000000 --- a/searchlib/src/vespa/searchlib/engine/transport_metrics.cpp +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "transport_metrics.h" - -namespace search::engine { - -TransportMetrics::QueryMetrics::QueryMetrics(metrics::MetricSet *parent) - : metrics::MetricSet("query", {}, "Query metrics", parent), - count("count", {{"logdefault"}}, "Query requests handled", this), - latency("latency", {{"logdefault"}}, "Query request latency", this) -{ -} - -TransportMetrics::QueryMetrics::~QueryMetrics() = default; - -TransportMetrics::DocsumMetrics::DocsumMetrics(metrics::MetricSet *parent) - : metrics::MetricSet("docsum", {}, "Docsum metrics", parent), - count("count", {{"logdefault"}}, "Docsum requests handled", this), - docs("docs", {{"logdefault"}}, "Total docsums returned", this), - latency("latency", {{"logdefault"}}, "Docsum request latency", this) -{ -} - -TransportMetrics::DocsumMetrics::~DocsumMetrics() = default; - -TransportMetrics::TransportMetrics() - : metrics::MetricSet("transport", {}, "Transport server metrics", nullptr), - updateLock(), - query(this), - docsum(this) -{ -} - -TransportMetrics::~TransportMetrics() = default; - -} - diff --git a/searchlib/src/vespa/searchlib/engine/transport_metrics.h b/searchlib/src/vespa/searchlib/engine/transport_metrics.h deleted file mode 100644 index 8b85438f688..00000000000 --- a/searchlib/src/vespa/searchlib/engine/transport_metrics.h +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/metrics/metrics.h> -#include <vespa/vespalib/util/sync.h> - -namespace search::engine { - -struct TransportMetrics : metrics::MetricSet -{ - struct QueryMetrics : metrics::MetricSet { - metrics::LongCountMetric count; - metrics::DoubleAverageMetric latency; - - QueryMetrics(metrics::MetricSet *parent); - ~QueryMetrics(); - }; - - struct DocsumMetrics : metrics::MetricSet { - metrics::LongCountMetric count; - metrics::LongCountMetric docs; - metrics::DoubleAverageMetric latency; - - DocsumMetrics(metrics::MetricSet *parent); - ~DocsumMetrics(); - }; - - vespalib::Lock updateLock; - QueryMetrics query; - DocsumMetrics docsum; - - TransportMetrics(); - ~TransportMetrics() override; -}; - -} - diff --git a/searchlib/src/vespa/searchlib/engine/transportserver.cpp b/searchlib/src/vespa/searchlib/engine/transportserver.cpp deleted file mode 100644 index d14735b7770..00000000000 --- a/searchlib/src/vespa/searchlib/engine/transportserver.cpp +++ /dev/null @@ -1,464 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "packetconverter.h" -#include "transportserver.h" -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/fnet/channel.h> -#include <vespa/fnet/connection.h> -#include <vespa/fnet/connector.h> -#include <vespa/fnet/iexecutable.h> -#include <vespa/vespalib/net/crypto_engine.h> - -#include <vespa/log/log.h> -LOG_SETUP(".engine.transportserver"); - -namespace search::engine { - -namespace { - -struct SearchRequestDecoder : SearchRequest::Source::Decoder { - PacketConverter::QUERYX *packet; - RelativeTime relative_time; - SearchRequestDecoder(PacketConverter::QUERYX *qx) - : packet(qx), relative_time(std::make_unique<FastosClock>()) {} - std::unique_ptr<SearchRequest> decode() override { - auto req = std::make_unique<SearchRequest>(std::move(relative_time)); - PacketConverter::toSearchRequest(*packet, *req); - return req; - } - ~SearchRequestDecoder() override { packet->Free(); } -}; - -std::unique_ptr<SearchRequest::Source::Decoder> search_request_decoder(PacketConverter::QUERYX *qx) { - return std::make_unique<SearchRequestDecoder>(qx); -} - -struct DocsumRequestDecoder : DocsumRequest::Source::Decoder { - PacketConverter::GETDOCSUMSX *packet; - RelativeTime relative_time; - DocsumRequestDecoder(PacketConverter::GETDOCSUMSX *gdx) - : packet(gdx), relative_time(std::make_unique<FastosClock>()) {} - std::unique_ptr<DocsumRequest> decode() override { - auto req = std::make_unique<DocsumRequest>(std::move(relative_time), false); - PacketConverter::toDocsumRequest(*packet, *req); - return req; - } - ~DocsumRequestDecoder() override { packet->Free(); } -}; - -std::unique_ptr<DocsumRequest::Source::Decoder> docsum_request_decoder(PacketConverter::GETDOCSUMSX *gdx) { - return std::make_unique<DocsumRequestDecoder>(gdx); -} - -} - -//----------------------------------------------------------------------------- - -typedef search::fs4transport::FS4PersistentPacketStreamer PacketStreamer; - -//----------------------------------------------------------------------------- - -constexpr uint32_t TransportServer::DEBUG_NONE; -constexpr uint32_t TransportServer::DEBUG_CONNECTION; -constexpr uint32_t TransportServer::DEBUG_CHANNEL; -constexpr uint32_t TransportServer::DEBUG_SEARCH; -constexpr uint32_t TransportServer::DEBUG_DOCSUM; -constexpr uint32_t TransportServer::DEBUG_MONITOR; -constexpr uint32_t TransportServer::DEBUG_UNHANDLED; -constexpr uint32_t TransportServer::DEBUG_ALL; - -void -TransportServer::SearchHandler::start() -{ - SearchReply::UP reply = parent._searchServer.search(std::move(request), *this); - if (reply) { - searchDone(std::move(reply)); - } -} - -void -TransportServer::SearchHandler::searchDone(SearchReply::UP reply) -{ - if (reply) { - const SearchReply &r = *reply; - if (r.valid) { - if (r.errorCode == 0) { - PacketConverter::QUERYRESULTX *p = new PacketConverter::QUERYRESULTX(); - PacketConverter::fromSearchReply(r, *p); - if (shouldLog(DEBUG_SEARCH)) { - logPacket("outgoing packet", p, channel, 0); - } - channel->Send(p); - } else { - PacketConverter::ERROR *p = new PacketConverter::ERROR(); - p->_errorCode = r.errorCode; - p->setErrorMessage(r.errorMessage); - if (shouldLog(DEBUG_SEARCH)) { - logPacket("outgoing packet", p, channel, 0); - } - channel->Send(p); - } - if (r.request) { - parent.updateQueryMetrics(r.request->getTimeUsed().sec()); // possible thread issue - } - } else { - PacketConverter::EOL *p = new PacketConverter::EOL(); - if (shouldLog(DEBUG_SEARCH)) { - logPacket("outgoing packet", p, channel, 0); - } - channel->Send(p); - } - } else { - LOG(warning, "got <null> search reply from back-end"); - } - delete this; // we are done -} - -TransportServer::SearchHandler::~SearchHandler() -{ - channel->Free(); -} - -//----------------------------------------------------------------------------- - -void -TransportServer::DocsumHandler::start() -{ - DocsumReply::UP reply = parent._docsumServer.getDocsums(std::move(request), *this); - if (reply) { - getDocsumsDone(std::move(reply)); - } -} - -void -TransportServer::DocsumHandler::getDocsumsDone(DocsumReply::UP reply) -{ - if (reply) { - const DocsumReply &r = *reply; - for (uint32_t i = 0; i < r.docsums.size(); ++i) { - PacketConverter::DOCSUM *p = new PacketConverter::DOCSUM(); - PacketConverter::fromDocsumReplyElement(r.docsums[i], *p); - if (shouldLog(DEBUG_DOCSUM)) { - logPacket("outgoing packet", p, channel, 0); - } - channel->Send(p); - } - PacketConverter::EOL *p = new PacketConverter::EOL(); - if (shouldLog(DEBUG_DOCSUM)) { - logPacket("outgoing packet", p, channel, 0); - } - channel->Send(p); - if (r.request) { - parent.updateDocsumMetrics(r.request->getTimeUsed().sec(), r.docsums.size()); - } - } else { - LOG(warning, "got <null> docsum reply from back-end"); - } - delete this; // we are done -} - -TransportServer::DocsumHandler::~DocsumHandler() -{ - channel->Free(); -} - -//----------------------------------------------------------------------------- - -void -TransportServer::MonitorHandler::start() -{ - MonitorReply::UP reply = parent._monitorServer.ping(std::move(request), *this); - if (reply) { - pingDone(std::move(reply)); - } -} - -void -TransportServer::MonitorHandler::pingDone(MonitorReply::UP reply) -{ - if (reply) { - const MonitorReply &r = *reply; - PacketConverter::MONITORRESULTX *p = new PacketConverter::MONITORRESULTX(); - PacketConverter::fromMonitorReply(r, *p); - if (shouldLog(DEBUG_MONITOR)) { - logPacket("outgoing packet", p, 0, connection); - } - connection->PostPacket(p, FNET_NOID); - } else { - LOG(warning, "got <null> monitor reply from back-end"); - } - delete this; // we are done -} - -TransportServer::MonitorHandler::~MonitorHandler() -{ - connection->SubRef(); -} - -//----------------------------------------------------------------------------- - -FNET_IPacketHandler::HP_RetCode -TransportServer::HandlePacket(FNET_Packet *packet, FNET_Context context) -{ - uint32_t pcode = packet->GetPCODE(); - FNET_Channel *channel = context._value.CHANNEL; - HP_RetCode rc = FNET_FREE_CHANNEL; - - if (channel->GetID() == FNET_NOID) { // admin packet - if (packet->IsChannelLostCMD()) { - _clients.erase(channel); - if (shouldLog(DEBUG_CONNECTION)) { - LOG(debug, "connection closed: tag=%u", channel->GetConnection()->GetContext()._value.INT); - } - } else if (pcode == search::fs4transport::PCODE_MONITORQUERYX) { - const PacketConverter::MONITORQUERYX &mqx = static_cast<PacketConverter::MONITORQUERYX&>(*packet); - if (shouldLog(DEBUG_MONITOR)) { - logPacket("incoming packet", packet, channel, 0); - } - MonitorRequest::UP req(new MonitorRequest()); - PacketConverter::toMonitorRequest(mqx, *req); - channel->GetConnection()->AddRef(); - _pending.push(new MonitorHandler(*this, std::move(req), channel->GetConnection())); - rc = FNET_KEEP_CHANNEL; - } else if (shouldLog(DEBUG_UNHANDLED)) { - logPacket("unhandled packet", packet, channel, 0); - } - } else { // search/docsum request - if (pcode == search::fs4transport::PCODE_QUERYX) { - PacketConverter::QUERYX * qx = static_cast<PacketConverter::QUERYX *>(packet); - if (shouldLog(DEBUG_SEARCH)) { - logPacket("incoming packet", packet, channel, 0); - } - SearchRequest::Source req(search_request_decoder(qx)); - packet = nullptr; - _pending.push(new SearchHandler(*this, std::move(req), channel, _clients.size())); - rc = FNET_CLOSE_CHANNEL; - } else if (pcode == search::fs4transport::PCODE_GETDOCSUMSX) { - PacketConverter::GETDOCSUMSX * gdx = static_cast<PacketConverter::GETDOCSUMSX *>(packet); - if (shouldLog(DEBUG_DOCSUM)) { - logPacket("incoming packet", packet, channel, 0); - } - DocsumRequest::Source req(docsum_request_decoder(gdx)); - packet = nullptr; - _pending.push(new DocsumHandler(*this, std::move(req), channel)); - rc = FNET_CLOSE_CHANNEL; - } else if (shouldLog(DEBUG_UNHANDLED)) { - logPacket("unhandled packet", packet, channel, 0); - } - } - if (packet != nullptr) { - packet->Free(); - } - return rc; -} - -bool -TransportServer::InitAdminChannel(FNET_Channel *channel) -{ - if (_listener == nullptr) { - // handle race where we get an incoming connection and - // disables listening at the 'same time'. Note that sync close - // is only allowed in the InitAdminChannel method - channel->GetConnection()->Close(); // sync close - return false; - } - channel->SetContext(channel); - channel->SetHandler(this); - assert(_clients.count(channel) == 0); - _clients.insert(channel); - channel->GetConnection()->SetContext(FNET_Context(++_connTag)); - if (shouldLog(DEBUG_CONNECTION)) { - LOG(debug, "connection established: tag=%u", _connTag); - } - return true; -} - -bool -TransportServer::InitChannel(FNET_Channel *channel, uint32_t pcode) -{ - channel->SetContext(channel); - channel->SetHandler(this); - if (shouldLog(DEBUG_CHANNEL)) { - LOG(debug, "new channel: id=%u, first pcode=%u", channel->GetID(), pcode); - } - return true; -} - -void -TransportServer::Run(FastOS_ThreadInterface *, void *) -{ - _dispatchTask.ScheduleNow(); - _ready = true; - _transport.Main(); // <- transport event loop - _dispatchTask.Kill(); - _listenTask.Kill(); - discardRequests(); -} - -bool -TransportServer::updateListen() -{ - bool doListen = _doListen; - if (doListen) { - if (_listener == nullptr) { // start listening - _listener = _transport.Listen(_listenSpec.c_str(), &PacketStreamer::Instance, this); - if (_listener == nullptr) { - LOG(error, "Could not bind fnet transport socket to %s", _listenSpec.c_str()); - _failed = true; - return false; - } - } - } else { - if (_listener != nullptr) { // stop listening - _transport.Close(_listener); // async close - _listener->SubRef(); - _listener = nullptr; - // also close client connections - std::set<FNET_Channel*>::iterator it = _clients.begin(); - for (; it != _clients.end(); ++it) { - _transport.Close((*it)->GetConnection()); // async close - } - } - } - return true; -} - -void -TransportServer::dispatchRequests() -{ - while (!_pending.empty()) { - Handler *h = _pending.front(); - _pending.pop(); - h->start(); - } -} - -void -TransportServer::discardRequests() -{ - while (!_pending.empty()) { - Handler *h = _pending.front(); - _pending.pop(); - delete h; - } -} - -void -TransportServer::logPacket(vespalib::stringref msg, FNET_Packet *p, FNET_Channel *ch, FNET_Connection *conn) -{ - uint32_t chid = -1; - uint32_t conntag = -1; - vespalib::string str; - if (ch != 0) { - chid = ch->GetID(); - conntag = ch->GetConnection()->GetContext()._value.INT; - } else if (conn != 0) { - conntag = conn->GetContext()._value.INT; - } - search::fs4transport::FS4Packet *fs4p = dynamic_cast<search::fs4transport::FS4Packet*>(p); - if (fs4p != 0) { - str = fs4p->toString(0); - } else { - str = vespalib::make_string("packet { pcode=%u }", p->GetPCODE()); - } - LOG(debug, "%s (chid=%u, conn=%u):\n%s", msg.data(), chid, conntag, str.c_str()); -} - -void -TransportServer::updateQueryMetrics(double latency_s) -{ - vespalib::LockGuard guard(_metrics.updateLock); - _metrics.query.count.inc(); - _metrics.query.latency.set(latency_s); -} - -void -TransportServer::updateDocsumMetrics(double latency_s, uint32_t numDocs) -{ - vespalib::LockGuard guard(_metrics.updateLock); - _metrics.docsum.count.inc(); - _metrics.docsum.docs.inc(numDocs); - _metrics.docsum.latency.set(latency_s); -} - -//----------------------------------------------------------------------------- - -bool -TransportServer::shouldLog(uint32_t msgType) { - return (((msgType & _debugMask) != 0) - && ((msgType != DEBUG_MONITOR && LOG_WOULD_LOG(debug)) || - (msgType == DEBUG_MONITOR && LOG_WOULD_LOG(spam)))); -} - -TransportServer::TransportServer(SearchServer &searchServer, - DocsumServer &docsumServer, - MonitorServer &monitorServer, - int port, uint32_t debugMask) - : _searchServer(searchServer), - _docsumServer(docsumServer), - _monitorServer(monitorServer), - _transport(std::make_shared<vespalib::NullCryptoEngine>(), 1), // disable encryption - _ready(false), - _failed(false), - _doListen(true), - _threadPool(256 * 1024), - _listenSpec(), - _listener(0), - _clients(), - _pending(), - _dispatchTask(*this), - _listenTask(*this), - _connTag(0), - _debugMask(debugMask), - _metrics() -{ - _listenSpec = vespalib::make_string("tcp/%d", port); -} - -bool -TransportServer::start() -{ - if (!updateListen()) { - return false; - } - if (_threadPool.NewThread(this) == 0) { - LOG(error, "Could not start internal transport thread"); - _failed = true; - return false; - } - return true; -} - -int -TransportServer::getListenPort() -{ - struct Cmd : public FNET_IExecutable { - TransportServer &server; - vespalib::Gate done; - int port; - Cmd(TransportServer &s) : server(s), done(), port(-1) {} - void execute() override { - if (server._listener != 0) { - port = server._listener->GetPortNumber(); - } - done.countDown(); - } - }; - Cmd cmd(*this); - if (_transport.execute(&cmd)) { - cmd.done.await(); - } - return cmd.port; -}; - -TransportServer::~TransportServer() -{ - shutDown(); // ensure shutdown - if (_listener != 0) { - _listener->SubRef(); - _listener = 0; - } -} - -} - diff --git a/searchlib/src/vespa/searchlib/engine/transportserver.h b/searchlib/src/vespa/searchlib/engine/transportserver.h deleted file mode 100644 index 60bcc79a7fe..00000000000 --- a/searchlib/src/vespa/searchlib/engine/transportserver.h +++ /dev/null @@ -1,325 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "transport_metrics.h" -#include "searchapi.h" -#include "docsumapi.h" -#include "monitorapi.h" -#include <vespa/searchlib/common/packets.h> -#include <vespa/fnet/iserveradapter.h> -#include <vespa/fnet/ipackethandler.h> -#include <vespa/fnet/task.h> -#include <vespa/fnet/transport.h> -#include <vespa/vespalib/util/sync.h> -#include <vespa/fastos/thread.h> -#include <set> -#include <queue> - -namespace search::engine { - -/** - * Common transport server implementation interacting with the - * underlying search engine using the common search api. This - * implementation has less optimization tricks compared to the - * previous ones being integrated into specific applications. - **/ -class TransportServer : public FastOS_Runnable, - public FNET_IServerAdapter, - public FNET_IPacketHandler -{ -private: - TransportServer(const TransportServer &); - TransportServer &operator=(const TransportServer &); - - /** - * Task used to update listen status - **/ - struct ListenTask : public FNET_Task - { - TransportServer &parent; - ListenTask(TransportServer &p) : FNET_Task(p._transport.GetScheduler()), parent(p) {} - void PerformTask() override { parent.updateListen(); } - }; - - /** - * Task used to dispatch incoming requests in an untangled way - * (aka not in the packet callback). - **/ - struct DispatchTask : public FNET_Task - { - TransportServer &parent; - DispatchTask(TransportServer &p) : FNET_Task(p._transport.GetScheduler()), parent(p) {} - void PerformTask() override { - parent.dispatchRequests(); - ScheduleNow(); // run each tick - } - }; - - struct Handler; - - SearchServer &_searchServer; - DocsumServer &_docsumServer; - MonitorServer &_monitorServer; - FNET_Transport _transport; - bool _ready; // flag indicating initial readyness - bool _failed; // flag indicating a critical failure - bool _doListen; // flag telling us to accept requests or not - FastOS_ThreadPool _threadPool; // thread pool owning transport thread - vespalib::string _listenSpec; // where to listen; FNET connect spec - FNET_Connector *_listener; // object accepting incoming connections - std::set<FNET_Channel*> _clients; // the admin channel of all client connections - std::queue<Handler*> _pending; // queue of incoming requests not yet started - DispatchTask _dispatchTask; // task used to dispatch incoming requests - ListenTask _listenTask; // task used to update listen status - uint32_t _connTag; // sequential number used to tag connections - uint32_t _debugMask; // enable more debug logging with this - TransportMetrics _metrics; // metrics for this transport server - - /** - * Toplevel class used to wrap incoming requests. Actual objects - * are used both to delay starting the request until we are not in - * the packet delivery callback and also as the callback target - * used by the underlying api objects to notify completion of - * individual requests. - **/ - struct Handler - { - TransportServer &parent; - uint32_t _debugMask; - Handler(TransportServer &p) : parent(p), _debugMask(p._debugMask) {} - bool shouldLog(uint32_t msgType) { return parent.shouldLog(msgType); } // possible thread issue - virtual void start() = 0; - virtual ~Handler() {} - private: - Handler(const Handler &rhs); - Handler &operator=(const Handler &rhs); - }; - - /** - * Wrapper for search requests - **/ - struct SearchHandler : public Handler, - public SearchClient - { - SearchRequest::Source request; - FNET_Channel *channel; - uint32_t clientCnt; - - SearchHandler(TransportServer &p, SearchRequest::Source req, FNET_Channel *ch, uint32_t cnt) - : Handler(p), request(std::move(req)), channel(ch), clientCnt(cnt) {} - void start() override; - void searchDone(SearchReply::UP reply) override; - ~SearchHandler(); - }; - - /** - * Wrapper for docsum requests - **/ - struct DocsumHandler : public Handler, - public DocsumClient - { - DocsumRequest::Source request; - FNET_Channel *channel; - - DocsumHandler(TransportServer &p, DocsumRequest::Source req, FNET_Channel *ch) - : Handler(p), request(std::move(req)), channel(ch) {} - void start() override; - void getDocsumsDone(DocsumReply::UP reply) override; - ~DocsumHandler(); - }; - - /** - * Wrapper for monitor requests - **/ - struct MonitorHandler : public Handler, - public MonitorClient - { - MonitorRequest::UP request; - FNET_Connection *connection; - - MonitorHandler(TransportServer &p, MonitorRequest::UP req, FNET_Connection *conn) - : Handler(p), request(std::move(req)), connection(conn) {} - void start() override; - void pingDone(MonitorReply::UP reply) override; - ~MonitorHandler(); - }; - - // handle incoming network packets - HP_RetCode HandlePacket(FNET_Packet *packet, FNET_Context context) override; - - // set up admin channel for new clients - bool InitAdminChannel(FNET_Channel *channel) override; - - // set up channel for individual request - bool InitChannel(FNET_Channel *channel, uint32_t pcode) override; - - // entry point for thread running transport thread - void Run(FastOS_ThreadInterface *thisThread, void *arg) override; - - // update listen status - bool updateListen(); - - // dispatch incoming requests - void dispatchRequests(); - - // discard any pending requests during shutdown - void discardRequests(); - - // convenience method used to log packets - static void logPacket(vespalib::stringref msg, FNET_Packet *p, FNET_Channel *ch, FNET_Connection *conn); - - void updateQueryMetrics(double latency_s); - void updateDocsumMetrics(double latency_s, uint32_t numDocs); - -public: - /** - * Convenience typedes. - */ - typedef std::unique_ptr<TransportServer> UP; - typedef std::shared_ptr<TransportServer> SP; - - /** no debug logging flags set **/ - static constexpr uint32_t DEBUG_NONE = 0x00000000; - - /** log connect disconnect from clients **/ - static constexpr uint32_t DEBUG_CONNECTION = 0x00000001; - - /** log channel open events **/ - static constexpr uint32_t DEBUG_CHANNEL = 0x00000002; - - /** log search related packets **/ - static constexpr uint32_t DEBUG_SEARCH = 0x00000004; - - /** log docsum related packets **/ - static constexpr uint32_t DEBUG_DOCSUM = 0x00000008; - - /** log monitor related packets **/ - static constexpr uint32_t DEBUG_MONITOR = 0x00000010; - - /** log unhandled packets **/ - static constexpr uint32_t DEBUG_UNHANDLED = 0x00000020; - - /** all debug logging flags set **/ - static constexpr uint32_t DEBUG_ALL = 0x0000003f; - - /** - * Check if we should log a debug message - * - * @return true if we should log a message for this event - * @param msgType the event we might want to log - **/ - bool shouldLog(uint32_t msgType); - - /** - * Create a transport server based on the given underlying api - * objects. An appropriate debug mask can be made by or'ing - * together the appropriate DEBUG_ constants defined in this - * class. - * - * @param searchServer search api - * @param docsumServer docsum api - * @param monitorServer monitor api - * @param port listen port. - * @param debugMask mask indicating what information should be logged as debug messages. - **/ - TransportServer(SearchServer &searchServer, - DocsumServer &docsumServer, - MonitorServer &monitorServer, - int port, uint32_t debugMask = DEBUG_NONE); - - /** - * Obtain the metrics used by this transport server. - * - * @return internal metrics - **/ - TransportMetrics &getMetrics() { return _metrics; } - - /** - * Obtain the listen spec used by this transport server - * - * @return listen spec - **/ - const vespalib::string &getListenSpec() const { return _listenSpec; } - - /** - * Start this server. - * - * @return success(true)/failure(false) - **/ - bool start(); - - /** - * Check for initial readyness. - * - * @return true if we are ready. - **/ - bool isReady() const { return _ready; } - - /** - * Check if a critical error has occurred. - * - * @return true if something bad has happened. - **/ - bool isFailed() const { return _failed; } - - /** - * Get a reference to the internal fnet scheduler. - * - * @return fnet scheduler - **/ - FNET_Scheduler &getScheduler() { return *(_transport.GetScheduler()); } - - /** - * Set a flag indicating whether we should accept incoming - * requests or not. Setting the flag to false will make this - * server unavailable to any client application. - * - * @param listen flag indicating if we should listen - **/ - void setListen(bool listen) { - _doListen = listen; - _listenTask.ScheduleNow(); - } - - /** - * Check which port this server is currently listening to. This - * method is useful when using automatically allocated port - * numbers (listening to port 0). - * - * @return current listening port number, -1 if not listening. - **/ - int getListenPort(); - - /** - * Enable or disable nagles algorithm. - * - * @param noDelay set to true to disable nagles algorithm - **/ - void setTCPNoDelay(bool noDelay) { _transport.SetTCPNoDelay(noDelay); } - - /** - * Set a limit on how long a connection may be idle before closing it. - * - * @param millisecs max idle time in milliseconds - **/ - void setIdleTimeout(double millisecs) { _transport.SetIOCTimeOut((uint32_t) millisecs); } - - /** - * Shut down this component. This method will block until the - * transport server has been shut down. After this method returns, - * no new requests will be generated by this component. - **/ - void shutDown() { - _transport.ShutDown(false); - _threadPool.Close(); - } - - /** - * Destructor will perform shutdown if needed. - **/ - ~TransportServer(); -}; - -} - diff --git a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp index efeb066135f..c0d9ec2de3a 100644 --- a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp +++ b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp @@ -4,7 +4,6 @@ #include <vespa/searchsummary/docsummary/docsumwriter.h> #include <vespa/searchsummary/docsummary/resultpacker.h> #include <vespa/searchsummary/docsummary/docsumstate.h> -#include <vespa/searchlib/common/transport.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> @@ -43,7 +42,7 @@ struct DocsumFixture : IDocsumStore, GetDocsumsStateCallback { ASSERT_GREATER(buf.GetUsedLen(), sizeof(classId)); memcpy(&classId, buf.GetDrainPos(), sizeof(classId)); buf.Drain(sizeof(classId)); - EXPECT_EQUAL(classId, ::search::fs4transport::SLIME_MAGIC_ID); + EXPECT_EQUAL(classId, SLIME_MAGIC_ID); EXPECT_GREATER(vespalib::slime::BinaryFormat ::decode(Memory(buf.GetDrainPos(), buf.GetUsedLen()), slime), 0u); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp index 4e3540fb573..0e647b43e78 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp @@ -4,7 +4,6 @@ #include "docsumstate.h" #include "docsum_field_writer_state.h" #include <vespa/searchcommon/common/undefinedvalues.h> -#include <vespa/searchlib/common/transport.h> #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchlib/attribute/iattributemanager.h> #include <vespa/vespalib/data/slime/slime.h> @@ -20,7 +19,7 @@ uint32_t IDocsumWriter::slime2RawBuf(const Slime & slime, RawBuf & buf) { const uint32_t preUsed = buf.GetUsedLen(); - const uint32_t magic = ::search::fs4transport::SLIME_MAGIC_ID; + const uint32_t magic = SLIME_MAGIC_ID; buf.append(&magic, sizeof(magic)); SlimeOutputRawBufAdapter adapter(buf); vespalib::slime::BinaryFormat::encode(slime, adapter); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h index 92b26d5cf14..e5dd2793089 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h @@ -16,6 +16,8 @@ using search::IAttributeManager; namespace search::docsummary { +static constexpr uint32_t SLIME_MAGIC_ID = 0x55555555; + class IDocsumWriter { public: diff --git a/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.cpp b/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.cpp index 5856797831a..8f8166a2806 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.cpp @@ -1,25 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "getdocsumargs.h" -#include "resultconfig.h" namespace search::docsummary { GetDocsumArgs::GetDocsumArgs() : _ranking(), - _qflags(0), _resultClassName(), + _dumpFeatures(false), _stackItems(0), _stackDump(), _location(), _timeout(30 * fastos::TimeStamp::SEC), - _flags(0u), - _propertiesMap(), - _isLocationSet(false) + _propertiesMap() { } -GetDocsumArgs::~GetDocsumArgs() { } +GetDocsumArgs::~GetDocsumArgs() = default; void GetDocsumArgs::setTimeout(const fastos::TimeStamp & timeout) @@ -33,48 +30,17 @@ GetDocsumArgs::getTimeout() const return _timeout; } - -void -GetDocsumArgs::Reset() -{ - _ranking.clear(); - _qflags = 0; - _stackItems = 0; - _timeout = 30 * fastos::TimeStamp::SEC; - _flags = 0; - _resultClassName.clear(); - _stackDump.clear(); - _location.clear(); - _isLocationSet = false; - { - PropsMap tmp; - std::swap(_propertiesMap, tmp); - } -} - - -void -GetDocsumArgs::Copy(GetDocsumArgs *src) -{ - if (src == this) { - return; - } - *src = *this; -} - void GetDocsumArgs::initFromDocsumRequest(const search::engine::DocsumRequest &req) { _ranking = req.ranking; - _qflags = req.queryFlags; + _dumpFeatures = req.dumpFeatures; _resultClassName = req.resultClassName; _stackItems = req.stackItems; _stackDump = req.stackDump; _location = req.location; _timeout = req.getTimeLeft(); - _flags = req._flags; _propertiesMap = req.propertiesMap; - _isLocationSet = (_location.size() > 0); } void diff --git a/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h b/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h index a5536a913e5..ce5dc695f08 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/getdocsumargs.h @@ -15,68 +15,40 @@ public: private: vespalib::string _ranking; - uint32_t _qflags; vespalib::string _resultClassName; + bool _dumpFeatures; uint32_t _stackItems; std::vector<char> _stackDump; vespalib::string _location; fastos::TimeStamp _timeout; - uint32_t _flags; PropsMap _propertiesMap; - - bool _isLocationSet; - public: GetDocsumArgs(); ~GetDocsumArgs(); - void Reset(); - void Copy(GetDocsumArgs *src); void initFromDocsumRequest(const search::engine::DocsumRequest &req); void SetRankProfile(const vespalib::string &ranking) { _ranking = ranking; } - void SetQueryFlags(uint32_t qflags) { _qflags = qflags; } - void SetResultClassName(uint32_t len, const char *name) { - _resultClassName.assign(name, len); - } void setResultClassName(vespalib::stringref name) { _resultClassName = name; } - void SetStackDump(uint32_t stackItems, - uint32_t stackDumpLen, const char *stackDump); - void SetLocation(uint32_t locationLen, const char *location) { - if ((_isLocationSet = (location != NULL))) { - _location.assign(location, locationLen); - } - } - - void - setFlags(uint32_t flags) - { - _flags = flags; + void SetStackDump(uint32_t stackItems, uint32_t stackDumpLen, const char *stackDump); + void setLocation(vespalib::stringref location) { + _location = location; } void setTimeout(const fastos::TimeStamp & timeout); fastos::TimeStamp getTimeout() const; - const vespalib::string & getRankProfile() const { return _ranking; } const vespalib::string & getResultClassName() const { return _resultClassName; } const vespalib::string & getLocation() const { return _location; } const vespalib::stringref getStackDump() const { return vespalib::stringref(&_stackDump[0], _stackDump.size()); } - uint32_t GetQueryFlags() const { return _qflags; } - uint32_t GetStackItems() const { return _stackItems; } - uint32_t GetLocationLen() const { return _location.size(); } - uint32_t getFlags() const { return _flags; } + void dumpFeatures(bool v) { _dumpFeatures = v; } + bool dumpFeatures() const { return _dumpFeatures; } const PropsMap &propertiesMap() const { return _propertiesMap; } - const search::fef::Properties &rankProperties() const { - return _propertiesMap.rankProperties(); - } - const search::fef::Properties &featureOverrides() const { - return _propertiesMap.featureOverrides(); - } const search::fef::Properties &highlightTerms() const { return _propertiesMap.highlightTerms(); } diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index 6b7dcc8c337..d6c681e8f49 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -11,10 +11,10 @@ #include <vespa/searchlib/common/packets.h> #include <vespa/searchlib/uca/ucaconverter.h> #include <vespa/searchlib/features/setup.h> -#include <vespa/searchlib/fef/fef.h> #include <vespa/vespalib/geo/zcurve.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/fnet/databuffer.h> #include <vespa/log/log.h> LOG_SETUP(".visitor.instance.searchvisitor"); @@ -50,6 +50,11 @@ ForceWordfolderInit::ForceWordfolderInit() static ForceWordfolderInit _G_forceNormWordFolderInit; +// Leftovers from FS4 protocol with limited use here. +enum queryflags { + QFLAG_DUMP_FEATURES = 0x00040000 +}; + AttributeVector::SP createMultiValueAttribute(const vespalib::string & name, const document::FieldValue & fv, bool arrayType) @@ -222,11 +227,11 @@ void SearchVisitor::init(const Parameters & params) LOG(debug, "Received rank profile: %s", _rankController.getRankProfile().c_str()); } - if (params.lookup("queryflags", valueRef) ) { - vespalib::string tmp(valueRef.data(), valueRef.size()); - LOG(debug, "Received query flags: 0x%lx", strtoul(tmp.c_str(), nullptr, 0)); - uint32_t queryFlags = strtoul(tmp.c_str(), nullptr, 0); - _rankController.setDumpFeatures((queryFlags & search::fs4transport::QFLAG_DUMP_FEATURES) != 0); + int queryFlags = params.get("queryflags", 0); + if (queryFlags) { + bool dumpFeatures = (queryFlags & QFLAG_DUMP_FEATURES) != 0; + _summaryGenerator.getDocsumState()._args.dumpFeatures(dumpFeatures); + _rankController.setDumpFeatures(dumpFeatures); LOG(debug, "QFLAG_DUMP_FEATURES: %s", _rankController.getDumpFeatures() ? "true" : "false"); } @@ -262,16 +267,11 @@ void SearchVisitor::init(const Parameters & params) _summaryGenerator.getDocsumState()._args.SetRankProfile(tmp); } - int queryFlags = 0; - if (params.get("queryflags", queryFlags)) { - _summaryGenerator.getDocsumState()._args.SetQueryFlags(queryFlags); - } - vespalib::string location; if (params.lookup("location", valueRef)) { location = vespalib::string(valueRef.data(), valueRef.size()); LOG(debug, "Location = '%s'", location.c_str()); - _summaryGenerator.getDocsumState()._args.SetLocation(valueRef.size(), (const char*)valueRef.data()); + _summaryGenerator.getDocsumState()._args.setLocation(valueRef); } Parameters::ValueRef searchClusterBlob; diff --git a/vdslib/src/vespa/vdslib/container/parameters.cpp b/vdslib/src/vespa/vdslib/container/parameters.cpp index c830f89d5a9..4358c42ff29 100644 --- a/vdslib/src/vespa/vdslib/container/parameters.cpp +++ b/vdslib/src/vespa/vdslib/container/parameters.cpp @@ -9,7 +9,7 @@ using namespace vdslib; -Parameters::Parameters() : _parameters() { } +Parameters::Parameters() = default; Parameters::Parameters(const document::DocumentTypeRepo &repo, document::ByteBuffer& buffer) : _parameters() @@ -17,9 +17,7 @@ Parameters::Parameters(const document::DocumentTypeRepo &repo, document::ByteBuf deserialize(repo, buffer); } -Parameters::~Parameters() -{ -} +Parameters::~Parameters() = default; size_t Parameters::getSerializedSize() const { @@ -74,23 +72,6 @@ Parameters::printXml(document::XmlOutputStream& xos) const xos << XmlEndTag(); } -/* -void -Parameters::deserializeXml(const document::XmlElement & xml) -{ - ParametersMap params; - for(document::XmlElement::ElementList::const_iterator it = xml.elements().begin(), mt = xml.elements().end(); it != mt; it++) { - const document::XmlElement & elem = *it; - assert( elem.elements().size() == 2); - - const document::XmlElement & name = (*it).elements()[0]; - const document::XmlElement & value = (*it).elements()[1]; - params[name.value()] = Value(value.value().c_str(), value.value().size()); - } - _parameters.swap(params); -} -*/ - bool Parameters::operator==(const Parameters &other) const { diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java index d3be7829927..bea9af458b4 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java @@ -1,5 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.identityprovider.client; import com.google.common.cache.CacheBuilder; @@ -67,7 +66,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen private final static Duration ROLE_TOKEN_EXPIRY = Duration.ofMinutes(30); // TODO Make path to trust store config - private static final Path DEFAULT_TRUST_STORE = Paths.get(Defaults.getDefaults().underVespaHome("share/ssl/certs/yahoo_certificate_bundle.jks")); + private static final Path DEFAULT_TRUST_STORE = Paths.get("/opt/yahoo/share/ssl/certs/yahoo_certificate_bundle.jks"); public static final String CERTIFICATE_EXPIRY_METRIC_NAME = "athenz-tenant-cert.expiry.seconds"; @@ -78,7 +77,6 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen private final ScheduledExecutorService scheduler; private final Clock clock; private final AthenzService identity; - private final String dnsSuffix; private final URI ztsEndpoint; private final MutableX509KeyManager identityKeyManager = new MutableX509KeyManager(); @@ -102,7 +100,6 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen } // Test only - AthenzIdentityProviderImpl(IdentityConfig config, Metric metric, Path trustStore, @@ -115,7 +112,6 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen this.scheduler = scheduler; this.clock = clock; this.identity = new AthenzService(config.domain(), config.service()); - this.dnsSuffix = config.athenzDnsSuffix(); this.ztsEndpoint = URI.create(config.ztsUrl()); roleSslContextCache = createCache(ROLE_SSL_CONTEXT_EXPIRY, this::createRoleSslContext); roleSpecificRoleTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createRoleToken); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java index 40f12b9c6db..4d7f4b1c397 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java @@ -52,6 +52,13 @@ public class SiaUtils { .resolve(String.format("%s.%s.cert.pem", service.getDomainName(), service.getName())); } + public static Path getCaCertificatesFile() { + // The contents of this is the same as /opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem installed + // by the yahoo_certificates_bundle RPM package, except the latter also contains a textual description + // (decoded) of the certificates. + return DEFAULT_SIA_DIRECTORY.resolve("certs").resolve("ca.cert.pem"); + } + public static Optional<PrivateKey> readPrivateKeyFile(AthenzIdentity service) { return readPrivateKeyFile(DEFAULT_SIA_DIRECTORY, service); } diff --git a/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java b/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java index b1fd8e8cffa..a32aa75d994 100755 --- a/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java +++ b/vespaclient-java/src/main/java/com/yahoo/dummyreceiver/DummyReceiver.java @@ -2,7 +2,6 @@ package com.yahoo.dummyreceiver; import com.yahoo.concurrent.DaemonThreadFactory; -import com.yahoo.documentapi.ThroughputLimitQueue; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; @@ -42,7 +41,6 @@ public class DummyReceiver implements MessageHandler { private boolean instant = false; private ThreadPoolExecutor executor = null; private int threads = 10; - private long maxQueueTime = -1; private BlockingQueue<Runnable> queue; private boolean verbose = false; private boolean helpOption = false; @@ -76,7 +74,7 @@ public class DummyReceiver implements MessageHandler { params.getMessageBusParams().setMaxPendingCount(0); params.getMessageBusParams().setMaxPendingSize(0); da = new MessageBusDocumentAccess(params); - queue = (maxQueueTime < 0) ? new LinkedBlockingDeque<>() : new ThroughputLimitQueue<>(maxQueueTime); + queue = new LinkedBlockingDeque<>(); session = da.getMessageBus().createDestinationSession("default", true, this); executor = new ThreadPoolExecutor(threads, threads, 5, TimeUnit.SECONDS, queue, new DaemonThreadFactory()); System.out.println("Registered listener at " + name + "/default with 0 max pending and sleep time of " + sleepTime); @@ -156,8 +154,6 @@ public class DummyReceiver implements MessageHandler { instant = true; } else if ("--silent".equals(arg)) { silentNum = Long.parseLong(getParam(args, arg)); - } else if ("--maxqueuetime".equals(arg)) { - maxQueueTime = Long.parseLong(getParam(args, arg)); } else if ("--threads".equals(arg)) { threads = Integer.parseInt(getParam(args, arg)); } else if ("--verbose".equals(arg)) { diff --git a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java index 1da013de012..f5ef88016ac 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java @@ -16,7 +16,12 @@ import java.util.Set; import java.util.function.DoubleBinaryOperator; /** - * An indexed (dense) tensor backed by an array. + * An indexed (dense) tensor. + * <p> + * Some methods on indexed tensors make use of a <b>standard value order</b>: Cells are ordered by increasing + * index where dimensions to the right are incremented before indexes to the left, where the order of dimensions are + * alphabetical by name. In consequence, tensor value ordering is independent of the order in which dimensions are + * specified, and the values of the right-most dimension are adjacent. * * @author bratseth */ @@ -34,9 +39,7 @@ public abstract class IndexedTensor implements Tensor { } /** - * Returns an iterator over the cells of this. - * Cells are returned in order of increasing indexes in each dimension, increasing - * indexes of later dimensions in the dimension type before earlier. + * Returns an iterator over the cells of this in the <i>standard value order</i>. */ @Override public Iterator<Cell> cellIterator() { @@ -58,11 +61,7 @@ public abstract class IndexedTensor implements Tensor { return new SubspaceIterator(iterateDimensions, startAddress, iterationSizes); } - /** - * Returns an iterator over the values of this. - * Values are returned in order of increasing indexes in each dimension, increasing - * indexes of later dimensions in the dimension type before earlier. - */ + /** Returns an iterator over the values of this returned in the <i>standard value order</i> */ @Override public Iterator<Double> valueIterator() { return new ValueIterator(); @@ -70,8 +69,8 @@ public abstract class IndexedTensor implements Tensor { /** * Returns an iterator over value iterators where the outer iterator is over each unique value of the dimensions - * given and the inner iterator is over each unique value of the rest of the dimensions, in the same order as - * other iterator. + * given and the inner iterator is over each unique value of the rest of the dimensions, in the + * <i>standard value order</i> * * @param dimensions the names of the dimensions of the superspace * @param sizes the size of each dimension in the space we are returning values for, containing @@ -120,8 +119,7 @@ public abstract class IndexedTensor implements Tensor { } /** - * Returns the value at the given index as a double by direct lookup. Only use - * if you know the underlying data layout. + * Returns the value at the given <i>standard value order</i> index as a double. * * @param valueIndex the direct index into the underlying data. * @throws IllegalArgumentException if index is out of bounds @@ -129,8 +127,7 @@ public abstract class IndexedTensor implements Tensor { public abstract double get(long valueIndex); /** - * Returns the value at the given index as a float by direct lookup. Only use - * if you know the underlying data layout. + * Returns the value at the given <i>standard value order</i> index as a float. * * @param valueIndex the direct index into the underlying data. * @throws IllegalArgumentException if index is out of bounds @@ -310,7 +307,8 @@ public abstract class IndexedTensor implements Tensor { * Creates a builder initialized with the given values * * @param type the type of the tensor to build - * @param values the initial values of the tensor. This <b>transfers ownership</b> of the value array - it + * @param values the initial values of the tensor in the <i>standard value order</i>. + * This <b>transfers ownership</b> of the value array - it * must not be further mutated by the caller */ public static Builder of(TensorType type, DimensionSizes sizes, float[] values) { @@ -329,7 +327,8 @@ public abstract class IndexedTensor implements Tensor { * Creates a builder initialized with the given values * * @param type the type of the tensor to build - * @param values the initial values of the tensor. This <b>transfers ownership</b> of the value array - it + * @param values the initial values of the tensor in the <i>standard value order</i>. + * This <b>transfers ownership</b> of the value array - it * must not be further mutated by the caller */ public static Builder of(TensorType type, DimensionSizes sizes, double[] values) { @@ -411,10 +410,10 @@ public abstract class IndexedTensor implements Tensor { DimensionSizes sizes() { return sizes; } - /** Sets a value by its right-adjacent traversal position */ + /** Sets a value by its <i>standard value order</i> index */ public abstract void cellByDirectIndex(long index, double value); - /** Sets a value by its right-adjacent traversal position */ + /** Sets a value by its <i>standard value order</i> index */ public abstract void cellByDirectIndex(long index, float value); } @@ -672,7 +671,7 @@ public abstract class IndexedTensor implements Tensor { * @param iterateDimensions the dimensions to iterate over, given as indexes in the dimension order of the * type of the tensor this iterates over. This iterator will iterate over these * dimensions to exhaustion in the order given (the first dimension index given is - * incremented to exhaustion first (i.e is etc.), while other dimensions will be held + * incremented to exhaustion first etc., while other dimensions will be held * at a constant position. * This may be any subset of the dimensions given by address and dimensionSizes. * This is treated as immutable. diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index 8b0aaa64551..afd82751137 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -295,21 +295,16 @@ public interface Tensor { // ----------------- serialization /** - * Returns this tensor on the form - * <code>{address1:value1,address2:value2,...}</code> - * where each address is on the form <code>{dimension1:label1,dimension2:label2,...}</code>, - * and values are numbers. - * <p> - * Cells are listed in the natural order of tensor addresses: Increasing size primarily - * and by element lexical order secondarily. - * <p> - * Note that while this is suggestive of JSON, it is not JSON. + * Returns this tensor on the + * <a href="https://docs.vespa.ai/documentation/reference/tensor.html#tensor-literal-form">tensor literal form</a> + * with type included. */ @Override String toString(); /** - * Call this from toString in implementations to return the standard string format. + * Call this from toString in implementations to return this tensor on the + * <a href="https://docs.vespa.ai/documentation/reference/tensor.html#tensor-literal-form">tensor literal form</a>. * (toString cannot be a default method because default methods cannot override super methods). * * @param tensor the tensor to return the standard string format of @@ -345,6 +340,7 @@ public interface Tensor { * Returns whether this tensor and the given tensor is mathematically equal: * That they have the same dimension *names* and the same content. */ + @Override boolean equals(Object o); /** @@ -381,7 +377,8 @@ public interface Tensor { // ----------------- Factories /** - * Returns a tensor instance containing the given data on the standard string format returned by toString + * Returns a tensor instance containing the given data on the + * <a href="https://docs.vespa.ai/documentation/reference/tensor.html#tensor-literal-form">tensor literal form</a>. * * @param type the type of the tensor to return * @param tensorString the tensor on the standard tensor string format @@ -391,7 +388,8 @@ public interface Tensor { } /** - * Returns a tensor instance containing the given data on the standard string format returned by toString + * Returns a tensor instance containing the given data on the + * <a href="https://docs.vespa.ai/documentation/reference/tensor.html#tensor-literal-form">tensor literal form</a>. * * @param tensorType the type of the tensor to return, as a string on the tensor type format, given in * {@link TensorType#fromSpec} @@ -402,8 +400,8 @@ public interface Tensor { } /** - * Returns a tensor instance containing the given data on the standard string format returned by toString. - * If a type is not specified it is derived from the first cell of the tensor + * Returns a tensor instance containing the given data on the + * <a href="https://docs.vespa.ai/documentation/reference/tensor.html#tensor-literal-form">tensor literal form</a>. */ static Tensor from(String tensorString) { return TensorParser.tensorFrom(tensorString, Optional.empty()); diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java index d64a62143f4..bafec70be59 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java @@ -98,7 +98,8 @@ public class TensorType { } /** - * Returns a tensor type instance from a string on the format + * Returns a tensor type instance from a + * <a href="https://docs.vespa.ai/documentation/reference/tensor.html#tensor-type-spec">tensor type spec</a>: * <code>tensor(dimension1, dimension2, ...)</code> * where each dimension is either * <ul> diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java index c73ff03a0eb..fa022e2bdd1 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java @@ -18,7 +18,7 @@ import java.util.Iterator; /** * Writes tensors on the JSON format used in Vespa tensor document fields: - * A JSON map containing a 'cells' array. + * A JSON map containing a 'cells' or 'values' array. * See a http://docs.vespa.ai/documentation/reference/document-json-put-format.html#tensor * * @author bratseth diff --git a/vespajlib/src/main/java/com/yahoo/text/Text.java b/vespajlib/src/main/java/com/yahoo/text/Text.java index 0acb2407e68..706fd1583a3 100644 --- a/vespajlib/src/main/java/com/yahoo/text/Text.java +++ b/vespajlib/src/main/java/com/yahoo/text/Text.java @@ -143,9 +143,8 @@ public final class Text { sb.append(' '); return sb; } - /** - * Returns a string where any invalid characters in the input string is replaced by spaces - */ + + /** Returns a string where any invalid characters in the input string is replaced by spaces */ public static String stripInvalidCharacters(String string) { StringBuilder stripped = null; // lazy, as most string will not need stripping for (int i = 0; i < string.length();) { diff --git a/vespalib/src/tests/net/socket/socket_test.cpp b/vespalib/src/tests/net/socket/socket_test.cpp index c7cb2a0b6d9..88146cd4fb2 100644 --- a/vespalib/src/tests/net/socket/socket_test.cpp +++ b/vespalib/src/tests/net/socket/socket_test.cpp @@ -324,6 +324,15 @@ TEST("require that sockets can be set blocking and non-blocking") { TEST_DO(verifier.verify_blocking(false)); } +TEST("require that server sockets use non-blocking underlying socket") { + ServerSocket tcp_server("tcp/0"); + ServerSocket ipc_server("ipc/file:my_socket"); + test::SocketOptionsVerifier tcp_verifier(tcp_server.get_fd()); + test::SocketOptionsVerifier ipc_verifier(ipc_server.get_fd()); + TEST_DO(tcp_verifier.verify_blocking(false)); + TEST_DO(ipc_verifier.verify_blocking(false)); +} + TEST("require that tcp nodelay can be enabled and disabled") { SocketHandle handle(socket(my_inet(), SOCK_STREAM, 0)); test::SocketOptionsVerifier verifier(handle.get()); diff --git a/vespalib/src/vespa/vespalib/net/server_socket.cpp b/vespalib/src/vespa/vespalib/net/server_socket.cpp index cc363f49763..2928a9a72b0 100644 --- a/vespalib/src/vespa/vespalib/net/server_socket.cpp +++ b/vespalib/src/vespa/vespalib/net/server_socket.cpp @@ -4,12 +4,27 @@ #include "socket_spec.h" #include <sys/stat.h> #include <dirent.h> +#include <errno.h> +#include <chrono> +#include <thread> #include <vespa/log/log.h> LOG_SETUP(".vespalib.net.server_socket"); namespace vespalib { +namespace { + +SocketHandle adjust_blocking(SocketHandle handle, bool value) { + if (handle.valid() && handle.set_blocking(value)) { + return handle; + } else { + return SocketHandle(); + } +} + +bool is_blocked(int err) { return ((err == EWOULDBLOCK) || (err == EAGAIN)); } + bool is_socket(const vespalib::string &path) { struct stat info; if (path.empty() || (lstat(path.c_str(), &info) != 0)) { @@ -18,6 +33,8 @@ bool is_socket(const vespalib::string &path) { return S_ISSOCK(info.st_mode); } +} + void ServerSocket::cleanup() { @@ -27,8 +44,10 @@ ServerSocket::cleanup() } ServerSocket::ServerSocket(const SocketSpec &spec) - : _handle(spec.server_address().listen()), - _path(spec.path()) + : _handle(adjust_blocking(spec.server_address().listen(), false)), + _path(spec.path()), + _blocking(true), + _shutdown(false) { if (!_handle.valid() && is_socket(_path)) { if (!spec.client_address().connect_async().valid()) { @@ -54,7 +73,9 @@ ServerSocket::ServerSocket(int port) ServerSocket::ServerSocket(ServerSocket &&rhs) : _handle(std::move(rhs._handle)), - _path(std::move(rhs._path)) + _path(std::move(rhs._path)), + _blocking(rhs._blocking), + _shutdown(rhs._shutdown.load(std::memory_order_acquire)) { rhs._path.clear(); } @@ -65,6 +86,8 @@ ServerSocket::operator=(ServerSocket &&rhs) cleanup(); _handle = std::move(rhs._handle); _path = std::move(rhs._path); + _blocking = rhs._blocking; + _shutdown.store(rhs._shutdown.load(std::memory_order_acquire), std::memory_order_release); rhs._path.clear(); return *this; } @@ -78,13 +101,28 @@ ServerSocket::address() const void ServerSocket::shutdown() { + _shutdown.store(true, std::memory_order_release); _handle.shutdown(); } SocketHandle ServerSocket::accept() { - return _handle.accept(); + if (!_blocking) { + return adjust_blocking(_handle.accept(), true); + } else { + for (;;) { + if (_shutdown.load(std::memory_order_acquire)) { + errno = EIO; + return SocketHandle(); + } + SocketHandle res = _handle.accept(); + if (res.valid() || !is_blocked(errno)) { + return adjust_blocking(std::move(res), true); + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + } } } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/net/server_socket.h b/vespalib/src/vespa/vespalib/net/server_socket.h index 1ad0a738117..af6f0a31175 100644 --- a/vespalib/src/vespa/vespalib/net/server_socket.h +++ b/vespalib/src/vespa/vespalib/net/server_socket.h @@ -4,6 +4,7 @@ #include "socket_handle.h" #include "socket_address.h" +#include <atomic> namespace vespalib { @@ -14,9 +15,9 @@ class ServerSocket private: SocketHandle _handle; vespalib::string _path; + bool _blocking; + std::atomic<bool> _shutdown; - explicit ServerSocket(SocketHandle handle); - static ServerSocket listen(const SocketSpec &spec); void cleanup(); public: ServerSocket() : _handle(), _path() {} @@ -30,7 +31,10 @@ public: int get_fd() const { return _handle.get(); } SocketAddress address() const; void shutdown(); - bool set_blocking(bool value) { return _handle.set_blocking(value); } + bool set_blocking(bool value) { + _blocking = value; + return true; + } SocketHandle accept(); }; diff --git a/vespalib/src/vespa/vespalib/websocket/acceptor.cpp b/vespalib/src/vespa/vespalib/websocket/acceptor.cpp index d99035e0be7..4ba1bf3b908 100644 --- a/vespalib/src/vespa/vespalib/websocket/acceptor.cpp +++ b/vespalib/src/vespa/vespalib/websocket/acceptor.cpp @@ -3,34 +3,15 @@ #include "acceptor.h" #include <vespa/vespalib/net/socket_spec.h> #include <functional> -#ifdef __APPLE__ -#include <poll.h> -#endif namespace vespalib::ws { void Acceptor::accept_main(Handler<Socket> &socket_handler) { -#ifdef __APPLE__ - _server_socket.set_blocking(false); -#endif while (!_is_closed) { -#ifdef __APPLE__ - pollfd fds; - fds.fd = _server_socket.get_fd(); - fds.events = POLLIN; - fds.revents = 0; - int res = poll(&fds, 1, 10); - if (res < 1 || fds.revents == 0 || _is_closed) { - continue; - } -#endif SocketHandle handle = _server_socket.accept(); if (handle.valid()) { -#ifdef __APPLE__ - handle.set_blocking(true); -#endif socket_handler.handle(std::make_unique<SimpleSocket>(std::move(handle))); } } diff --git a/vespalog/src/main/java/com/yahoo/log/LevelControllerRepo.java b/vespalog/src/main/java/com/yahoo/log/LevelControllerRepo.java index 1a0dbac7de9..364420796d3 100644 --- a/vespalog/src/main/java/com/yahoo/log/LevelControllerRepo.java +++ b/vespalog/src/main/java/com/yahoo/log/LevelControllerRepo.java @@ -14,10 +14,10 @@ public interface LevelControllerRepo { * @param component The component name string. * @return The LevelController corresponding to that component. Return null if not found. */ - public LevelController getLevelController(String component); + LevelController getLevelController(String component); /** * Close down the level controller repository. Cleanup should be done here. */ - public void close(); + void close(); } diff --git a/vespalog/src/main/java/com/yahoo/log/VespaFormatter.java b/vespalog/src/main/java/com/yahoo/log/VespaFormatter.java index 72a80227138..482d7049079 100644 --- a/vespalog/src/main/java/com/yahoo/log/VespaFormatter.java +++ b/vespalog/src/main/java/com/yahoo/log/VespaFormatter.java @@ -39,13 +39,14 @@ public class VespaFormatter extends SimpleFormatter { } private String serviceName; - private String componentPrefix; + private final String componentPrefix; /** * Default constructor */ public VespaFormatter() { this.serviceName = serviceNameUnsetValue; + this.componentPrefix = null; } /** @@ -193,9 +194,6 @@ public class VespaFormatter extends SimpleFormatter { String message = t.getMessage(); if (t.getCause() == null) { if (message == null) return t.getClass().getSimpleName(); - } else { - if (message == null) return null; - //if (message.equals(t.getCause().getClass().getName() + ": " + t.getCause().getMessage())) return null; } return message; } diff --git a/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java b/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java index d47f78521a2..85d92075827 100644 --- a/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java +++ b/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java @@ -49,13 +49,11 @@ public class VespaLevelControllerRepo implements LevelControllerRepo { * or logging without a logcontrol file) **/ private LevelController defaultLevelCtrl; - private String defaultLogLevel; public VespaLevelControllerRepo(String logCtlFn, String logLevel, String applicationPrefix) { this.logControlFilename = logCtlFn; - this.defaultLogLevel = logLevel; this.appPrefix = applicationPrefix; - defaultLevelCtrl = new DefaultLevelController(defaultLogLevel); + defaultLevelCtrl = new DefaultLevelController(logLevel); openCtlFile(); } @@ -158,7 +156,7 @@ public class VespaLevelControllerRepo implements LevelControllerRepo { if (ctlFile == null) { return defaultLevelCtrl; } - LevelController inherit = null; + LevelController inherit; int lastdot = suffix.lastIndexOf('.'); if (lastdot != -1) { @@ -173,18 +171,18 @@ public class VespaLevelControllerRepo implements LevelControllerRepo { } try { long len = ctlFile.length(); - String append; + StringBuilder sb = new StringBuilder(); if (suffix.equals("")) { - append = "default" + ": "; + sb.append("default: "); } else { - append = "." + suffix + ": "; + sb.append(".").append(suffix).append(": "); } - while ((len + append.length()) % 4 != 0) { - append = append + " "; + while ((len + sb.length()) % 4 != 0) { + sb.append(" "); } - append = append + inherit.getOnOffString() + "\n"; + sb.append(inherit.getOnOffString()).append("\n"); ctlFile.seek(ctlFile.length()); - ctlFile.writeBytes(append); + ctlFile.writeBytes(sb.toString()); extendMapping(); ctrl = levelControllerRepo.getLevelController(suffix); } catch(java.nio.channels.ClosedByInterruptException e) { diff --git a/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java b/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java index 336e9e06cb3..7eede7e1e35 100644 --- a/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java +++ b/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java @@ -31,7 +31,7 @@ class VespaLogHandler extends StreamHandler { * <DD> Log to specified file in append mode * </DL> */ - public VespaLogHandler(LogTarget logTarget, + VespaLogHandler(LogTarget logTarget, LevelControllerRepo levelControllerRepo, String serviceName, String applicationPrefix) { this.logTarget = logTarget; @@ -62,8 +62,7 @@ class VespaLogHandler extends StreamHandler { // provokes rotation of target setOutputStream(logTarget.open()); } catch (RuntimeException e) { - LogRecord r = new LogRecord(Level.SEVERE, - "Unable to open file target"); + LogRecord r = new LogRecord(Level.SEVERE, "Unable to open file target"); r.setThrown(e); emergencyLog(r); setOutputStream(System.err); @@ -73,7 +72,7 @@ class VespaLogHandler extends StreamHandler { closeFileTarget(); } - public LevelController getLevelControl(String component) { + LevelController getLevelControl(String component) { return repo.getLevelController(component); } @@ -96,8 +95,7 @@ class VespaLogHandler extends StreamHandler { emergencyLog(r); } catch (RuntimeException e) { - LogRecord r = new LogRecord(Level.SEVERE, - "Unable to open file target"); + LogRecord r = new LogRecord(Level.SEVERE, "Unable to open file target"); r.setThrown(e); emergencyLog(r); setOutputStream(System.err); @@ -106,7 +104,7 @@ class VespaLogHandler extends StreamHandler { /** Closes the target log file, if there is one */ - public void closeFileTarget() { + synchronized void closeFileTarget() { try { logTarget.close(); } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java index 4f9622de556..a543d43abcd 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java @@ -87,6 +87,7 @@ public class Curator implements AutoCloseable { .connectionTimeoutMs(ZK_CONNECTION_TIMEOUT) .connectString(connectionSpec) .zookeeperFactory(new DNSResolvingFixerZooKeeperFactory(UNKNOWN_HOST_TIMEOUT_MILLIS)) + .dontUseContainerParents() // TODO: Remove when we know ZooKeeper 3.5 works fine, consider waiting until Vespa 8 .build()); } diff --git a/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java b/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java index 5b52f757dad..5615e1133b8 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java @@ -30,7 +30,8 @@ public class ZooKeeperServer extends AbstractComponent implements Runnable { ZooKeeperServer(ZookeeperServerConfig zookeeperServerConfig, boolean startServer) { this.zookeeperServerConfig = zookeeperServerConfig; System.setProperty("zookeeper.jmx.log4j.disable", "true"); - System.setProperty(ZOOKEEPER_JUTE_MAX_BUFFER, "" + zookeeperServerConfig.juteMaxBuffer()); + System.setProperty("zookeeper.snapshot.trust.empty", Boolean.valueOf(zookeeperServerConfig.trustEmptySnapshot()).toString()); + System.setProperty(ZOOKEEPER_JUTE_MAX_BUFFER, Integer.valueOf(zookeeperServerConfig.juteMaxBuffer()).toString()); writeConfigToDisk(zookeeperServerConfig); zkServerThread = new Thread(this, "zookeeper server"); @@ -113,7 +114,7 @@ public class ZooKeeperServer extends AbstractComponent implements Runnable { public void run() { System.setProperty(ZOOKEEPER_JMX_LOG4J_DISABLE, "true"); String[] args = new String[]{getDefaults().underVespaHome(zookeeperServerConfig.zooKeeperConfigFile())}; - log.log(LogLevel.DEBUG, "Starting ZooKeeper server with config: " + args[0]); + log.log(LogLevel.DEBUG, "Starting ZooKeeper server with config file " + args[0]); log.log(LogLevel.INFO, "Trying to establish ZooKeeper quorum (from " + zookeeperServerHostnames(zookeeperServerConfig) + ")"); org.apache.zookeeper.server.quorum.QuorumPeerMain.main(args); } diff --git a/zookeeper-command-line-client/pom.xml b/zookeeper-command-line-client/pom.xml index 1633f14e6dc..f4b75aba906 100644 --- a/zookeeper-command-line-client/pom.xml +++ b/zookeeper-command-line-client/pom.xml @@ -20,6 +20,11 @@ <groupId>commons-cli</groupId> <artifactId>commons-cli</artifactId> </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <version>1.7.25</version> + </dependency> </dependencies> <build> <plugins> diff --git a/zookeeper-command-line-client/src/main/resources/log4j-vespa.properties b/zookeeper-command-line-client/src/main/resources/log4j-vespa.properties index 34a4b174335..add1f89ddb4 100644 --- a/zookeeper-command-line-client/src/main/resources/log4j-vespa.properties +++ b/zookeeper-command-line-client/src/main/resources/log4j-vespa.properties @@ -1,6 +1,7 @@ -log4j.rootLogger=WARN +log4j.rootLogger=debug,console -# CONSOLE is set to be a ConsoleAppender using a PatternLayout -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %m%n +# console is set to be a ConsoleAppender using a PatternLayout +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=[%-5p] %m%n +log4j.appender.console.threshold=warn |