diff options
70 files changed, 796 insertions, 1275 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogForwarder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogForwarder.java index beb96ab8cc8..5ce5c2f64bf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogForwarder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogForwarder.java @@ -52,6 +52,8 @@ public class LogForwarder extends AbstractService implements LogforwarderConfig. this.config = config; setProp("clustertype", "hosts"); setProp("clustername", "admin"); + // Make role information available in supermodel + Optional.ofNullable(config.role).map(r -> setProp("identity.role", r)); } public static Config cfg() { diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java b/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java new file mode 100644 index 00000000000..ec3381033dc --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/SchemaToDerivedConfigExporter.java @@ -0,0 +1,52 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.document.config.DocumenttypesConfig; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.configmodel.producers.DocumentTypes; + +import java.io.IOException; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * Utility class for deriving backend configs for a given schema and exporting them to a set of files. + * + * This can be used by e.g C++ unit tests to generate configs that are checked in as part of the test. + * Place the schema file in a directory $HOME/mydir and run the following in $HOME/git/vespa: + * mvn test -Dtest=SchemaToDerivedConfigExporter -Dschema.exporter.path=$HOME/mydir -pl config-model + * + * Note that the directory path must be absolute. + * + * @author geirst + */ +public class SchemaToDerivedConfigExporter { + + private void exportConfig(String dirPath, DerivedConfiguration config, ApplicationBuilder builder) throws IOException { + config.export(dirPath); + DerivedConfiguration.exportDocuments(new DocumentTypes().produce(builder.getModel(), new DocumenttypesConfig.Builder()), dirPath); + } + + @Test + public void deriveAndExport() throws IOException, ParseException { + var dirPath = System.getProperty("schema.exporter.path"); + // The test is marked as skipped if this is not true. + assumeTrue(dirPath != null); + + var props = new TestProperties(); + var logger = new TestableDeployLogger(); + var builder = ApplicationBuilder.createFromDirectory(dirPath, new MockFileRegistry(), logger, props); + var derived = new DerivedConfiguration(builder.getSchema(null), + new DeployState.Builder().properties(props) + .deployLogger(logger) + .rankProfileRegistry(builder.getRankProfileRegistry()) + .queryProfiles(builder.getQueryProfileRegistry()) + .build()); + exportConfig(dirPath, derived, builder); + } +} diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java index 4cdbab5f89b..f9a2e3bc1a6 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java @@ -17,14 +17,11 @@ import jakarta.servlet.AsyncEvent; import jakarta.servlet.AsyncListener; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; -import org.eclipse.jetty.http2.HTTP2Session; -import org.eclipse.jetty.http2.frames.GoAwayFrame; import org.eclipse.jetty.http2.server.HTTP2ServerConnection; import org.eclipse.jetty.io.Connection; import org.eclipse.jetty.io.EofException; import org.eclipse.jetty.server.HttpConnection; import org.eclipse.jetty.server.Request; -import org.eclipse.jetty.util.Callback; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -182,7 +179,7 @@ class HttpRequestDispatch { // Graceful shutdown implies a GOAWAY frame with 'Error Code' = 'NO_ERROR' and 'Last-Stream-ID' = 2^31-1. // In-flight requests will be allowed to complete before connection is terminated. // See https://datatracker.ietf.org/doc/html/rfc9113#name-goaway for details - ((HTTP2Session)http2.getSession()).goAway(GoAwayFrame.GRACEFUL, Callback.NOOP); + http2.getSession().shutdown(); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java index 0b6a6a97a35..37f374b8c1b 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java @@ -6,6 +6,7 @@ import ai.vespa.searchlib.searchprotocol.protobuf.SearchProtocol.StringProperty; import ai.vespa.searchlib.searchprotocol.protobuf.SearchProtocol.TensorProperty; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; +import com.yahoo.data.access.helpers.MatchFeatureData; import com.yahoo.data.access.simple.Value; import com.yahoo.data.access.slime.SlimeAdapter; import com.yahoo.fs4.GetDocSumsPacket; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index b07c2f14271..08a3285c9b4 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -407,7 +407,7 @@ public class Flags { "new_iddoc_layout", false, List.of("tokle", "bjorncs", "olaa"), "2023-04-24", "2023-05-31", "Whether to use new identity document layout", "Takes effect on node reboot", - HOSTNAME); + HOSTNAME, APPLICATION_ID, VESPA_VERSION); public static final UnboundBooleanFlag RANDOMIZED_ENDPOINT_NAMES = defineFeatureFlag( "randomized-endpoint-names", false, List.of("andreer"), "2023-04-26", "2023-06-30", diff --git a/jdisc_core/src/test/resources/exportPackages.properties b/jdisc_core/src/test/resources/exportPackages.properties index 8af15fe591d..4c09c9efd81 100644 --- a/jdisc_core/src/test/resources/exportPackages.properties +++ b/jdisc_core/src/test/resources/exportPackages.properties @@ -1,3 +1,3 @@ #generated by com.yahoo.jdisc.core.ExportPackages -#Tue Apr 25 14:00:54 CEST 2023 -exportPackages=org.osgi.framework; version\="1.10.0", org.osgi.framework.connect; version\="1.0.0", org.osgi.framework.dto; uses\:\="org.osgi.dto"; version\="1.8.0", org.osgi.framework.hooks.bundle; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.resolver; uses\:\="org.osgi.framework.wiring"; version\="1.0.0", org.osgi.framework.hooks.service; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.weaving; uses\:\="org.osgi.framework.wiring"; version\="1.1.0", org.osgi.framework.launch; uses\:\="org.osgi.framework"; version\="1.2.0", org.osgi.framework.namespace; uses\:\="org.osgi.resource"; version\="1.2.0", org.osgi.framework.startlevel; uses\:\="org.osgi.framework"; version\="1.0.0", org.osgi.framework.startlevel.dto; uses\:\="org.osgi.dto"; version\="1.0.0", org.osgi.framework.wiring; uses\:\="org.osgi.framework,org.osgi.resource"; version\="1.2.0", org.osgi.framework.wiring.dto; uses\:\="org.osgi.dto,org.osgi.resource.dto"; version\="1.3.0", org.osgi.resource; version\="1.0.1", org.osgi.resource.dto; uses\:\="org.osgi.dto"; version\="1.0.1", org.osgi.service.packageadmin; uses\:\="org.osgi.framework"; version\="1.2.1", org.osgi.service.startlevel; uses\:\="org.osgi.framework"; version\="1.1.1", org.osgi.service.url; version\="1.0.1", org.osgi.service.resolver; uses\:\="org.osgi.resource"; version\="1.1.1", org.osgi.util.tracker; uses\:\="org.osgi.framework"; version\="1.5.3", org.osgi.dto; version\="1.1.1", org.osgi.service.condition; version\="1.0.0", javax.crypto.interfaces; version\="0.0.0.JavaSE_020", java.util; version\="0.0.0.JavaSE_020", java.security; version\="0.0.0.JavaSE_020", java.nio.file.spi; version\="0.0.0.JavaSE_020", java.nio; version\="0.0.0.JavaSE_020", java.util.jar; version\="0.0.0.JavaSE_020", javax.security.auth; version\="0.0.0.JavaSE_020", java.lang.ref; version\="0.0.0.JavaSE_020", java.util.regex; version\="0.0.0.JavaSE_020", java.net.spi; version\="0.0.0.JavaSE_020", java.lang.module; version\="0.0.0.JavaSE_020", java.lang.invoke; version\="0.0.0.JavaSE_020", java.time.format; version\="0.0.0.JavaSE_020", java.util.concurrent.locks; version\="0.0.0.JavaSE_020", java.time.temporal; version\="0.0.0.JavaSE_020", java.util.zip; version\="0.0.0.JavaSE_020", java.nio.file.attribute; version\="0.0.0.JavaSE_020", java.util.random; version\="0.0.0.JavaSE_020", java.text; version\="0.0.0.JavaSE_020", javax.crypto.spec; version\="0.0.0.JavaSE_020", java.util.stream; version\="0.0.0.JavaSE_020", java.time; version\="0.0.0.JavaSE_020", java.lang; version\="0.0.0.JavaSE_020", java.lang.runtime; version\="0.0.0.JavaSE_020", java.util.function; version\="0.0.0.JavaSE_020", javax.net; version\="0.0.0.JavaSE_020", javax.security.auth.x500; version\="0.0.0.JavaSE_020", java.lang.reflect; version\="0.0.0.JavaSE_020", javax.security.auth.callback; version\="0.0.0.JavaSE_020", javax.security.auth.login; version\="0.0.0.JavaSE_020", javax.net.ssl; version\="0.0.0.JavaSE_020", java.lang.constant; version\="0.0.0.JavaSE_020", java.security.interfaces; version\="0.0.0.JavaSE_020", java.text.spi; version\="0.0.0.JavaSE_020", java.nio.channels.spi; version\="0.0.0.JavaSE_020", java.math; version\="0.0.0.JavaSE_020", java.nio.file; version\="0.0.0.JavaSE_020", java.util.concurrent.atomic; version\="0.0.0.JavaSE_020", java.security.cert; version\="0.0.0.JavaSE_020", java.security.spec; version\="0.0.0.JavaSE_020", java.nio.channels; version\="0.0.0.JavaSE_020", java.time.chrono; version\="0.0.0.JavaSE_020", javax.crypto; version\="0.0.0.JavaSE_020", java.time.zone; version\="0.0.0.JavaSE_020", java.nio.charset; version\="0.0.0.JavaSE_020", java.lang.foreign; version\="0.0.0.JavaSE_020", java.io; version\="0.0.0.JavaSE_020", java.util.spi; version\="0.0.0.JavaSE_020", java.net; version\="0.0.0.JavaSE_020", javax.security.cert; version\="0.0.0.JavaSE_020", java.lang.annotation; version\="0.0.0.JavaSE_020", javax.security.auth.spi; version\="0.0.0.JavaSE_020", java.util.concurrent; version\="0.0.0.JavaSE_020", java.nio.charset.spi; version\="0.0.0.JavaSE_020", javax.tools; version\="0.0.0.JavaSE_020", javax.lang.model.type; version\="0.0.0.JavaSE_020", javax.lang.model.util; version\="0.0.0.JavaSE_020", javax.lang.model; version\="0.0.0.JavaSE_020", javax.annotation.processing; version\="0.0.0.JavaSE_020", javax.lang.model.element; version\="0.0.0.JavaSE_020", java.awt.datatransfer; version\="0.0.0.JavaSE_020", javax.accessibility; version\="0.0.0.JavaSE_020", java.awt.event; version\="0.0.0.JavaSE_020", javax.swing.text.html; version\="0.0.0.JavaSE_020", javax.imageio.spi; version\="0.0.0.JavaSE_020", javax.swing.border; version\="0.0.0.JavaSE_020", javax.sound.sampled.spi; version\="0.0.0.JavaSE_020", javax.imageio.plugins.bmp; version\="0.0.0.JavaSE_020", java.awt.color; version\="0.0.0.JavaSE_020", java.awt.geom; version\="0.0.0.JavaSE_020", javax.imageio.plugins.jpeg; version\="0.0.0.JavaSE_020", javax.swing.tree; version\="0.0.0.JavaSE_020", javax.imageio.plugins.tiff; version\="0.0.0.JavaSE_020", java.awt.print; version\="0.0.0.JavaSE_020", java.awt.image; version\="0.0.0.JavaSE_020", javax.imageio.metadata; version\="0.0.0.JavaSE_020", javax.swing.table; version\="0.0.0.JavaSE_020", javax.sound.midi.spi; version\="0.0.0.JavaSE_020", javax.print.attribute.standard; version\="0.0.0.JavaSE_020", javax.swing.colorchooser; version\="0.0.0.JavaSE_020", javax.swing; version\="0.0.0.JavaSE_020", java.awt.image.renderable; version\="0.0.0.JavaSE_020", javax.swing.plaf.multi; version\="0.0.0.JavaSE_020", java.awt.im; version\="0.0.0.JavaSE_020", javax.print.event; version\="0.0.0.JavaSE_020", javax.swing.plaf.metal; version\="0.0.0.JavaSE_020", java.beans.beancontext; version\="0.0.0.JavaSE_020", java.awt; version\="0.0.0.JavaSE_020", javax.imageio.stream; version\="0.0.0.JavaSE_020", javax.swing.event; version\="0.0.0.JavaSE_020", java.awt.desktop; version\="0.0.0.JavaSE_020", javax.swing.plaf.synth; version\="0.0.0.JavaSE_020", java.beans; version\="0.0.0.JavaSE_020", javax.swing.text.html.parser; version\="0.0.0.JavaSE_020", javax.swing.text.rtf; version\="0.0.0.JavaSE_020", java.awt.font; version\="0.0.0.JavaSE_020", javax.imageio; version\="0.0.0.JavaSE_020", java.awt.im.spi; version\="0.0.0.JavaSE_020", java.applet; version\="0.0.0.JavaSE_020", javax.sound.midi; version\="0.0.0.JavaSE_020", java.awt.dnd; version\="0.0.0.JavaSE_020", javax.swing.text; version\="0.0.0.JavaSE_020", javax.swing.plaf.basic; version\="0.0.0.JavaSE_020", javax.swing.undo; version\="0.0.0.JavaSE_020", javax.swing.plaf; version\="0.0.0.JavaSE_020", javax.swing.filechooser; version\="0.0.0.JavaSE_020", javax.imageio.event; version\="0.0.0.JavaSE_020", javax.sound.sampled; version\="0.0.0.JavaSE_020", javax.print.attribute; version\="0.0.0.JavaSE_020", javax.print; version\="0.0.0.JavaSE_020", javax.swing.plaf.nimbus; version\="0.0.0.JavaSE_020", java.lang.instrument; version\="0.0.0.JavaSE_020", java.util.logging; version\="0.0.0.JavaSE_020", javax.management.remote; version\="0.0.0.JavaSE_020", javax.management.monitor; version\="0.0.0.JavaSE_020", javax.management.modelmbean; version\="0.0.0.JavaSE_020", javax.management.timer; version\="0.0.0.JavaSE_020", javax.management; version\="0.0.0.JavaSE_020", javax.management.relation; version\="0.0.0.JavaSE_020", javax.management.loading; version\="0.0.0.JavaSE_020", javax.management.openmbean; version\="0.0.0.JavaSE_020", java.lang.management; version\="0.0.0.JavaSE_020", javax.management.remote.rmi; version\="0.0.0.JavaSE_020", javax.naming; version\="0.0.0.JavaSE_020", javax.naming.spi; version\="0.0.0.JavaSE_020", javax.naming.event; version\="0.0.0.JavaSE_020", javax.naming.ldap.spi; version\="0.0.0.JavaSE_020", javax.naming.ldap; version\="0.0.0.JavaSE_020", javax.naming.directory; version\="0.0.0.JavaSE_020", java.net.http; version\="0.0.0.JavaSE_020", java.util.prefs; version\="0.0.0.JavaSE_020", javax.rmi.ssl; version\="0.0.0.JavaSE_020", java.rmi.dgc; version\="0.0.0.JavaSE_020", java.rmi; version\="0.0.0.JavaSE_020", java.rmi.server; version\="0.0.0.JavaSE_020", java.rmi.registry; version\="0.0.0.JavaSE_020", javax.script; version\="0.0.0.JavaSE_020", javax.security.auth.kerberos; version\="0.0.0.JavaSE_020", org.ietf.jgss; version\="0.0.0.JavaSE_020", javax.security.sasl; version\="0.0.0.JavaSE_020", javax.smartcardio; version\="0.0.0.JavaSE_020", java.sql; version\="0.0.0.JavaSE_020", javax.sql; version\="0.0.0.JavaSE_020", javax.sql.rowset.spi; version\="0.0.0.JavaSE_020", javax.sql.rowset.serial; version\="0.0.0.JavaSE_020", javax.sql.rowset; version\="0.0.0.JavaSE_020", javax.transaction.xa; version\="0.0.0.JavaSE_020", org.w3c.dom.events; version\="0.0.0.JavaSE_020", org.w3c.dom.ranges; version\="0.0.0.JavaSE_020", org.w3c.dom.ls; version\="0.0.0.JavaSE_020", javax.xml.stream.util; version\="0.0.0.JavaSE_020", javax.xml.namespace; version\="0.0.0.JavaSE_020", javax.xml.transform.stax; version\="0.0.0.JavaSE_020", org.xml.sax.helpers; version\="0.0.0.JavaSE_020", org.w3c.dom.views; version\="0.0.0.JavaSE_020", org.w3c.dom.bootstrap; version\="0.0.0.JavaSE_020", org.w3c.dom; version\="0.0.0.JavaSE_020", javax.xml.transform.stream; version\="0.0.0.JavaSE_020", javax.xml.transform.dom; version\="0.0.0.JavaSE_020", javax.xml.validation; version\="0.0.0.JavaSE_020", javax.xml.parsers; version\="0.0.0.JavaSE_020", org.xml.sax.ext; version\="0.0.0.JavaSE_020", javax.xml; version\="0.0.0.JavaSE_020", javax.xml.transform.sax; version\="0.0.0.JavaSE_020", javax.xml.datatype; version\="0.0.0.JavaSE_020", javax.xml.catalog; version\="0.0.0.JavaSE_020", org.w3c.dom.traversal; version\="0.0.0.JavaSE_020", javax.xml.stream.events; version\="0.0.0.JavaSE_020", javax.xml.stream; version\="0.0.0.JavaSE_020", org.xml.sax; version\="0.0.0.JavaSE_020", javax.xml.transform; version\="0.0.0.JavaSE_020", javax.xml.xpath; version\="0.0.0.JavaSE_020", javax.xml.crypto; version\="0.0.0.JavaSE_020", javax.xml.crypto.dsig; version\="0.0.0.JavaSE_020", javax.xml.crypto.dsig.spec; version\="0.0.0.JavaSE_020", javax.xml.crypto.dsig.keyinfo; version\="0.0.0.JavaSE_020", javax.xml.crypto.dsig.dom; version\="0.0.0.JavaSE_020", javax.xml.crypto.dom; version\="0.0.0.JavaSE_020", com.sun.java.accessibility.util; version\="0.0.0.JavaSE_020", com.sun.tools.attach; version\="0.0.0.JavaSE_020", com.sun.tools.attach.spi; version\="0.0.0.JavaSE_020", com.sun.source.tree; version\="0.0.0.JavaSE_020", com.sun.source.util; version\="0.0.0.JavaSE_020", com.sun.tools.javac; version\="0.0.0.JavaSE_020", com.sun.source.doctree; version\="0.0.0.JavaSE_020", jdk.dynalink.support; version\="0.0.0.JavaSE_020", jdk.dynalink; version\="0.0.0.JavaSE_020", jdk.dynalink.linker; version\="0.0.0.JavaSE_020", jdk.dynalink.beans; version\="0.0.0.JavaSE_020", jdk.dynalink.linker.support; version\="0.0.0.JavaSE_020", com.sun.net.httpserver; version\="0.0.0.JavaSE_020", com.sun.net.httpserver.spi; version\="0.0.0.JavaSE_020", com.sun.jarsigner; version\="0.0.0.JavaSE_020", jdk.security.jarsigner; version\="0.0.0.JavaSE_020", jdk.javadoc.doclet; version\="0.0.0.JavaSE_020", com.sun.tools.jconsole; version\="0.0.0.JavaSE_020", com.sun.jdi; version\="0.0.0.JavaSE_020", com.sun.jdi.request; version\="0.0.0.JavaSE_020", com.sun.jdi.connect; version\="0.0.0.JavaSE_020", com.sun.jdi.event; version\="0.0.0.JavaSE_020", com.sun.jdi.connect.spi; version\="0.0.0.JavaSE_020", jdk.jfr.consumer; version\="0.0.0.JavaSE_020", jdk.jfr; version\="0.0.0.JavaSE_020", jdk.jshell.spi; version\="0.0.0.JavaSE_020", jdk.jshell.tool; version\="0.0.0.JavaSE_020", jdk.jshell; version\="0.0.0.JavaSE_020", jdk.jshell.execution; version\="0.0.0.JavaSE_020", netscape.javascript; version\="0.0.0.JavaSE_020", com.sun.management; version\="0.0.0.JavaSE_020", jdk.management.jfr; version\="0.0.0.JavaSE_020", jdk.net; version\="0.0.0.JavaSE_020", jdk.nio; version\="0.0.0.JavaSE_020", jdk.nio.mapmode; version\="0.0.0.JavaSE_020", com.sun.nio.sctp; version\="0.0.0.JavaSE_020", com.sun.security.auth.login; version\="0.0.0.JavaSE_020", com.sun.security.auth; version\="0.0.0.JavaSE_020", com.sun.security.auth.callback; version\="0.0.0.JavaSE_020", com.sun.security.auth.module; version\="0.0.0.JavaSE_020", com.sun.security.jgss; version\="0.0.0.JavaSE_020", sun.reflect; version\="0.0.0.JavaSE_020", sun.misc; version\="0.0.0.JavaSE_020", com.sun.nio.file; version\="0.0.0.JavaSE_020", jdk.swing.interop; version\="0.0.0.JavaSE_020", org.w3c.dom.xpath; version\="0.0.0.JavaSE_020", org.w3c.dom.css; version\="0.0.0.JavaSE_020", org.w3c.dom.stylesheets; version\="0.0.0.JavaSE_020", org.w3c.dom.html; version\="0.0.0.JavaSE_020", com.yahoo.jdisc, com.yahoo.jdisc.application, com.yahoo.jdisc.handler, com.yahoo.jdisc.service, com.yahoo.jdisc.statistics, com.yahoo.jdisc.refcount, javax.inject;version\=1.0.0, org.aopalliance.intercept, org.aopalliance.aop, com.google.common.annotations;version\="27.1.0",com.google.common.base;version\="27.1.0",com.google.common.cache;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent",com.google.common.collect;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.escape;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.eventbus;version\="27.1.0",com.google.common.graph;version\="27.1.0";uses\:\="com.google.common.collect",com.google.common.hash;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.html;version\="27.1.0";uses\:\="com.google.common.escape",com.google.common.io;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.graph,com.google.common.hash",com.google.common.math;version\="27.1.0",com.google.common.net;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.escape",com.google.common.primitives;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.reflect;version\="27.1.0";uses\:\="com.google.common.collect,com.google.common.io",com.google.common.util.concurrent;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent.internal",com.google.common.xml;version\="27.1.0";uses\:\="com.google.common.escape", com.google.inject;version\="1.4",com.google.inject.binder;version\="1.4",com.google.inject.matcher;version\="1.4",com.google.inject.multibindings;version\="1.4",com.google.inject.name;version\="1.4",com.google.inject.spi;version\="1.4",com.google.inject.util;version\="1.4", org.slf4j;version\=1.7.32, org.slf4j.spi;version\=1.7.32, org.slf4j.helpers;version\=1.7.32, org.slf4j.event;version\=1.7.32, org.slf4j.impl;version\=1.7.32, org.apache.commons.logging;version\=1.2, org.apache.commons.logging.impl;version\=1.2, com.sun.jna;version\=5.11.0, com.sun.jna.ptr;version\=5.11.0, com.sun.jna.win32;version\=5.11.0, org.apache.log4j;version\=1.2.17,org.apache.log4j.helpers;version\=1.2.17,org.apache.log4j.spi;version\=1.2.17,org.apache.log4j.xml;version\=1.2.17, com.yahoo.component.annotation;version\="1.0.0", com.yahoo.config;version\=1.0.0, com.yahoo.vespa.defaults;version\=1.0.0, ai.vespa.http;version\=1.0.0,ai.vespa.llm.completion;version\=1.0.0,ai.vespa.llm.test;version\=1.0.0,ai.vespa.llm;version\=1.0.0,ai.vespa.validation;version\=1.0.0,com.yahoo.binaryprefix;version\=1.0.0,com.yahoo.collections;version\=1.0.0,com.yahoo.compress;version\=1.0.0,com.yahoo.concurrent.classlock;version\=1.0.0,com.yahoo.concurrent.maintenance;version\=1.0.0,com.yahoo.concurrent;version\=1.0.0,com.yahoo.data.access.simple;version\=1.0.0,com.yahoo.data.access.slime;version\=1.0.0,com.yahoo.data.access;version\=1.0.0,com.yahoo.errorhandling;version\=1.0.0,com.yahoo.exception;version\=1.0.0,com.yahoo.geo;version\=1.0.0,com.yahoo.io.reader;version\=1.0.0,com.yahoo.io;version\=1.0.0,com.yahoo.javacc;version\=1.0.0,com.yahoo.lang;version\=1.0.0,com.yahoo.nativec;version\=1.0.0,com.yahoo.net;version\=1.0.0,com.yahoo.path;version\=1.0.0,com.yahoo.protect;version\=1.0.0,com.yahoo.reflection;version\=1.0.0,com.yahoo.slime;version\=1.0.0,com.yahoo.stream;version\=1.0.0,com.yahoo.system.execution;version\=1.0.0,com.yahoo.system;version\=1.0.0,com.yahoo.tensor.evaluation;version\=1.0.0,com.yahoo.tensor.functions;version\=1.0.0,com.yahoo.tensor.serialization;version\=1.0.0,com.yahoo.tensor;version\=1.0.0,com.yahoo.text.internal;version\=1.0.0,com.yahoo.text;version\=1.0.0,com.yahoo.time;version\=1.0.0,com.yahoo.transaction;version\=1.0.0,com.yahoo.vespa.objects;version\=1.0.0,com.yahoo.yolean.chain;version\=1.0.0,com.yahoo.yolean.concurrent;version\=1.0.0,com.yahoo.yolean.function;version\=1.0.0,com.yahoo.yolean.system;version\=1.0.0,com.yahoo.yolean.trace;version\=1.0.0,com.yahoo.yolean;version\=1.0.0, com.yahoo.log.event;version\=1.0.0,com.yahoo.log.impl;version\=1.0.0,com.yahoo.log;version\=1.0.0, javax.xml.bind;version\="2.3";uses\:\="javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.annotation;version\="2.3";uses\:\="javax.xml.bind,javax.xml.parsers,javax.xml.transform,javax.xml.transform.dom,org.w3c.dom",javax.xml.bind.annotation.adapters;version\="2.3",javax.xml.bind.attachment;version\="2.3";uses\:\="javax.activation",javax.xml.bind.helpers;version\="2.3";uses\:\="javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.util;version\="2.3";uses\:\="javax.xml.bind,javax.xml.transform.sax", com.sun.istack;version\="3.0.5";uses\:\="javax.activation,javax.xml.stream,org.xml.sax,org.xml.sax.helpers",com.sun.istack.localization;version\="3.0.5",com.sun.istack.logging;version\="3.0.5",com.sun.xml.bind;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.annotation;version\="2.3.0",com.sun.xml.bind.api;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.api.impl;version\="2.3.0",com.sun.xml.bind.marshaller;uses\:\="javax.xml.parsers,org.w3c.dom,org.xml.sax,org.xml.sax.helpers";version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;version\="2.3.0",com.sun.xml.bind.v2;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.core;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.impl,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.namespace,javax.xml.transform";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.nav";version\="2.3.0",com.sun.xml.bind.v2.model.nav;uses\:\="com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.util;uses\:\="javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.xml.bind.v2.model.annotation,javax.activation,javax.xml.bind,javax.xml.bind.annotation.adapters";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen.episode;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="javax.xml.parsers,javax.xml.transform,javax.xml.validation,javax.xml.xpath";version\="2.3.0",com.sun.xml.txw2;uses\:\="com.sun.xml.txw2.output,javax.xml.namespace";version\="2.3.0",com.sun.xml.txw2.annotation;version\="2.3.0",com.sun.xml.txw2.output;uses\:\="com.sun.xml.txw2,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.dom,javax.xml.transform.sax,javax.xml.transform.stream,org.w3c.dom,org.xml.sax,org.xml.sax.ext,org.xml.sax.helpers";version\="2.3.0", com.sun.xml.bind;uses\:\="com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.datatype,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.api;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.xml.bind,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.marshaller;version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;uses\:\="com.sun.xml.bind,javax.xml.bind.helpers,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,javax.xml.bind";version\="2.3.0",com.sun.xml.bind.v2.bytecode;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.model.runtime;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.namespace,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.istack,com.sun.xml.bind.api,com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.property,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.output;uses\:\="com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.runtime,com.sun.xml.fastinfoset.stax,javax.xml.stream,org.jvnet.staxex,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.property;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,com.sun.xml.bind.v2.runtime.unmarshaller,com.sun.xml.bind.v2.util,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect.opt;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="com.sun.xml.bind,com.sun.xml.bind.api,com.sun.xml.bind.unmarshaller,com.sun.xml.bind.util,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.reflect,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.txw2.output,javax.xml.bind,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.schemagen.xmlschema;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.namespace,javax.xml.transform.stream,org.xml.sax";version\="2.3.0", javax.activation;uses\:\="com.sun.activation.registries";version\="1.2",com.sun.activation.viewers;uses\:\="javax.activation";version\="1.2.0",com.sun.activation.registries;version\="1.2.0" +#Thu Apr 27 20:18:58 UTC 2023 +exportPackages=org.osgi.framework; version\="1.10.0", org.osgi.framework.connect; version\="1.0.0", org.osgi.framework.dto; uses\:\="org.osgi.dto"; version\="1.8.0", org.osgi.framework.hooks.bundle; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.resolver; uses\:\="org.osgi.framework.wiring"; version\="1.0.0", org.osgi.framework.hooks.service; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.weaving; uses\:\="org.osgi.framework.wiring"; version\="1.1.0", org.osgi.framework.launch; uses\:\="org.osgi.framework"; version\="1.2.0", org.osgi.framework.namespace; uses\:\="org.osgi.resource"; version\="1.2.0", org.osgi.framework.startlevel; uses\:\="org.osgi.framework"; version\="1.0.0", org.osgi.framework.startlevel.dto; uses\:\="org.osgi.dto"; version\="1.0.0", org.osgi.framework.wiring; uses\:\="org.osgi.framework,org.osgi.resource"; version\="1.2.0", org.osgi.framework.wiring.dto; uses\:\="org.osgi.dto,org.osgi.resource.dto"; version\="1.3.0", org.osgi.resource; version\="1.0.1", org.osgi.resource.dto; uses\:\="org.osgi.dto"; version\="1.0.1", org.osgi.service.packageadmin; uses\:\="org.osgi.framework"; version\="1.2.1", org.osgi.service.startlevel; uses\:\="org.osgi.framework"; version\="1.1.1", org.osgi.service.url; version\="1.0.1", org.osgi.service.resolver; uses\:\="org.osgi.resource"; version\="1.1.1", org.osgi.util.tracker; uses\:\="org.osgi.framework"; version\="1.5.3", org.osgi.dto; version\="1.1.1", org.osgi.service.condition; version\="1.0.0", java.lang.module; version\="0.0.0.JavaSE_017", java.lang.invoke; version\="0.0.0.JavaSE_017", java.time.format; version\="0.0.0.JavaSE_017", java.util.concurrent.locks; version\="0.0.0.JavaSE_017", java.time.temporal; version\="0.0.0.JavaSE_017", java.util.zip; version\="0.0.0.JavaSE_017", java.nio.file.attribute; version\="0.0.0.JavaSE_017", java.util.random; version\="0.0.0.JavaSE_017", java.text; version\="0.0.0.JavaSE_017", javax.crypto.spec; version\="0.0.0.JavaSE_017", java.util.stream; version\="0.0.0.JavaSE_017", java.time; version\="0.0.0.JavaSE_017", java.lang; version\="0.0.0.JavaSE_017", java.lang.runtime; version\="0.0.0.JavaSE_017", java.util.function; version\="0.0.0.JavaSE_017", javax.net; version\="0.0.0.JavaSE_017", javax.security.auth.x500; version\="0.0.0.JavaSE_017", java.lang.reflect; version\="0.0.0.JavaSE_017", javax.security.auth.callback; version\="0.0.0.JavaSE_017", javax.security.auth.login; version\="0.0.0.JavaSE_017", javax.net.ssl; version\="0.0.0.JavaSE_017", java.lang.constant; version\="0.0.0.JavaSE_017", java.security.interfaces; version\="0.0.0.JavaSE_017", java.text.spi; version\="0.0.0.JavaSE_017", java.nio.channels.spi; version\="0.0.0.JavaSE_017", java.math; version\="0.0.0.JavaSE_017", java.nio.file; version\="0.0.0.JavaSE_017", java.util.concurrent.atomic; version\="0.0.0.JavaSE_017", java.security.cert; version\="0.0.0.JavaSE_017", java.security.spec; version\="0.0.0.JavaSE_017", java.nio.channels; version\="0.0.0.JavaSE_017", java.time.chrono; version\="0.0.0.JavaSE_017", javax.crypto; version\="0.0.0.JavaSE_017", java.time.zone; version\="0.0.0.JavaSE_017", java.nio.charset; version\="0.0.0.JavaSE_017", java.io; version\="0.0.0.JavaSE_017", java.util.spi; version\="0.0.0.JavaSE_017", java.net; version\="0.0.0.JavaSE_017", javax.security.cert; version\="0.0.0.JavaSE_017", java.lang.annotation; version\="0.0.0.JavaSE_017", javax.security.auth.spi; version\="0.0.0.JavaSE_017", java.util.concurrent; version\="0.0.0.JavaSE_017", java.nio.charset.spi; version\="0.0.0.JavaSE_017", javax.crypto.interfaces; version\="0.0.0.JavaSE_017", java.util; version\="0.0.0.JavaSE_017", java.security; version\="0.0.0.JavaSE_017", java.nio.file.spi; version\="0.0.0.JavaSE_017", java.nio; version\="0.0.0.JavaSE_017", java.util.jar; version\="0.0.0.JavaSE_017", javax.security.auth; version\="0.0.0.JavaSE_017", java.lang.ref; version\="0.0.0.JavaSE_017", java.util.regex; version\="0.0.0.JavaSE_017", java.net.spi; version\="0.0.0.JavaSE_017", javax.tools; version\="0.0.0.JavaSE_017", javax.lang.model.type; version\="0.0.0.JavaSE_017", javax.lang.model.util; version\="0.0.0.JavaSE_017", javax.lang.model; version\="0.0.0.JavaSE_017", javax.annotation.processing; version\="0.0.0.JavaSE_017", javax.lang.model.element; version\="0.0.0.JavaSE_017", java.awt.datatransfer; version\="0.0.0.JavaSE_017", javax.imageio.plugins.tiff; version\="0.0.0.JavaSE_017", java.awt.print; version\="0.0.0.JavaSE_017", java.awt.image; version\="0.0.0.JavaSE_017", javax.imageio.metadata; version\="0.0.0.JavaSE_017", javax.swing.table; version\="0.0.0.JavaSE_017", javax.sound.midi.spi; version\="0.0.0.JavaSE_017", javax.print.attribute.standard; version\="0.0.0.JavaSE_017", javax.swing.colorchooser; version\="0.0.0.JavaSE_017", javax.swing; version\="0.0.0.JavaSE_017", java.awt.image.renderable; version\="0.0.0.JavaSE_017", javax.swing.plaf.multi; version\="0.0.0.JavaSE_017", java.awt.im; version\="0.0.0.JavaSE_017", javax.print.event; version\="0.0.0.JavaSE_017", javax.swing.plaf.metal; version\="0.0.0.JavaSE_017", java.beans.beancontext; version\="0.0.0.JavaSE_017", java.awt; version\="0.0.0.JavaSE_017", javax.imageio.stream; version\="0.0.0.JavaSE_017", javax.swing.event; version\="0.0.0.JavaSE_017", java.awt.desktop; version\="0.0.0.JavaSE_017", javax.swing.plaf.synth; version\="0.0.0.JavaSE_017", java.beans; version\="0.0.0.JavaSE_017", javax.swing.text.html.parser; version\="0.0.0.JavaSE_017", javax.swing.text.rtf; version\="0.0.0.JavaSE_017", java.awt.font; version\="0.0.0.JavaSE_017", javax.imageio; version\="0.0.0.JavaSE_017", java.awt.im.spi; version\="0.0.0.JavaSE_017", java.applet; version\="0.0.0.JavaSE_017", javax.sound.midi; version\="0.0.0.JavaSE_017", java.awt.dnd; version\="0.0.0.JavaSE_017", javax.swing.text; version\="0.0.0.JavaSE_017", javax.swing.plaf.basic; version\="0.0.0.JavaSE_017", javax.swing.undo; version\="0.0.0.JavaSE_017", javax.swing.plaf; version\="0.0.0.JavaSE_017", javax.swing.filechooser; version\="0.0.0.JavaSE_017", javax.imageio.event; version\="0.0.0.JavaSE_017", javax.sound.sampled; version\="0.0.0.JavaSE_017", javax.print.attribute; version\="0.0.0.JavaSE_017", javax.print; version\="0.0.0.JavaSE_017", javax.swing.plaf.nimbus; version\="0.0.0.JavaSE_017", javax.accessibility; version\="0.0.0.JavaSE_017", java.awt.event; version\="0.0.0.JavaSE_017", javax.swing.text.html; version\="0.0.0.JavaSE_017", javax.imageio.spi; version\="0.0.0.JavaSE_017", javax.swing.border; version\="0.0.0.JavaSE_017", javax.sound.sampled.spi; version\="0.0.0.JavaSE_017", javax.imageio.plugins.bmp; version\="0.0.0.JavaSE_017", java.awt.color; version\="0.0.0.JavaSE_017", java.awt.geom; version\="0.0.0.JavaSE_017", javax.imageio.plugins.jpeg; version\="0.0.0.JavaSE_017", javax.swing.tree; version\="0.0.0.JavaSE_017", java.lang.instrument; version\="0.0.0.JavaSE_017", java.util.logging; version\="0.0.0.JavaSE_017", javax.management.remote; version\="0.0.0.JavaSE_017", javax.management.monitor; version\="0.0.0.JavaSE_017", javax.management.modelmbean; version\="0.0.0.JavaSE_017", javax.management.timer; version\="0.0.0.JavaSE_017", javax.management; version\="0.0.0.JavaSE_017", javax.management.relation; version\="0.0.0.JavaSE_017", javax.management.loading; version\="0.0.0.JavaSE_017", javax.management.openmbean; version\="0.0.0.JavaSE_017", java.lang.management; version\="0.0.0.JavaSE_017", javax.management.remote.rmi; version\="0.0.0.JavaSE_017", javax.naming.ldap; version\="0.0.0.JavaSE_017", javax.naming.directory; version\="0.0.0.JavaSE_017", javax.naming.event; version\="0.0.0.JavaSE_017", javax.naming.ldap.spi; version\="0.0.0.JavaSE_017", javax.naming; version\="0.0.0.JavaSE_017", javax.naming.spi; version\="0.0.0.JavaSE_017", java.net.http; version\="0.0.0.JavaSE_017", java.util.prefs; version\="0.0.0.JavaSE_017", java.rmi; version\="0.0.0.JavaSE_017", java.rmi.server; version\="0.0.0.JavaSE_017", java.rmi.registry; version\="0.0.0.JavaSE_017", javax.rmi.ssl; version\="0.0.0.JavaSE_017", java.rmi.dgc; version\="0.0.0.JavaSE_017", javax.script; version\="0.0.0.JavaSE_017", javax.security.auth.kerberos; version\="0.0.0.JavaSE_017", org.ietf.jgss; version\="0.0.0.JavaSE_017", javax.security.sasl; version\="0.0.0.JavaSE_017", javax.smartcardio; version\="0.0.0.JavaSE_017", java.sql; version\="0.0.0.JavaSE_017", javax.sql; version\="0.0.0.JavaSE_017", javax.sql.rowset.serial; version\="0.0.0.JavaSE_017", javax.sql.rowset; version\="0.0.0.JavaSE_017", javax.sql.rowset.spi; version\="0.0.0.JavaSE_017", javax.transaction.xa; version\="0.0.0.JavaSE_017", javax.xml.namespace; version\="0.0.0.JavaSE_017", javax.xml.transform.stax; version\="0.0.0.JavaSE_017", org.xml.sax.helpers; version\="0.0.0.JavaSE_017", org.w3c.dom.views; version\="0.0.0.JavaSE_017", org.w3c.dom.bootstrap; version\="0.0.0.JavaSE_017", org.w3c.dom; version\="0.0.0.JavaSE_017", javax.xml.transform.stream; version\="0.0.0.JavaSE_017", javax.xml.transform.dom; version\="0.0.0.JavaSE_017", javax.xml.validation; version\="0.0.0.JavaSE_017", javax.xml.parsers; version\="0.0.0.JavaSE_017", org.xml.sax.ext; version\="0.0.0.JavaSE_017", javax.xml; version\="0.0.0.JavaSE_017", javax.xml.transform.sax; version\="0.0.0.JavaSE_017", javax.xml.datatype; version\="0.0.0.JavaSE_017", javax.xml.catalog; version\="0.0.0.JavaSE_017", org.w3c.dom.traversal; version\="0.0.0.JavaSE_017", javax.xml.stream.events; version\="0.0.0.JavaSE_017", javax.xml.stream; version\="0.0.0.JavaSE_017", org.xml.sax; version\="0.0.0.JavaSE_017", javax.xml.transform; version\="0.0.0.JavaSE_017", javax.xml.xpath; version\="0.0.0.JavaSE_017", org.w3c.dom.events; version\="0.0.0.JavaSE_017", org.w3c.dom.ranges; version\="0.0.0.JavaSE_017", org.w3c.dom.ls; version\="0.0.0.JavaSE_017", javax.xml.stream.util; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.spec; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.keyinfo; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto; version\="0.0.0.JavaSE_017", com.sun.java.accessibility.util; version\="0.0.0.JavaSE_017", com.sun.tools.attach; version\="0.0.0.JavaSE_017", com.sun.tools.attach.spi; version\="0.0.0.JavaSE_017", com.sun.source.tree; version\="0.0.0.JavaSE_017", com.sun.source.util; version\="0.0.0.JavaSE_017", com.sun.tools.javac; version\="0.0.0.JavaSE_017", com.sun.source.doctree; version\="0.0.0.JavaSE_017", jdk.dynalink; version\="0.0.0.JavaSE_017", jdk.dynalink.linker; version\="0.0.0.JavaSE_017", jdk.dynalink.beans; version\="0.0.0.JavaSE_017", jdk.dynalink.linker.support; version\="0.0.0.JavaSE_017", jdk.dynalink.support; version\="0.0.0.JavaSE_017", com.sun.net.httpserver; version\="0.0.0.JavaSE_017", com.sun.net.httpserver.spi; version\="0.0.0.JavaSE_017", com.sun.jarsigner; version\="0.0.0.JavaSE_017", jdk.security.jarsigner; version\="0.0.0.JavaSE_017", jdk.javadoc.doclet; version\="0.0.0.JavaSE_017", com.sun.tools.jconsole; version\="0.0.0.JavaSE_017", com.sun.jdi.request; version\="0.0.0.JavaSE_017", com.sun.jdi.connect; version\="0.0.0.JavaSE_017", com.sun.jdi.event; version\="0.0.0.JavaSE_017", com.sun.jdi.connect.spi; version\="0.0.0.JavaSE_017", com.sun.jdi; version\="0.0.0.JavaSE_017", jdk.jfr; version\="0.0.0.JavaSE_017", jdk.jfr.consumer; version\="0.0.0.JavaSE_017", jdk.jshell.spi; version\="0.0.0.JavaSE_017", jdk.jshell.tool; version\="0.0.0.JavaSE_017", jdk.jshell; version\="0.0.0.JavaSE_017", jdk.jshell.execution; version\="0.0.0.JavaSE_017", netscape.javascript; version\="0.0.0.JavaSE_017", com.sun.management; version\="0.0.0.JavaSE_017", jdk.management.jfr; version\="0.0.0.JavaSE_017", jdk.net; version\="0.0.0.JavaSE_017", jdk.nio; version\="0.0.0.JavaSE_017", jdk.nio.mapmode; version\="0.0.0.JavaSE_017", com.sun.nio.sctp; version\="0.0.0.JavaSE_017", com.sun.security.auth.callback; version\="0.0.0.JavaSE_017", com.sun.security.auth.module; version\="0.0.0.JavaSE_017", com.sun.security.auth.login; version\="0.0.0.JavaSE_017", com.sun.security.auth; version\="0.0.0.JavaSE_017", com.sun.security.jgss; version\="0.0.0.JavaSE_017", sun.reflect; version\="0.0.0.JavaSE_017", sun.misc; version\="0.0.0.JavaSE_017", com.sun.nio.file; version\="0.0.0.JavaSE_017", jdk.swing.interop; version\="0.0.0.JavaSE_017", org.w3c.dom.xpath; version\="0.0.0.JavaSE_017", org.w3c.dom.css; version\="0.0.0.JavaSE_017", org.w3c.dom.stylesheets; version\="0.0.0.JavaSE_017", org.w3c.dom.html; version\="0.0.0.JavaSE_017", com.yahoo.jdisc, com.yahoo.jdisc.application, com.yahoo.jdisc.handler, com.yahoo.jdisc.service, com.yahoo.jdisc.statistics, com.yahoo.jdisc.refcount, javax.inject;version\=1.0.0, org.aopalliance.intercept, org.aopalliance.aop, com.google.common.annotations;version\="27.1.0",com.google.common.base;version\="27.1.0",com.google.common.cache;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent",com.google.common.collect;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.escape;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.eventbus;version\="27.1.0",com.google.common.graph;version\="27.1.0";uses\:\="com.google.common.collect",com.google.common.hash;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.html;version\="27.1.0";uses\:\="com.google.common.escape",com.google.common.io;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.graph,com.google.common.hash",com.google.common.math;version\="27.1.0",com.google.common.net;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.escape",com.google.common.primitives;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.reflect;version\="27.1.0";uses\:\="com.google.common.collect,com.google.common.io",com.google.common.util.concurrent;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent.internal",com.google.common.xml;version\="27.1.0";uses\:\="com.google.common.escape", com.google.inject;version\="1.4",com.google.inject.binder;version\="1.4",com.google.inject.matcher;version\="1.4",com.google.inject.multibindings;version\="1.4",com.google.inject.name;version\="1.4",com.google.inject.spi;version\="1.4",com.google.inject.util;version\="1.4", org.slf4j;version\=1.7.32, org.slf4j.spi;version\=1.7.32, org.slf4j.helpers;version\=1.7.32, org.slf4j.event;version\=1.7.32, org.slf4j.impl;version\=1.7.32, org.apache.commons.logging;version\=1.2, org.apache.commons.logging.impl;version\=1.2, com.sun.jna;version\=5.11.0, com.sun.jna.ptr;version\=5.11.0, com.sun.jna.win32;version\=5.11.0, org.apache.log4j;version\=1.2.17,org.apache.log4j.helpers;version\=1.2.17,org.apache.log4j.spi;version\=1.2.17,org.apache.log4j.xml;version\=1.2.17, com.yahoo.component.annotation;version\="1.0.0", com.yahoo.config;version\=1.0.0, com.yahoo.vespa.defaults;version\=1.0.0, ai.vespa.http;version\=1.0.0,ai.vespa.llm.completion;version\=1.0.0,ai.vespa.llm.test;version\=1.0.0,ai.vespa.llm;version\=1.0.0,ai.vespa.validation;version\=1.0.0,com.yahoo.binaryprefix;version\=1.0.0,com.yahoo.collections;version\=1.0.0,com.yahoo.compress;version\=1.0.0,com.yahoo.concurrent.classlock;version\=1.0.0,com.yahoo.concurrent.maintenance;version\=1.0.0,com.yahoo.concurrent;version\=1.0.0,com.yahoo.data.access.helpers;version\=1.0.0,com.yahoo.data.access.simple;version\=1.0.0,com.yahoo.data.access.slime;version\=1.0.0,com.yahoo.data.access;version\=1.0.0,com.yahoo.errorhandling;version\=1.0.0,com.yahoo.exception;version\=1.0.0,com.yahoo.geo;version\=1.0.0,com.yahoo.io.reader;version\=1.0.0,com.yahoo.io;version\=1.0.0,com.yahoo.javacc;version\=1.0.0,com.yahoo.lang;version\=1.0.0,com.yahoo.nativec;version\=1.0.0,com.yahoo.net;version\=1.0.0,com.yahoo.path;version\=1.0.0,com.yahoo.protect;version\=1.0.0,com.yahoo.reflection;version\=1.0.0,com.yahoo.slime;version\=1.0.0,com.yahoo.stream;version\=1.0.0,com.yahoo.system.execution;version\=1.0.0,com.yahoo.system;version\=1.0.0,com.yahoo.tensor.evaluation;version\=1.0.0,com.yahoo.tensor.functions;version\=1.0.0,com.yahoo.tensor.serialization;version\=1.0.0,com.yahoo.tensor;version\=1.0.0,com.yahoo.text.internal;version\=1.0.0,com.yahoo.text;version\=1.0.0,com.yahoo.time;version\=1.0.0,com.yahoo.transaction;version\=1.0.0,com.yahoo.vespa.objects;version\=1.0.0,com.yahoo.yolean.chain;version\=1.0.0,com.yahoo.yolean.concurrent;version\=1.0.0,com.yahoo.yolean.function;version\=1.0.0,com.yahoo.yolean.system;version\=1.0.0,com.yahoo.yolean.trace;version\=1.0.0,com.yahoo.yolean;version\=1.0.0, com.yahoo.log.event;version\=1.0.0,com.yahoo.log.impl;version\=1.0.0,com.yahoo.log;version\=1.0.0, javax.xml.bind;version\="2.3";uses\:\="javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.annotation;version\="2.3";uses\:\="javax.xml.bind,javax.xml.parsers,javax.xml.transform,javax.xml.transform.dom,org.w3c.dom",javax.xml.bind.annotation.adapters;version\="2.3",javax.xml.bind.attachment;version\="2.3";uses\:\="javax.activation",javax.xml.bind.helpers;version\="2.3";uses\:\="javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.util;version\="2.3";uses\:\="javax.xml.bind,javax.xml.transform.sax", com.sun.istack;version\="3.0.5";uses\:\="javax.activation,javax.xml.stream,org.xml.sax,org.xml.sax.helpers",com.sun.istack.localization;version\="3.0.5",com.sun.istack.logging;version\="3.0.5",com.sun.xml.bind;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.annotation;version\="2.3.0",com.sun.xml.bind.api;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.api.impl;version\="2.3.0",com.sun.xml.bind.marshaller;uses\:\="javax.xml.parsers,org.w3c.dom,org.xml.sax,org.xml.sax.helpers";version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;version\="2.3.0",com.sun.xml.bind.v2;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.core;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.impl,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.namespace,javax.xml.transform";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.nav";version\="2.3.0",com.sun.xml.bind.v2.model.nav;uses\:\="com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.util;uses\:\="javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.xml.bind.v2.model.annotation,javax.activation,javax.xml.bind,javax.xml.bind.annotation.adapters";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen.episode;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="javax.xml.parsers,javax.xml.transform,javax.xml.validation,javax.xml.xpath";version\="2.3.0",com.sun.xml.txw2;uses\:\="com.sun.xml.txw2.output,javax.xml.namespace";version\="2.3.0",com.sun.xml.txw2.annotation;version\="2.3.0",com.sun.xml.txw2.output;uses\:\="com.sun.xml.txw2,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.dom,javax.xml.transform.sax,javax.xml.transform.stream,org.w3c.dom,org.xml.sax,org.xml.sax.ext,org.xml.sax.helpers";version\="2.3.0", com.sun.xml.bind;uses\:\="com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.datatype,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.api;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.xml.bind,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.marshaller;version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;uses\:\="com.sun.xml.bind,javax.xml.bind.helpers,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,javax.xml.bind";version\="2.3.0",com.sun.xml.bind.v2.bytecode;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.model.runtime;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.namespace,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.istack,com.sun.xml.bind.api,com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.property,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.output;uses\:\="com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.runtime,com.sun.xml.fastinfoset.stax,javax.xml.stream,org.jvnet.staxex,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.property;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,com.sun.xml.bind.v2.runtime.unmarshaller,com.sun.xml.bind.v2.util,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect.opt;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="com.sun.xml.bind,com.sun.xml.bind.api,com.sun.xml.bind.unmarshaller,com.sun.xml.bind.util,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.reflect,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.txw2.output,javax.xml.bind,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.schemagen.xmlschema;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.namespace,javax.xml.transform.stream,org.xml.sax";version\="2.3.0", javax.activation;uses\:\="com.sun.activation.registries";version\="1.2",com.sun.activation.viewers;uses\:\="javax.activation";version\="1.2.0",com.sun.activation.registries;version\="1.2.0" diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index 3ab1fdf211b..d5b48bc2609 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -363,8 +363,13 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { Get the document version to ask for */ private int documentVersion(NodeAgentContext context) { + var version = context.node().currentVespaVersion() + .orElse(context.node().wantedVespaVersion().orElse(Version.emptyVersion)); + var appId = context.node().owner().orElse(ApplicationId.defaultId()); return useNewIdentityDocumentLayout .with(FetchVector.Dimension.HOSTNAME, context.hostname().value()) + .with(FetchVector.Dimension.VESPA_VERSION, version.toFullString()) + .with(FetchVector.Dimension.APPLICATION_ID, appId.serializedForm()) .value() ? SignedIdentityDocument.DEFAULT_DOCUMENT_VERSION : SignedIdentityDocument.LEGACY_DEFAULT_DOCUMENT_VERSION; diff --git a/searchcore/src/tests/proton/server/initialize_threads_calculator/initialize_threads_calculator_test.cpp b/searchcore/src/tests/proton/server/initialize_threads_calculator/initialize_threads_calculator_test.cpp index 7aad69076ca..fc899f3a8c7 100644 --- a/searchcore/src/tests/proton/server/initialize_threads_calculator/initialize_threads_calculator_test.cpp +++ b/searchcore/src/tests/proton/server/initialize_threads_calculator/initialize_threads_calculator_test.cpp @@ -16,7 +16,8 @@ public: void expect_successful_init(uint32_t exp_threads) { - InitializeThreadsCalculator i("tmp", 9); + constexpr uint32_t cfg_threads = 9; + InitializeThreadsCalculator i(HwInfo::Cpu(cfg_threads), "tmp", cfg_threads); EXPECT_EQ(exp_threads, i.num_threads()); EXPECT_TRUE(i.threads().get() != nullptr); EXPECT_EQ(exp_threads, dynamic_cast<const ThreadStackExecutor&>(*i.threads()).getNumThreads()); @@ -27,7 +28,7 @@ expect_successful_init(uint32_t exp_threads) void expect_aborted_init(uint32_t exp_threads, uint32_t cfg_threads = 9) { - InitializeThreadsCalculator i("tmp", cfg_threads); + InitializeThreadsCalculator i(HwInfo::Cpu(cfg_threads), "tmp", cfg_threads); EXPECT_EQ(exp_threads, i.num_threads()); EXPECT_TRUE(i.threads().get() != nullptr); EXPECT_EQ(exp_threads, dynamic_cast<const ThreadStackExecutor&>(*i.threads()).getNumThreads()); @@ -53,7 +54,7 @@ TEST_F(InitializeThreadsCalculatorTest, initialize_threads_cut_in_half_when_init TEST_F(InitializeThreadsCalculatorTest, zero_initialize_threads_is_special) { { - InitializeThreadsCalculator i("tmp", 0); + InitializeThreadsCalculator i(HwInfo::Cpu(10), "tmp", 0); EXPECT_EQ(0, i.num_threads()); EXPECT_TRUE(i.threads().get() == nullptr); } @@ -61,4 +62,20 @@ TEST_F(InitializeThreadsCalculatorTest, zero_initialize_threads_is_special) expect_aborted_init(1, 0); } +void +expect_lower(uint32_t cores, uint32_t configured) { + InitializeThreadsCalculator i(HwInfo::Cpu(cores), "tmp", configured); + EXPECT_EQ(std::min(cores, configured), i.num_threads()); + i.init_done(); +} + +TEST_F(InitializeThreadsCalculatorTest, lower_of_wanted_and_cores) +{ + expect_lower(1, 7); + expect_lower(6, 7); + expect_lower(7, 7); + expect_lower(7, 6); + expect_lower(7, 1); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/vespa/searchcore/proton/server/initialize_threads_calculator.cpp b/searchcore/src/vespa/searchcore/proton/server/initialize_threads_calculator.cpp index 17ddabebb65..bb1325acf04 100644 --- a/searchcore/src/vespa/searchcore/proton/server/initialize_threads_calculator.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/initialize_threads_calculator.cpp @@ -39,10 +39,11 @@ const vespalib::string file_name = "initialize-threads.txt"; namespace proton { -InitializeThreadsCalculator::InitializeThreadsCalculator(const vespalib::string& base_dir, +InitializeThreadsCalculator::InitializeThreadsCalculator(const HwInfo::Cpu & cpu_info, + const vespalib::string& base_dir, uint32_t configured_num_threads) : _path(base_dir + "/" + file_name), - _num_threads(configured_num_threads), + _num_threads(std::min(cpu_info.cores(), configured_num_threads)), _threads() { if (std::filesystem::exists(_path)) { diff --git a/searchcore/src/vespa/searchcore/proton/server/initialize_threads_calculator.h b/searchcore/src/vespa/searchcore/proton/server/initialize_threads_calculator.h index 509db24ebf3..e7e1b22f181 100644 --- a/searchcore/src/vespa/searchcore/proton/server/initialize_threads_calculator.h +++ b/searchcore/src/vespa/searchcore/proton/server/initialize_threads_calculator.h @@ -2,6 +2,7 @@ #pragma once +#include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/threadexecutor.h> #include <filesystem> @@ -25,8 +26,7 @@ namespace proton { InitializeThreads _threads; public: - InitializeThreadsCalculator(const vespalib::string& base_dir, - uint32_t configured_num_threads); + InitializeThreadsCalculator(const HwInfo::Cpu & cpu_info, const vespalib::string& base_dir, uint32_t configured_num_threads); ~InitializeThreadsCalculator(); uint32_t num_threads() const { return _num_threads; } InitializeThreads threads() const { return _threads; } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 8ec904760cf..7f3dc02aba8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -352,7 +352,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) vespalib::string fileConfigId; _compile_cache_executor_binding = vespalib::eval::CompileCache::bind(_shared_service->shared_raw()); - InitializeThreadsCalculator calc(protonConfig.basedir, protonConfig.initialize.threads); + InitializeThreadsCalculator calc(hwInfo.cpu(), protonConfig.basedir, protonConfig.initialize.threads); LOG(info, "Start initializing components: threads=%u, configured=%u", calc.num_threads(), protonConfig.initialize.threads); _initDocumentDbsInSequence = (calc.num_threads() == 1); diff --git a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp index fd07529795a..b9599a0c75d 100644 --- a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp +++ b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp @@ -29,7 +29,6 @@ using search::attribute::DistanceMetric; using search::feature_t; using search::tensor::DenseTensorAttribute; using search::tensor::DistanceCalculator; -using search::tensor::DistanceFunction; using search::tensor::SerializedFastValueAttribute; using search::tensor::TensorAttribute; using vespalib::eval::CellType; @@ -49,9 +48,6 @@ vespalib::string denseSpecDouble("tensor(x[2])"); vespalib::string denseSpecFloat("tensor<float>(x[2])"); vespalib::string mixed_spec("tensor(m{},x[2])"); -DistanceFunction::UP euclid_d = search::tensor::make_distance_function(DistanceMetric::Euclidean, CellType::DOUBLE); -DistanceFunction::UP euclid_f = search::tensor::make_distance_function(DistanceMetric::Euclidean, CellType::FLOAT); - std::unique_ptr<Value> createTensor(const TensorSpec &spec) { return SimpleValue::from_spec(spec); } @@ -119,14 +115,6 @@ struct Fixture { auto t = createTensor(_typeSpec, v1, v2); setTensor(docId, *t); } - - const DistanceFunction &dist_fun() const { - if (_cfg.tensorType().cell_type() == CellType::FLOAT) { - return *euclid_f; - } else { - return *euclid_d; - } - } }; template <bool strict> @@ -136,9 +124,11 @@ SimpleResult find_matches(Fixture &env, const Value &qtv, double threshold = std auto &attr = *(env._attr); auto dff = search::tensor::make_distance_function_factory(DistanceMetric::Euclidean, qtv.cells().type); - DistanceCalculator dist_calc(attr, dff->for_query_vector(qtv.cells())); + auto df = dff->for_query_vector(qtv.cells()); + threshold = df->convert_threshold(threshold); + DistanceCalculator dist_calc(attr, std::move(df)); NearestNeighborDistanceHeap dh(2); - dh.set_distance_threshold(env.dist_fun().convert_threshold(threshold)); + dh.set_distance_threshold(threshold); const GlobalFilter &filter = *env._global_filter; auto search = NearestNeighborIterator::create(strict, tfmd, dist_calc, dh, filter); if (strict) { diff --git a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp index 9b8ad0d26ce..a1b29c90986 100644 --- a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp +++ b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp @@ -76,10 +76,6 @@ namespace { const double sq_root_half = std::sqrt(0.5); } TEST(DistanceFunctionsTest, euclidean_gives_expected_score) { - auto ct = vespalib::eval::CellType::DOUBLE; - - auto euclid = make_distance_function(DistanceMetric::Euclidean, ct); - std::vector<double> p0{0.0, 0.0, 0.0}; std::vector<double> p1{1.0, 0.0, 0.0}; std::vector<double> p2{0.0, 1.0, 0.0}; @@ -92,6 +88,9 @@ TEST(DistanceFunctionsTest, euclidean_gives_expected_score) EXPECT_FLOAT_EQ(n4, 1.0); double d12 = computeEuclideanChecked(t(p1), t(p2)); EXPECT_EQ(d12, 2.0); + + EuclideanDistanceFunctionFactory<double> dff; + auto euclid = dff.for_query_vector(t(p0)); EXPECT_DOUBLE_EQ(euclid->to_rawscore(d12), 1.0/(1.0 + sqrt(2.0))); double threshold = euclid->convert_threshold(8.0); EXPECT_EQ(threshold, 64.0); @@ -142,10 +141,6 @@ TEST(DistanceFunctionsTest, euclidean_gives_expected_score) TEST(DistanceFunctionsTest, euclidean_int8_smoketest) { - auto ct = vespalib::eval::CellType::INT8; - - auto euclid = make_distance_function(DistanceMetric::Euclidean, ct); - std::vector<Int8Float> p0{0.0, 0.0, 0.0}; std::vector<Int8Float> p1{1.0, 0.0, 0.0}; std::vector<Int8Float> p5{0.0,-1.0, 0.0}; @@ -345,60 +340,10 @@ TEST(DistanceFunctionsTest, prenormalized_angular_gives_expected_score) EXPECT_DOUBLE_EQ(threshold, 1.0); } -TEST(DistanceFunctionsTest, innerproduct_gives_expected_score) -{ - auto ct = vespalib::eval::CellType::DOUBLE; - - auto innerproduct = make_distance_function(DistanceMetric::InnerProduct, ct); - - std::vector<double> p0{0.0, 0.0, 0.0}; - std::vector<double> p1{1.0, 0.0, 0.0}; - std::vector<double> p2{0.0, 1.0, 0.0}; - std::vector<double> p3{0.0, 0.0, 1.0}; - std::vector<double> p4{0.5, 0.5, sq_root_half}; - std::vector<double> p5{0.0,-1.0, 0.0}; - std::vector<double> p6{1.0, 2.0, 2.0}; - - double i12 = innerproduct->calc(t(p1), t(p2)); - double i13 = innerproduct->calc(t(p1), t(p3)); - double i23 = innerproduct->calc(t(p2), t(p3)); - EXPECT_DOUBLE_EQ(i12, 1.0); - EXPECT_DOUBLE_EQ(i13, 1.0); - EXPECT_DOUBLE_EQ(i23, 1.0); - - double i14 = innerproduct->calc(t(p1), t(p4)); - double i24 = innerproduct->calc(t(p2), t(p4)); - EXPECT_DOUBLE_EQ(i14, 0.5); - EXPECT_DOUBLE_EQ(i24, 0.5); - double i34 = innerproduct->calc(t(p3), t(p4)); - EXPECT_FLOAT_EQ(i34, 1.0 - sq_root_half); - - double i25 = innerproduct->calc(t(p2), t(p5)); - EXPECT_DOUBLE_EQ(i25, 2.0); - - double i44 = innerproduct->calc(t(p4), t(p4)); - EXPECT_GE(i44, 0.0); - EXPECT_LT(i44, 0.000001); - - double i66 = innerproduct->calc(t(p6), t(p6)); - EXPECT_GE(i66, 0.0); - EXPECT_LT(i66, 0.000001); - - double threshold = innerproduct->convert_threshold(0.25); - EXPECT_DOUBLE_EQ(threshold, 0.25); - threshold = innerproduct->convert_threshold(0.5); - EXPECT_DOUBLE_EQ(threshold, 0.5); - threshold = innerproduct->convert_threshold(1.0); - EXPECT_DOUBLE_EQ(threshold, 1.0); -} TEST(DistanceFunctionsTest, hamming_gives_expected_score) { - static HammingDistanceFunctionFactory<Int8Float> dff; - auto ct = vespalib::eval::CellType::DOUBLE; - - auto hamming = make_distance_function(DistanceMetric::Hamming, ct); - + static HammingDistanceFunctionFactory<double> dff; std::vector<std::vector<double>> points{{0.0, 0.0, 0.0}, {1.0, 0.0, 0.0}, @@ -407,31 +352,30 @@ TEST(DistanceFunctionsTest, hamming_gives_expected_score) {0.5, 0.5, 0.5}, {0.0,-1.0, 1.0}, {1.0, 1.0, 1.0}}; + auto hamming = dff.for_query_vector(t(points[0])); for (const auto & p : points) { - double h0 = hamming->calc(t(p), t(p)); - EXPECT_EQ(h0, 0.0); - EXPECT_EQ(hamming->to_rawscore(h0), 1.0); auto dist_fun = dff.for_query_vector(t(p)); - EXPECT_EQ(dist_fun->calc(t(p)), 0.0); + double h0 = dist_fun->calc(t(p)); + EXPECT_EQ(h0, 0.0); EXPECT_EQ(dist_fun->to_rawscore(h0), 1.0); } - double d12 = hamming->calc(t(points[1]), t(points[2])); + double d12 = dff.for_query_vector(t(points[1]))->calc(t(points[2])); EXPECT_EQ(d12, 3.0); EXPECT_DOUBLE_EQ(hamming->to_rawscore(d12), 1.0/(1.0 + 3.0)); - double d16 = hamming->calc(t(points[1]), t(points[6])); + double d16 = dff.for_query_vector(t(points[1]))->calc(t(points[6])); EXPECT_EQ(d16, 2.0); EXPECT_DOUBLE_EQ(hamming->to_rawscore(d16), 1.0/(1.0 + 2.0)); - double d23 = hamming->calc(t(points[2]), t(points[3])); + double d23 = dff.for_query_vector(t(points[2]))->calc(t(points[3])); EXPECT_EQ(d23, 3.0); EXPECT_DOUBLE_EQ(hamming->to_rawscore(d23), 1.0/(1.0 + 3.0)); - double d24 = hamming->calc(t(points[2]), t(points[4])); + double d24 = dff.for_query_vector(t(points[2]))->calc(t(points[4])); EXPECT_EQ(d24, 3.0); EXPECT_DOUBLE_EQ(hamming->to_rawscore(d24), 1.0/(1.0 + 3.0)); - double d25 = hamming->calc(t(points[2]), t(points[5])); + double d25 = dff.for_query_vector(t(points[2]))->calc(t(points[5])); EXPECT_EQ(d25, 1.0); EXPECT_DOUBLE_EQ(hamming->to_rawscore(d25), 1.0/(1.0 + 1.0)); @@ -445,8 +389,8 @@ TEST(DistanceFunctionsTest, hamming_gives_expected_score) std::vector<Int8Float> bytes_a = { 0, 1, 2, 4, 8, 16, 32, 64, -128, 0, 1, 2, 4, 8, 16, 32, 64, -128, 0, 1, 2 }; std::vector<Int8Float> bytes_b = { 1, 2, 2, 4, 8, 16, 32, 65, -128, 0, 1, 0, 4, 8, 16, 32, 64, -128, 0, 1, -1 }; // expect diff: 1 2 1 1 7 - EXPECT_EQ(hamming->calc(TypedCells(bytes_a), TypedCells(bytes_b)), 12.0); - auto dist_fun = dff.for_query_vector(TypedCells(bytes_a)); + HammingDistanceFunctionFactory<Int8Float> factory_int8; + auto dist_fun = factory_int8.for_query_vector(TypedCells(bytes_a)); EXPECT_EQ(dist_fun->calc(TypedCells(bytes_b)), 12.0); } diff --git a/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp b/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp index 0dcd77ec392..b2e5f8863c3 100644 --- a/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp @@ -133,8 +133,6 @@ public: } }; -using FloatSqEuclideanDistance = SquaredEuclideanDistanceHW<float>; - template <typename IndexType> class Stressor : public ::testing::Test { private: diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index 2e874ffa4ae..362bceaf0da 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -25,7 +25,6 @@ vespa_add_library(searchlib_tensor OBJECT hnsw_single_best_neighbors.cpp imported_tensor_attribute_vector.cpp imported_tensor_attribute_vector_read_guard.cpp - inner_product_distance.cpp inv_log_level_generator.cpp large_subspaces_buffer_type.cpp nearest_neighbor_index.cpp diff --git a/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp b/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp index a7ae02bb9f4..ab8f2b30df9 100644 --- a/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/angular_distance.cpp @@ -41,17 +41,6 @@ struct CalcAngular { } -double -AngularDistance::calc(const vespalib::eval::TypedCells& lhs, - const vespalib::eval::TypedCells& rhs) const -{ - return typify_invoke<2,TypifyCellType,CalcAngular>(lhs.type, rhs.type, lhs, rhs); -} - -template class AngularDistanceHW<float>; -template class AngularDistanceHW<double>; - - template<typename FloatType> class BoundAngularDistance : public BoundDistanceFunction { private: diff --git a/searchlib/src/vespa/searchlib/tensor/angular_distance.h b/searchlib/src/vespa/searchlib/tensor/angular_distance.h index caed5d8c216..50d3e617cf4 100644 --- a/searchlib/src/vespa/searchlib/tensor/angular_distance.h +++ b/searchlib/src/vespa/searchlib/tensor/angular_distance.h @@ -13,69 +13,10 @@ namespace search::tensor { /** * Calculates angular distance between vectors - */ -class AngularDistance : public DistanceFunction { -public: - AngularDistance(vespalib::eval::CellType expected) : DistanceFunction(expected) {} - double calc(const vespalib::eval::TypedCells& lhs, const vespalib::eval::TypedCells& rhs) const override; - double convert_threshold(double threshold) const override { - double cosine_similarity = cos(threshold); - return 1.0 - cosine_similarity; - } - double to_rawscore(double distance) const override { - double cosine_similarity = 1.0 - distance; - // should be in the range [-1,1] but roundoff may cause problems: - cosine_similarity = std::min(1.0, cosine_similarity); - cosine_similarity = std::max(-1.0, cosine_similarity); - double angle_distance = acos(cosine_similarity); // in range [0,pi] - double score = 1.0 / (1.0 + angle_distance); - return score; - } - double calc_with_limit(const vespalib::eval::TypedCells& lhs, - const vespalib::eval::TypedCells& rhs, - double /*limit*/) const override - { - return calc(lhs, rhs); - } -}; - -/** - * Calculates angular distance between vectors * Will use instruction optimal for the cpu it is running on - * when both vectors have the expected cell type. + * after converting both vectors to an optimal cell type. */ template <typename FloatType> -class AngularDistanceHW : public AngularDistance { -public: - AngularDistanceHW() - : AngularDistance(vespalib::eval::get_cell_type<FloatType>()), - _computer(vespalib::hwaccelrated::IAccelrated::getAccelerator()) - { - assert(expected_cell_type() == vespalib::eval::get_cell_type<FloatType>()); - } - double calc(const vespalib::eval::TypedCells& lhs, const vespalib::eval::TypedCells& rhs) const override { - constexpr vespalib::eval::CellType expected = vespalib::eval::get_cell_type<FloatType>(); - assert(lhs.type == expected && rhs.type == expected); - auto lhs_vector = lhs.typify<FloatType>(); - auto rhs_vector = rhs.typify<FloatType>(); - size_t sz = lhs_vector.size(); - assert(sz == rhs_vector.size()); - auto a = lhs_vector.data(); - auto b = rhs_vector.data(); - double a_norm_sq = _computer.dotProduct(a, a, sz); - double b_norm_sq = _computer.dotProduct(b, b, sz); - double squared_norms = a_norm_sq * b_norm_sq; - double dot_product = _computer.dotProduct(a, b, sz); - double div = (squared_norms > 0) ? sqrt(squared_norms) : 1.0; - double cosine_similarity = dot_product / div; - double distance = 1.0 - cosine_similarity; // in range [0,2] - return distance; - } -private: - const vespalib::hwaccelrated::IAccelrated & _computer; -}; - -template <typename FloatType> class AngularDistanceFunctionFactory : public DistanceFunctionFactory { public: AngularDistanceFunctionFactory() = default; diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function.h b/searchlib/src/vespa/searchlib/tensor/distance_function.h index 443191a272c..31a837d17f4 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_function.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_function.h @@ -20,34 +20,4 @@ public: virtual double to_rawscore(double distance) const = 0; }; -/** - * Interface used to calculate the distance between two n-dimensional vectors. - * - * The vectors must be of same size and same cell type (float or double). - * The actual implementation must know which type the vectors are. - */ -class DistanceFunction : public DistanceConverter { -private: - vespalib::eval::CellType _expect_cell_type; -public: - using UP = std::unique_ptr<DistanceFunction>; - - DistanceFunction(vespalib::eval::CellType expected) : _expect_cell_type(expected) {} - - virtual ~DistanceFunction() = default; - - // input (query) vectors must be converted to this cell type: - vespalib::eval::CellType expected_cell_type() const { - return _expect_cell_type; - } - - // calculate internal distance (comparable) - virtual double calc(const vespalib::eval::TypedCells& lhs, const vespalib::eval::TypedCells& rhs) const = 0; - - // calculate internal distance, early return allowed if > limit - virtual double calc_with_limit(const vespalib::eval::TypedCells& lhs, - const vespalib::eval::TypedCells& rhs, - double limit) const = 0; -}; - } diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp index 3a4bd94de23..bec0f430a51 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp +++ b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp @@ -15,79 +15,6 @@ using vespalib::eval::ValueType; namespace search::tensor { -DistanceFunction::UP -make_distance_function(DistanceMetric variant, CellType cell_type) -{ - switch (variant) { - case DistanceMetric::Euclidean: - switch (cell_type) { - case CellType::FLOAT: return std::make_unique<SquaredEuclideanDistanceHW<float>>(); - case CellType::DOUBLE: return std::make_unique<SquaredEuclideanDistanceHW<double>>(); - case CellType::INT8: return std::make_unique<SquaredEuclideanDistanceHW<vespalib::eval::Int8Float>>(); - default: return std::make_unique<SquaredEuclideanDistance>(CellType::FLOAT); - } - case DistanceMetric::Angular: - switch (cell_type) { - case CellType::FLOAT: return std::make_unique<AngularDistanceHW<float>>(); - case CellType::DOUBLE: return std::make_unique<AngularDistanceHW<double>>(); - default: return std::make_unique<AngularDistance>(CellType::FLOAT); - } - case DistanceMetric::GeoDegrees: - return std::make_unique<GeoDegreesDistance>(CellType::DOUBLE); - case DistanceMetric::PrenormalizedAngular: - case DistanceMetric::InnerProduct: - switch (cell_type) { - case CellType::FLOAT: return std::make_unique<InnerProductDistanceHW<float>>(); - case CellType::DOUBLE: return std::make_unique<InnerProductDistanceHW<double>>(); - default: return std::make_unique<InnerProductDistance>(CellType::FLOAT); - } - case DistanceMetric::Hamming: - return std::make_unique<HammingDistance>(cell_type); - } - // not reached: - return DistanceFunction::UP(); -} - - -class SimpleBoundDistanceFunction : public BoundDistanceFunction { - const vespalib::eval::TypedCells _lhs; - const DistanceFunction &_df; -public: - SimpleBoundDistanceFunction(const vespalib::eval::TypedCells& lhs, - const DistanceFunction &df) - : _lhs(lhs), - _df(df) - {} - - double calc(const vespalib::eval::TypedCells& rhs) const override { - return _df.calc(_lhs, rhs); - } - double convert_threshold(double threshold) const override { - return _df.convert_threshold(threshold); - } - double to_rawscore(double distance) const override { - return _df.to_rawscore(distance); - } - double calc_with_limit(const vespalib::eval::TypedCells& rhs, double limit) const override { - return _df.calc_with_limit(_lhs, rhs, limit); - } -}; - -class SimpleDistanceFunctionFactory : public DistanceFunctionFactory { - DistanceFunction::UP _df; -public: - SimpleDistanceFunctionFactory(DistanceFunction::UP df) - : _df(std::move(df)) - {} - - BoundDistanceFunction::UP for_query_vector(const vespalib::eval::TypedCells& lhs) override { - return std::make_unique<SimpleBoundDistanceFunction>(lhs, *_df); - } - BoundDistanceFunction::UP for_insertion_vector(const vespalib::eval::TypedCells& lhs) override { - return std::make_unique<SimpleBoundDistanceFunction>(lhs, *_df); - } -}; - std::unique_ptr<DistanceFunctionFactory> make_distance_function_factory(search::attribute::DistanceMetric variant, vespalib::eval::CellType cell_type) diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.h b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.h index 433c8e1962a..a69c7cff739 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.h @@ -23,13 +23,9 @@ struct DistanceFunctionFactory { }; /** - * Create a distance function object customized for the given metric + * Create a distance function factory customized for the given metric * variant and (attribute) cell type. **/ -DistanceFunction::UP -make_distance_function(search::attribute::DistanceMetric variant, - vespalib::eval::CellType cell_type); - DistanceFunctionFactory::UP make_distance_function_factory(search::attribute::DistanceMetric variant, vespalib::eval::CellType cell_type); diff --git a/searchlib/src/vespa/searchlib/tensor/distance_functions.h b/searchlib/src/vespa/searchlib/tensor/distance_functions.h index 2300dba2db1..ec401b9313d 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_functions.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_functions.h @@ -7,5 +7,4 @@ #include "euclidean_distance.h" #include "geo_degrees_distance.h" #include "hamming_distance.h" -#include "inner_product_distance.h" #include "prenormalized_angular_distance.h" diff --git a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp index a98b37cb6cc..1b8924ab351 100644 --- a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.cpp @@ -30,25 +30,6 @@ struct CalcEuclidean { } -double -SquaredEuclideanDistance::calc(const vespalib::eval::TypedCells& lhs, - const vespalib::eval::TypedCells& rhs) const -{ - return typify_invoke<2,TypifyCellType,CalcEuclidean>(lhs.type, rhs.type, lhs, rhs); -} - -double -SquaredEuclideanDistance::calc_with_limit(const vespalib::eval::TypedCells& lhs, - const vespalib::eval::TypedCells& rhs, - double) const -{ - // maybe optimize this: - return typify_invoke<2,TypifyCellType,CalcEuclidean>(lhs.type, rhs.type, lhs, rhs); -} - -template class SquaredEuclideanDistanceHW<float>; -template class SquaredEuclideanDistanceHW<double>; - using vespalib::eval::Int8Float; using vespalib::BFloat16; diff --git a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.h b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.h index 06295caee9e..fc663c50ccd 100644 --- a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.h +++ b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.h @@ -12,75 +12,10 @@ namespace search::tensor { /** * Calculates the square of the standard Euclidean distance. - */ -class SquaredEuclideanDistance : public DistanceFunction { -public: - SquaredEuclideanDistance(vespalib::eval::CellType expected) : DistanceFunction(expected) {} - double calc(const vespalib::eval::TypedCells& lhs, const vespalib::eval::TypedCells& rhs) const override; - double calc_with_limit(const vespalib::eval::TypedCells& lhs, - const vespalib::eval::TypedCells& rhs, - double limit) const override; - double convert_threshold(double threshold) const override { - return threshold*threshold; - } - double to_rawscore(double distance) const override { - double d = sqrt(distance); - double score = 1.0 / (1.0 + d); - return score; - } -}; - -/** - * Calculates the square of the standard Euclidean distance. * Will use instruction optimal for the cpu it is running on - * when both vectors have the expected cell type. + * after converting both vectors to an optimal cell type. */ template <typename FloatType> -class SquaredEuclideanDistanceHW : public SquaredEuclideanDistance { -public: - SquaredEuclideanDistanceHW() - : SquaredEuclideanDistance(vespalib::eval::get_cell_type<FloatType>()), - _computer(vespalib::hwaccelrated::IAccelrated::getAccelerator()) - { - assert(expected_cell_type() == vespalib::eval::get_cell_type<FloatType>()); - } - - static const double *cast(const double * p) { return p; } - static const float *cast(const float * p) { return p; } - static const int8_t *cast(const vespalib::eval::Int8Float * p) { return reinterpret_cast<const int8_t *>(p); } - double calc(const vespalib::eval::TypedCells& lhs, const vespalib::eval::TypedCells& rhs) const override { - constexpr vespalib::eval::CellType expected = vespalib::eval::get_cell_type<FloatType>(); - assert(lhs.type == expected && rhs.type == expected); - auto lhs_vector = lhs.typify<FloatType>(); - auto rhs_vector = rhs.typify<FloatType>(); - size_t sz = lhs_vector.size(); - assert(sz == rhs_vector.size()); - return _computer.squaredEuclideanDistance(cast(&lhs_vector[0]), cast(&rhs_vector[0]), sz); - } - - double calc_with_limit(const vespalib::eval::TypedCells& lhs, - const vespalib::eval::TypedCells& rhs, - double limit) const override - { - constexpr vespalib::eval::CellType expected = vespalib::eval::get_cell_type<FloatType>(); - assert(lhs.type == expected && rhs.type == expected); - auto lhs_vector = lhs.typify<FloatType>(); - auto rhs_vector = rhs.typify<FloatType>(); - double sum = 0.0; - size_t sz = lhs_vector.size(); - assert(sz == rhs_vector.size()); - for (size_t i = 0; i < sz && sum <= limit; ++i) { - double diff = lhs_vector[i] - rhs_vector[i]; - sum += diff*diff; - } - return sum; - } -private: - const vespalib::hwaccelrated::IAccelrated & _computer; -}; - - -template <typename FloatType> class EuclideanDistanceFunctionFactory : public DistanceFunctionFactory { public: EuclideanDistanceFunctionFactory() = default; diff --git a/searchlib/src/vespa/searchlib/tensor/geo_degrees_distance.cpp b/searchlib/src/vespa/searchlib/tensor/geo_degrees_distance.cpp index 38ba8205c90..0212830efb6 100644 --- a/searchlib/src/vespa/searchlib/tensor/geo_degrees_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/geo_degrees_distance.cpp @@ -8,54 +8,28 @@ using vespalib::eval::TypifyCellType; namespace search::tensor { -namespace { - -struct CalcGeoDegrees { - template <typename LCT, typename RCT> - static double invoke(const vespalib::eval::TypedCells& lhs, - const vespalib::eval::TypedCells& rhs) - { - auto lhs_vector = lhs.unsafe_typify<LCT>(); - auto rhs_vector = rhs.unsafe_typify<RCT>(); - - assert(2 == lhs_vector.size()); - assert(2 == rhs_vector.size()); - // convert to radians: - double lat_A = lhs_vector[0] * GeoDegreesDistance::degrees_to_radians; - double lat_B = rhs_vector[0] * GeoDegreesDistance::degrees_to_radians; - double lon_A = lhs_vector[1] * GeoDegreesDistance::degrees_to_radians; - double lon_B = rhs_vector[1] * GeoDegreesDistance::degrees_to_radians; - - double lat_diff = lat_A - lat_B; - double lon_diff = lon_A - lon_B; - - // haversines of differences: - double hav_lat = GeoDegreesDistance::hav(lat_diff); - double hav_lon = GeoDegreesDistance::hav(lon_diff); - - // haversine of central angle between the two points: - double hav_central_angle = hav_lat + cos(lat_A)*cos(lat_B)*hav_lon; - return hav_central_angle; - } -}; - -} - -double -GeoDegreesDistance::calc(const vespalib::eval::TypedCells& lhs, - const vespalib::eval::TypedCells& rhs) const -{ - return typify_invoke<2,TypifyCellType,CalcGeoDegrees>(lhs.type, rhs.type, lhs, rhs); -} - -using vespalib::eval::TypedCells; - +/** + * Calculates great-circle distance between Latitude/Longitude pairs, + * measured in degrees. Output distance is measured in kilometers. + * Uses the haversine formula directly from: + * https://en.wikipedia.org/wiki/Haversine_formula + **/ class BoundGeoDistance : public BoundDistanceFunction { private: mutable TemporaryVectorStore<double> _tmpSpace; const vespalib::ConstArrayRef<double> _lh_vector; - static GeoDegreesDistance _g_d_helper; public: + // in km, as defined by IUGG, see: + // https://en.wikipedia.org/wiki/Earth_radius#Mean_radius + static constexpr double earth_mean_radius = 6371.0088; + static constexpr double degrees_to_radians = M_PI / 180.0; + + // haversine function: + static double haversine(double angle) { + double s = sin(0.5*angle); + return s*s; + } + BoundGeoDistance(const vespalib::eval::TypedCells& lhs) : _tmpSpace(lhs.size), _lh_vector(_tmpSpace.storeLhs(lhs)) @@ -65,27 +39,33 @@ public: assert(2 == _lh_vector.size()); assert(2 == rhs_vector.size()); // convert to radians: - double lat_A = _lh_vector[0] * GeoDegreesDistance::degrees_to_radians; - double lat_B = rhs_vector[0] * GeoDegreesDistance::degrees_to_radians; - double lon_A = _lh_vector[1] * GeoDegreesDistance::degrees_to_radians; - double lon_B = rhs_vector[1] * GeoDegreesDistance::degrees_to_radians; + double lat_A = _lh_vector[0] * degrees_to_radians; + double lat_B = rhs_vector[0] * degrees_to_radians; + double lon_A = _lh_vector[1] * degrees_to_radians; + double lon_B = rhs_vector[1] * degrees_to_radians; double lat_diff = lat_A - lat_B; double lon_diff = lon_A - lon_B; // haversines of differences: - double hav_lat = GeoDegreesDistance::hav(lat_diff); - double hav_lon = GeoDegreesDistance::hav(lon_diff); + double hav_lat = haversine(lat_diff); + double hav_lon = haversine(lon_diff); // haversine of central angle between the two points: double hav_central_angle = hav_lat + cos(lat_A)*cos(lat_B)*hav_lon; return hav_central_angle; } double convert_threshold(double threshold) const override { - return _g_d_helper.convert_threshold(threshold); + double half_angle = threshold / (2 * earth_mean_radius); + double rt_hav = sin(half_angle); + return rt_hav * rt_hav; } double to_rawscore(double distance) const override { - return _g_d_helper.to_rawscore(distance); + double hav_diff = sqrt(distance); + // distance in kilometers: + double d = 2 * asin(hav_diff) * earth_mean_radius; + // km to rawscore: + return 1.0 / (1.0 + d); } double calc_with_limit(const vespalib::eval::TypedCells& rhs, double) const override { return calc(rhs); diff --git a/searchlib/src/vespa/searchlib/tensor/geo_degrees_distance.h b/searchlib/src/vespa/searchlib/tensor/geo_degrees_distance.h index 56f75f6daed..801ee56aacf 100644 --- a/searchlib/src/vespa/searchlib/tensor/geo_degrees_distance.h +++ b/searchlib/src/vespa/searchlib/tensor/geo_degrees_distance.h @@ -13,44 +13,9 @@ namespace search::tensor { /** * Calculates great-circle distance between Latitude/Longitude pairs, - * measured in degrees. Output distance is measured in meters. - * Uses the haversine formula directly from: - * https://en.wikipedia.org/wiki/Haversine_formula + * where input is given as degrees. + * Output distance is measured in kilometers. **/ -class GeoDegreesDistance : public DistanceFunction { -public: - // in km, as defined by IUGG, see: - // https://en.wikipedia.org/wiki/Earth_radius#Mean_radius - static constexpr double earth_mean_radius = 6371.0088; - static constexpr double degrees_to_radians = M_PI / 180.0; - - GeoDegreesDistance(vespalib::eval::CellType expected) : DistanceFunction(expected) {} - // haversine function: - static double hav(double angle) { - double s = sin(0.5*angle); - return s*s; - } - double calc(const vespalib::eval::TypedCells& lhs, const vespalib::eval::TypedCells& rhs) const override; - double convert_threshold(double threshold) const override { - double half_angle = threshold / (2 * earth_mean_radius); - double rt_hav = sin(half_angle); - return rt_hav * rt_hav; - } - double to_rawscore(double distance) const override { - double hav_diff = sqrt(distance); - // distance in kilometers: - double d = 2 * asin(hav_diff) * earth_mean_radius; - // km to rawscore: - return 1.0 / (1.0 + d); - } - double calc_with_limit(const vespalib::eval::TypedCells& lhs, - const vespalib::eval::TypedCells& rhs, - double /*limit*/) const override - { - return calc(lhs, rhs); - } -}; - class GeoDistanceFunctionFactory : public DistanceFunctionFactory { public: GeoDistanceFunctionFactory() = default; diff --git a/searchlib/src/vespa/searchlib/tensor/hamming_distance.cpp b/searchlib/src/vespa/searchlib/tensor/hamming_distance.cpp index f4f6842715f..5b6528ef403 100644 --- a/searchlib/src/vespa/searchlib/tensor/hamming_distance.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hamming_distance.cpp @@ -30,29 +30,6 @@ struct CalcHamming { } -double -HammingDistance::calc(const vespalib::eval::TypedCells& lhs, - const vespalib::eval::TypedCells& rhs) const -{ - constexpr auto expected = vespalib::eval::CellType::INT8; - if (__builtin_expect((lhs.type == expected && rhs.type == expected), true)) { - size_t sz = lhs.size; - assert(sz == rhs.size); - return (double) vespalib::binary_hamming_distance(lhs.data, rhs.data, sz); - } else { - return typify_invoke<2,TypifyCellType,CalcHamming>(lhs.type, rhs.type, lhs, rhs); - } -} - -double -HammingDistance::calc_with_limit(const vespalib::eval::TypedCells& lhs, - const vespalib::eval::TypedCells& rhs, - double) const -{ - // consider optimizing: - return calc(lhs, rhs); -} - using vespalib::eval::Int8Float; template<typename FloatType> diff --git a/searchlib/src/vespa/searchlib/tensor/hamming_distance.h b/searchlib/src/vespa/searchlib/tensor/hamming_distance.h index 3022e0da503..47b022afc13 100644 --- a/searchlib/src/vespa/searchlib/tensor/hamming_distance.h +++ b/searchlib/src/vespa/searchlib/tensor/hamming_distance.h @@ -16,20 +16,6 @@ namespace search::tensor { * or (for int8 cells, aka binary data only) * "number of bits that are different" */ -class HammingDistance final : public DistanceFunction { -public: - HammingDistance(vespalib::eval::CellType expected) : DistanceFunction(expected) {} - double calc(const vespalib::eval::TypedCells& lhs, const vespalib::eval::TypedCells& rhs) const override; - double convert_threshold(double threshold) const override { - return threshold; - } - double to_rawscore(double distance) const override { - double score = 1.0 / (1.0 + distance); - return score; - } - double calc_with_limit(const vespalib::eval::TypedCells& lhs, const vespalib::eval::TypedCells& rhs, double) const override; -}; - template <typename FloatType> class HammingDistanceFunctionFactory : public DistanceFunctionFactory { public: diff --git a/searchlib/src/vespa/searchlib/tensor/inner_product_distance.cpp b/searchlib/src/vespa/searchlib/tensor/inner_product_distance.cpp deleted file mode 100644 index 2e8c970ba91..00000000000 --- a/searchlib/src/vespa/searchlib/tensor/inner_product_distance.cpp +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "inner_product_distance.h" - -using vespalib::typify_invoke; -using vespalib::eval::TypifyCellType; - -namespace search::tensor { - -namespace { - -struct CalcInnerProduct { - template <typename LCT, typename RCT> - static double invoke(const vespalib::eval::TypedCells& lhs, - const vespalib::eval::TypedCells& rhs) - { - auto lhs_vector = lhs.unsafe_typify<LCT>(); - auto rhs_vector = rhs.unsafe_typify<RCT>(); - size_t sz = lhs_vector.size(); - assert(sz == rhs_vector.size()); - double dot_product = 0.0; - for (size_t i = 0; i < sz; ++i) { - double a = lhs_vector[i]; - double b = rhs_vector[i]; - dot_product += a*b; - } - double score = 1.0 - dot_product; // in range [0,2] - return std::max(0.0, score); - } -}; - -} - -double -InnerProductDistance::calc(const vespalib::eval::TypedCells& lhs, - const vespalib::eval::TypedCells& rhs) const -{ - return typify_invoke<2,TypifyCellType,CalcInnerProduct>(lhs.type, rhs.type, lhs, rhs); -} - -template class InnerProductDistanceHW<float>; -template class InnerProductDistanceHW<double>; - -} diff --git a/searchlib/src/vespa/searchlib/tensor/inner_product_distance.h b/searchlib/src/vespa/searchlib/tensor/inner_product_distance.h deleted file mode 100644 index 135bb186fd4..00000000000 --- a/searchlib/src/vespa/searchlib/tensor/inner_product_distance.h +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "distance_function.h" -#include <vespa/eval/eval/typed_cells.h> -#include <vespa/vespalib/hwaccelrated/iaccelrated.h> -#include <cmath> - -namespace search::tensor { - -/** - * Calculates inner-product "distance" between vectors with assumed norm 1. - * Should give same ordering as Angular distance, but is less expensive. - */ -class InnerProductDistance : public DistanceFunction { -public: - InnerProductDistance(vespalib::eval::CellType expected) : DistanceFunction(expected) {} - double calc(const vespalib::eval::TypedCells& lhs, const vespalib::eval::TypedCells& rhs) const override; - double convert_threshold(double threshold) const override { - return threshold; - } - double to_rawscore(double distance) const override { - double score = 1.0 / (1.0 + distance); - return score; - } - double calc_with_limit(const vespalib::eval::TypedCells& lhs, - const vespalib::eval::TypedCells& rhs, - double /*limit*/) const override - { - return calc(lhs, rhs); - } -}; - -/** - * Calculates inner-product "distance" between vectors with assumed norm 1. - * Should give same ordering as Angular distance, but is less expensive. - * Will use instruction optimal for the cpu it is running on - * when both vectors have the expected cell type. - */ -template <typename FloatType> -class InnerProductDistanceHW : public InnerProductDistance { -public: - InnerProductDistanceHW() - : InnerProductDistance(vespalib::eval::get_cell_type<FloatType>()), - _computer(vespalib::hwaccelrated::IAccelrated::getAccelerator()) - { - assert(expected_cell_type() == vespalib::eval::get_cell_type<FloatType>()); - } - double calc(const vespalib::eval::TypedCells& lhs, const vespalib::eval::TypedCells& rhs) const override { - constexpr vespalib::eval::CellType expected = vespalib::eval::get_cell_type<FloatType>(); - assert(lhs.type == expected && rhs.type == expected); - auto lhs_vector = lhs.typify<FloatType>(); - auto rhs_vector = rhs.typify<FloatType>(); - size_t sz = lhs_vector.size(); - assert(sz == rhs_vector.size()); - double score = 1.0 - _computer.dotProduct(lhs_vector.data(), rhs_vector.data(), sz); - return std::max(0.0, score); - } -private: - const vespalib::hwaccelrated::IAccelrated & _computer; -}; - -} diff --git a/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.h b/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.h index be1699c36ad..b91efbddb3e 100644 --- a/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.h +++ b/searchlib/src/vespa/searchlib/tensor/prenormalized_angular_distance.h @@ -11,7 +11,7 @@ namespace search::tensor { /** - * Calculates inner-product "distance" between vectors with assumed norm 1. + * Calculates inner-product "distance" between vectors assuming a common norm. * Should give same ordering as Angular distance, but is less expensive. */ template <typename FloatType> diff --git a/streamingvisitors/src/tests/hitcollector/hitcollector_test.cpp b/streamingvisitors/src/tests/hitcollector/hitcollector_test.cpp index 059d3c9f597..362aaeda938 100644 --- a/streamingvisitors/src/tests/hitcollector/hitcollector_test.cpp +++ b/streamingvisitors/src/tests/hitcollector/hitcollector_test.cpp @@ -11,6 +11,7 @@ #include <vespa/eval/eval/value_codec.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/stringfmt.h> using namespace document; using namespace search::fef; @@ -22,7 +23,39 @@ using vespalib::eval::DoubleValue; using vespalib::eval::SimpleValue; using vespalib::eval::TensorSpec; using vespalib::eval::Value; +using vespalib::make_string_short::fmt; +using FeatureValue = FeatureSet::Value; + +namespace { + +double as_double(const FeatureValue& v) { + EXPECT_TRUE(v.is_double()); + return v.as_double(); +} + +TensorSpec as_spec(const FeatureValue& v) { + EXPECT_TRUE(v.is_data()); + auto mem = v.as_data(); + nbostream buf(mem.data, mem.size); + return spec_from_value(*SimpleValue::from_stream(buf)); +} + +ConstArrayRef<FeatureValue> as_value_slice(FeatureValues& mf, uint32_t index, uint32_t num_features) +{ + return { mf.values.data() + index * num_features, num_features }; +} + +void check_match_features(ConstArrayRef<FeatureValue> v, uint32_t docid) +{ + SCOPED_TRACE(fmt("Checking docid %u for expected match features", docid)); + // The following values should have been set by MyRankProgram::run() + EXPECT_EQ(10 + docid, as_double(v[0])); + EXPECT_EQ(30 + docid, as_double(v[1])); + EXPECT_EQ(TensorSpec("tensor(x{})").add({{"x", "a"}}, 20 + docid), as_spec(v[2])); +} + +} namespace streaming { @@ -326,6 +359,33 @@ TEST_F(HitCollectorTest, feature_set) assertHit(30, 4, 2, sr); } +TEST_F(HitCollectorTest, match_features) +{ + HitCollector hc(3); + + addHit(hc, 0, 10); + addHit(hc, 1, 50); // on heap + addHit(hc, 2, 20); + addHit(hc, 3, 40); // on heap + addHit(hc, 4, 30); // on heap + + MyRankProgram rankProgram; + FeatureResolver resolver(rankProgram.get_resolver()); + search::StringStringMap renames; + renames["bar"] = "qux"; + auto mf = hc.get_match_features(rankProgram, resolver, renames); + auto num_features = resolver.num_features(); + + EXPECT_EQ(num_features, mf.names.size()); + EXPECT_EQ("foo", mf.names[0]); + EXPECT_EQ("qux", mf.names[1]); + EXPECT_EQ("baz", mf.names[2]); + EXPECT_EQ(num_features * 3, mf.values.size()); + check_match_features(as_value_slice(mf, 0, num_features), 1); + check_match_features(as_value_slice(mf, 1, num_features), 3); + check_match_features(as_value_slice(mf, 2, num_features), 4); +} + } // namespace streaming GTEST_MAIN_RUN_ALL_TESTS() diff --git a/streamingvisitors/src/tests/nearest_neighbor_field_searcher/nearest_neighbor_field_searcher_test.cpp b/streamingvisitors/src/tests/nearest_neighbor_field_searcher/nearest_neighbor_field_searcher_test.cpp index b64d477fd4c..2cd9017653f 100644 --- a/streamingvisitors/src/tests/nearest_neighbor_field_searcher/nearest_neighbor_field_searcher_test.cpp +++ b/streamingvisitors/src/tests/nearest_neighbor_field_searcher/nearest_neighbor_field_searcher_test.cpp @@ -57,7 +57,6 @@ public: vsm::test::MockFieldSearcherEnv env; ValueType tensor_type; TensorDataType data_type; - SquaredEuclideanDistance dist_func; vsm::FieldIdT field_id; NearestNeighborFieldSearcher searcher; MockQuery query; @@ -66,7 +65,6 @@ public: : env(), tensor_type(ValueType::from_spec("tensor(x[2])")), data_type(tensor_type), - dist_func(CellType::DOUBLE), field_id(2), searcher(field_id, DistanceMetric::Euclidean), query() @@ -97,7 +95,7 @@ public: expect_match(exp_square_distance, node); } void expect_match(double exp_square_distance, const NearestNeighborQueryNode& node) { - double exp_raw_score = dist_func.to_rawscore(exp_square_distance); + double exp_raw_score = 1.0 / (1.0 + std::sqrt(exp_square_distance)); EXPECT_TRUE(node.evaluate()); EXPECT_DOUBLE_EQ(exp_square_distance, node.get_distance().value()); EXPECT_DOUBLE_EQ(exp_raw_score, node.get_raw_score().value()); diff --git a/streamingvisitors/src/tests/searchvisitor/cfg/documenttypes.cfg b/streamingvisitors/src/tests/searchvisitor/cfg/documenttypes.cfg index db0df6fba25..2eb6e552274 100644 --- a/streamingvisitors/src/tests/searchvisitor/cfg/documenttypes.cfg +++ b/streamingvisitors/src/tests/searchvisitor/cfg/documenttypes.cfg @@ -1,301 +1,57 @@ -enablecompression false -documenttype[0].id 1843830320 -documenttype[0].name "maptest" -documenttype[0].version 0 -documenttype[0].headerstruct -91088113 -documenttype[0].bodystruct -1659731740 -documenttype[0].inherits[0].id 8 -documenttype[0].datatype[0].id 3474528 -documenttype[0].datatype[0].type STRUCT -documenttype[0].datatype[0].array.element.id 0 -documenttype[0].datatype[0].map.key.id 0 -documenttype[0].datatype[0].map.value.id 0 -documenttype[0].datatype[0].wset.key.id 0 -documenttype[0].datatype[0].wset.createifnonexistent false -documenttype[0].datatype[0].wset.removeifzero false -documenttype[0].datatype[0].annotationref.annotation.id 0 -documenttype[0].datatype[0].sstruct.name "s1" -documenttype[0].datatype[0].sstruct.version 0 -documenttype[0].datatype[0].sstruct.compression.type NONE -documenttype[0].datatype[0].sstruct.compression.level 0 -documenttype[0].datatype[0].sstruct.compression.threshold 95 -documenttype[0].datatype[0].sstruct.compression.minsize 200 -documenttype[0].datatype[0].sstruct.field[0].name "a" -documenttype[0].datatype[0].sstruct.field[0].id 493339625 -documenttype[0].datatype[0].sstruct.field[0].datatype 2 -documenttype[0].datatype[0].sstruct.field[1].name "b" -documenttype[0].datatype[0].sstruct.field[1].id 441632370 -documenttype[0].datatype[0].sstruct.field[1].datatype 2 -documenttype[0].datatype[1].id 339965458 -documenttype[0].datatype[1].type MAP -documenttype[0].datatype[1].array.element.id 0 -documenttype[0].datatype[1].map.key.id 2 -documenttype[0].datatype[1].map.value.id 2 -documenttype[0].datatype[1].wset.key.id 0 -documenttype[0].datatype[1].wset.createifnonexistent false -documenttype[0].datatype[1].wset.removeifzero false -documenttype[0].datatype[1].annotationref.annotation.id 0 -documenttype[0].datatype[1].sstruct.name "" -documenttype[0].datatype[1].sstruct.version 0 -documenttype[0].datatype[1].sstruct.compression.type NONE -documenttype[0].datatype[1].sstruct.compression.level 0 -documenttype[0].datatype[1].sstruct.compression.threshold 95 -documenttype[0].datatype[1].sstruct.compression.minsize 200 -documenttype[0].datatype[2].id 1888564261 -documenttype[0].datatype[2].type MAP -documenttype[0].datatype[2].array.element.id 0 -documenttype[0].datatype[2].map.key.id 2 -documenttype[0].datatype[2].map.value.id 3474528 -documenttype[0].datatype[2].wset.key.id 0 -documenttype[0].datatype[2].wset.createifnonexistent false -documenttype[0].datatype[2].wset.removeifzero false -documenttype[0].datatype[2].annotationref.annotation.id 0 -documenttype[0].datatype[2].sstruct.name "" -documenttype[0].datatype[2].sstruct.version 0 -documenttype[0].datatype[2].sstruct.compression.type NONE -documenttype[0].datatype[2].sstruct.compression.level 0 -documenttype[0].datatype[2].sstruct.compression.threshold 95 -documenttype[0].datatype[2].sstruct.compression.minsize 200 -documenttype[0].datatype[3].id -1486737430 -documenttype[0].datatype[3].type ARRAY -documenttype[0].datatype[3].array.element.id 2 -documenttype[0].datatype[3].map.key.id 0 -documenttype[0].datatype[3].map.value.id 0 -documenttype[0].datatype[3].wset.key.id 0 -documenttype[0].datatype[3].wset.createifnonexistent false -documenttype[0].datatype[3].wset.removeifzero false -documenttype[0].datatype[3].annotationref.annotation.id 0 -documenttype[0].datatype[3].sstruct.name "" -documenttype[0].datatype[3].sstruct.version 0 -documenttype[0].datatype[3].sstruct.compression.type NONE -documenttype[0].datatype[3].sstruct.compression.level 0 -documenttype[0].datatype[3].sstruct.compression.threshold 95 -documenttype[0].datatype[3].sstruct.compression.minsize 200 -documenttype[0].datatype[4].id -1220861393 -documenttype[0].datatype[4].type MAP -documenttype[0].datatype[4].array.element.id 0 -documenttype[0].datatype[4].map.key.id 2 -documenttype[0].datatype[4].map.value.id -1486737430 -documenttype[0].datatype[4].wset.key.id 0 -documenttype[0].datatype[4].wset.createifnonexistent false -documenttype[0].datatype[4].wset.removeifzero false -documenttype[0].datatype[4].annotationref.annotation.id 0 -documenttype[0].datatype[4].sstruct.name "" -documenttype[0].datatype[4].sstruct.version 0 -documenttype[0].datatype[4].sstruct.compression.type NONE -documenttype[0].datatype[4].sstruct.compression.level 0 -documenttype[0].datatype[4].sstruct.compression.threshold 95 -documenttype[0].datatype[4].sstruct.compression.minsize 200 -documenttype[0].datatype[5].id 1070047409 -documenttype[0].datatype[5].type MAP -documenttype[0].datatype[5].array.element.id 0 -documenttype[0].datatype[5].map.key.id 2 -documenttype[0].datatype[5].map.value.id 339965458 -documenttype[0].datatype[5].wset.key.id 0 -documenttype[0].datatype[5].wset.createifnonexistent false -documenttype[0].datatype[5].wset.removeifzero false -documenttype[0].datatype[5].annotationref.annotation.id 0 -documenttype[0].datatype[5].sstruct.name "" -documenttype[0].datatype[5].sstruct.version 0 -documenttype[0].datatype[5].sstruct.compression.type NONE -documenttype[0].datatype[5].sstruct.compression.level 0 -documenttype[0].datatype[5].sstruct.compression.threshold 95 -documenttype[0].datatype[5].sstruct.compression.minsize 200 -documenttype[0].datatype[6].id -91088113 -documenttype[0].datatype[6].type STRUCT -documenttype[0].datatype[6].array.element.id 0 -documenttype[0].datatype[6].map.key.id 0 -documenttype[0].datatype[6].map.value.id 0 -documenttype[0].datatype[6].wset.key.id 0 -documenttype[0].datatype[6].wset.createifnonexistent false -documenttype[0].datatype[6].wset.removeifzero false -documenttype[0].datatype[6].annotationref.annotation.id 0 -documenttype[0].datatype[6].sstruct.name "maptest.header" -documenttype[0].datatype[6].sstruct.version 0 -documenttype[0].datatype[6].sstruct.compression.type NONE -documenttype[0].datatype[6].sstruct.compression.level 0 -documenttype[0].datatype[6].sstruct.compression.threshold 95 -documenttype[0].datatype[6].sstruct.compression.minsize 200 -documenttype[0].datatype[6].sstruct.field[0].name "name" -documenttype[0].datatype[6].sstruct.field[0].id 1160796772 -documenttype[0].datatype[6].sstruct.field[0].datatype 2 -documenttype[0].datatype[6].sstruct.field[1].name "m1" -documenttype[0].datatype[6].sstruct.field[1].id 656260193 -documenttype[0].datatype[6].sstruct.field[1].datatype 339965458 -documenttype[0].datatype[6].sstruct.field[2].name "m2" -documenttype[0].datatype[6].sstruct.field[2].id 1105173090 -documenttype[0].datatype[6].sstruct.field[2].datatype 1888564261 -documenttype[0].datatype[6].sstruct.field[3].name "m3" -documenttype[0].datatype[6].sstruct.field[3].id 1834987989 -documenttype[0].datatype[6].sstruct.field[3].datatype -1220861393 -documenttype[0].datatype[6].sstruct.field[4].name "m4" -documenttype[0].datatype[6].sstruct.field[4].id 1696105521 -documenttype[0].datatype[6].sstruct.field[4].datatype 1070047409 -documenttype[0].datatype[7].id -1659731740 -documenttype[0].datatype[7].type STRUCT -documenttype[0].datatype[7].array.element.id 0 -documenttype[0].datatype[7].map.key.id 0 -documenttype[0].datatype[7].map.value.id 0 -documenttype[0].datatype[7].wset.key.id 0 -documenttype[0].datatype[7].wset.createifnonexistent false -documenttype[0].datatype[7].wset.removeifzero false -documenttype[0].datatype[7].annotationref.annotation.id 0 -documenttype[0].datatype[7].sstruct.name "maptest.body" -documenttype[0].datatype[7].sstruct.version 0 -documenttype[0].datatype[7].sstruct.compression.type NONE -documenttype[0].datatype[7].sstruct.compression.level 0 -documenttype[0].datatype[7].sstruct.compression.threshold 95 -documenttype[0].datatype[7].sstruct.compression.minsize 200 -documenttype[1].id -753106277 -documenttype[1].name "maptest_search" -documenttype[1].version 0 -documenttype[1].headerstruct 919697476 -documenttype[1].bodystruct -125720743 -documenttype[1].inherits[0].id 8 -documenttype[1].datatype[0].id 3474528 -documenttype[1].datatype[0].type STRUCT -documenttype[1].datatype[0].array.element.id 0 -documenttype[1].datatype[0].map.key.id 0 -documenttype[1].datatype[0].map.value.id 0 -documenttype[1].datatype[0].wset.key.id 0 -documenttype[1].datatype[0].wset.createifnonexistent false -documenttype[1].datatype[0].wset.removeifzero false -documenttype[1].datatype[0].annotationref.annotation.id 0 -documenttype[1].datatype[0].sstruct.name "s1" -documenttype[1].datatype[0].sstruct.version 0 -documenttype[1].datatype[0].sstruct.compression.type NONE -documenttype[1].datatype[0].sstruct.compression.level 0 -documenttype[1].datatype[0].sstruct.compression.threshold 95 -documenttype[1].datatype[0].sstruct.compression.minsize 200 -documenttype[1].datatype[0].sstruct.field[0].name "a" -documenttype[1].datatype[0].sstruct.field[0].id 493339625 -documenttype[1].datatype[0].sstruct.field[0].datatype 2 -documenttype[1].datatype[0].sstruct.field[1].name "b" -documenttype[1].datatype[0].sstruct.field[1].id 441632370 -documenttype[1].datatype[0].sstruct.field[1].datatype 2 -documenttype[1].datatype[1].id 339965458 -documenttype[1].datatype[1].type MAP -documenttype[1].datatype[1].array.element.id 0 -documenttype[1].datatype[1].map.key.id 2 -documenttype[1].datatype[1].map.value.id 2 -documenttype[1].datatype[1].wset.key.id 0 -documenttype[1].datatype[1].wset.createifnonexistent false -documenttype[1].datatype[1].wset.removeifzero false -documenttype[1].datatype[1].annotationref.annotation.id 0 -documenttype[1].datatype[1].sstruct.name "" -documenttype[1].datatype[1].sstruct.version 0 -documenttype[1].datatype[1].sstruct.compression.type NONE -documenttype[1].datatype[1].sstruct.compression.level 0 -documenttype[1].datatype[1].sstruct.compression.threshold 95 -documenttype[1].datatype[1].sstruct.compression.minsize 200 -documenttype[1].datatype[2].id 1888564261 -documenttype[1].datatype[2].type MAP -documenttype[1].datatype[2].array.element.id 0 -documenttype[1].datatype[2].map.key.id 2 -documenttype[1].datatype[2].map.value.id 3474528 -documenttype[1].datatype[2].wset.key.id 0 -documenttype[1].datatype[2].wset.createifnonexistent false -documenttype[1].datatype[2].wset.removeifzero false -documenttype[1].datatype[2].annotationref.annotation.id 0 -documenttype[1].datatype[2].sstruct.name "" -documenttype[1].datatype[2].sstruct.version 0 -documenttype[1].datatype[2].sstruct.compression.type NONE -documenttype[1].datatype[2].sstruct.compression.level 0 -documenttype[1].datatype[2].sstruct.compression.threshold 95 -documenttype[1].datatype[2].sstruct.compression.minsize 200 -documenttype[1].datatype[3].id -1486737430 -documenttype[1].datatype[3].type ARRAY -documenttype[1].datatype[3].array.element.id 2 -documenttype[1].datatype[3].map.key.id 0 -documenttype[1].datatype[3].map.value.id 0 -documenttype[1].datatype[3].wset.key.id 0 -documenttype[1].datatype[3].wset.createifnonexistent false -documenttype[1].datatype[3].wset.removeifzero false -documenttype[1].datatype[3].annotationref.annotation.id 0 -documenttype[1].datatype[3].sstruct.name "" -documenttype[1].datatype[3].sstruct.version 0 -documenttype[1].datatype[3].sstruct.compression.type NONE -documenttype[1].datatype[3].sstruct.compression.level 0 -documenttype[1].datatype[3].sstruct.compression.threshold 95 -documenttype[1].datatype[3].sstruct.compression.minsize 200 -documenttype[1].datatype[4].id -1220861393 -documenttype[1].datatype[4].type MAP -documenttype[1].datatype[4].array.element.id 0 -documenttype[1].datatype[4].map.key.id 2 -documenttype[1].datatype[4].map.value.id -1486737430 -documenttype[1].datatype[4].wset.key.id 0 -documenttype[1].datatype[4].wset.createifnonexistent false -documenttype[1].datatype[4].wset.removeifzero false -documenttype[1].datatype[4].annotationref.annotation.id 0 -documenttype[1].datatype[4].sstruct.name "" -documenttype[1].datatype[4].sstruct.version 0 -documenttype[1].datatype[4].sstruct.compression.type NONE -documenttype[1].datatype[4].sstruct.compression.level 0 -documenttype[1].datatype[4].sstruct.compression.threshold 95 -documenttype[1].datatype[4].sstruct.compression.minsize 200 -documenttype[1].datatype[5].id 1070047409 -documenttype[1].datatype[5].type MAP -documenttype[1].datatype[5].array.element.id 0 -documenttype[1].datatype[5].map.key.id 2 -documenttype[1].datatype[5].map.value.id 339965458 -documenttype[1].datatype[5].wset.key.id 0 -documenttype[1].datatype[5].wset.createifnonexistent false -documenttype[1].datatype[5].wset.removeifzero false -documenttype[1].datatype[5].annotationref.annotation.id 0 -documenttype[1].datatype[5].sstruct.name "" -documenttype[1].datatype[5].sstruct.version 0 -documenttype[1].datatype[5].sstruct.compression.type NONE -documenttype[1].datatype[5].sstruct.compression.level 0 -documenttype[1].datatype[5].sstruct.compression.threshold 95 -documenttype[1].datatype[5].sstruct.compression.minsize 200 -documenttype[1].datatype[6].id 919697476 -documenttype[1].datatype[6].type STRUCT -documenttype[1].datatype[6].array.element.id 0 -documenttype[1].datatype[6].map.key.id 0 -documenttype[1].datatype[6].map.value.id 0 -documenttype[1].datatype[6].wset.key.id 0 -documenttype[1].datatype[6].wset.createifnonexistent false -documenttype[1].datatype[6].wset.removeifzero false -documenttype[1].datatype[6].annotationref.annotation.id 0 -documenttype[1].datatype[6].sstruct.name "maptest_search.header" -documenttype[1].datatype[6].sstruct.version 0 -documenttype[1].datatype[6].sstruct.compression.type NONE -documenttype[1].datatype[6].sstruct.compression.level 0 -documenttype[1].datatype[6].sstruct.compression.threshold 95 -documenttype[1].datatype[6].sstruct.compression.minsize 200 -documenttype[1].datatype[6].sstruct.field[0].name "name" -documenttype[1].datatype[6].sstruct.field[0].id 1160796772 -documenttype[1].datatype[6].sstruct.field[0].datatype 2 -documenttype[1].datatype[6].sstruct.field[1].name "m1" -documenttype[1].datatype[6].sstruct.field[1].id 656260193 -documenttype[1].datatype[6].sstruct.field[1].datatype 339965458 -documenttype[1].datatype[6].sstruct.field[2].name "m2" -documenttype[1].datatype[6].sstruct.field[2].id 1105173090 -documenttype[1].datatype[6].sstruct.field[2].datatype 1888564261 -documenttype[1].datatype[6].sstruct.field[3].name "m3" -documenttype[1].datatype[6].sstruct.field[3].id 1834987989 -documenttype[1].datatype[6].sstruct.field[3].datatype -1220861393 -documenttype[1].datatype[6].sstruct.field[4].name "m4" -documenttype[1].datatype[6].sstruct.field[4].id 1696105521 -documenttype[1].datatype[6].sstruct.field[4].datatype 1070047409 -documenttype[1].datatype[6].sstruct.field[5].name "rankfeatures" -documenttype[1].datatype[6].sstruct.field[5].id 1883197392 -documenttype[1].datatype[6].sstruct.field[5].datatype 2 -documenttype[1].datatype[6].sstruct.field[6].name "summaryfeatures" -documenttype[1].datatype[6].sstruct.field[6].id 1840337115 -documenttype[1].datatype[6].sstruct.field[6].datatype 2 -documenttype[1].datatype[7].id -125720743 -documenttype[1].datatype[7].type STRUCT -documenttype[1].datatype[7].array.element.id 0 -documenttype[1].datatype[7].map.key.id 0 -documenttype[1].datatype[7].map.value.id 0 -documenttype[1].datatype[7].wset.key.id 0 -documenttype[1].datatype[7].wset.createifnonexistent false -documenttype[1].datatype[7].wset.removeifzero false -documenttype[1].datatype[7].annotationref.annotation.id 0 -documenttype[1].datatype[7].sstruct.name "maptest_search.body" -documenttype[1].datatype[7].sstruct.version 0 -documenttype[1].datatype[7].sstruct.compression.type NONE -documenttype[1].datatype[7].sstruct.compression.level 0 -documenttype[1].datatype[7].sstruct.compression.threshold 95 -documenttype[1].datatype[7].sstruct.compression.minsize 200 +ignoreundefinedfields false +usev8geopositions false +doctype[0].name "document" +doctype[0].idx 10000 +doctype[0].internalid 8 +doctype[0].contentstruct 10001 +doctype[0].primitivetype[0].idx 10002 +doctype[0].primitivetype[0].name "bool" +doctype[0].primitivetype[1].idx 10003 +doctype[0].primitivetype[1].name "byte" +doctype[0].primitivetype[2].idx 10004 +doctype[0].primitivetype[2].name "double" +doctype[0].primitivetype[3].idx 10005 +doctype[0].primitivetype[3].name "float" +doctype[0].primitivetype[4].idx 10006 +doctype[0].primitivetype[4].name "float16" +doctype[0].primitivetype[5].idx 10007 +doctype[0].primitivetype[5].name "int" +doctype[0].primitivetype[6].idx 10008 +doctype[0].primitivetype[6].name "long" +doctype[0].primitivetype[7].idx 10010 +doctype[0].primitivetype[7].name "predicate" +doctype[0].primitivetype[8].idx 10011 +doctype[0].primitivetype[8].name "raw" +doctype[0].primitivetype[9].idx 10012 +doctype[0].primitivetype[9].name "string" +doctype[0].primitivetype[10].idx 10014 +doctype[0].primitivetype[10].name "uri" +doctype[0].wsettype[0].idx 10013 +doctype[0].wsettype[0].elementtype 10012 +doctype[0].wsettype[0].createifnonexistent true +doctype[0].wsettype[0].removeifzero true +doctype[0].wsettype[0].internalid 18 +doctype[0].structtype[0].idx 10001 +doctype[0].structtype[0].name "document.header" +doctype[0].structtype[0].internalid -284186494 +doctype[0].structtype[1].idx 10009 +doctype[0].structtype[1].name "position" +doctype[0].structtype[1].field[0].name "x" +doctype[0].structtype[1].field[0].internalid 914677694 +doctype[0].structtype[1].field[0].type 10007 +doctype[0].structtype[1].field[1].name "y" +doctype[0].structtype[1].field[1].internalid 900009410 +doctype[0].structtype[1].field[1].type 10007 +doctype[0].structtype[1].internalid 1381038251 +doctype[1].name "test" +doctype[1].idx 10015 +doctype[1].internalid -877171244 +doctype[1].inherits[0].idx 10000 +doctype[1].contentstruct 10016 +doctype[1].fieldsets{[document]}.fields[0] "id" +doctype[1].structtype[0].idx 10016 +doctype[1].structtype[0].name "test.header" +doctype[1].structtype[0].field[0].name "id" +doctype[1].structtype[0].field[0].internalid 2086523186 +doctype[1].structtype[0].field[0].type 10007 +doctype[1].structtype[0].internalid 306916075 diff --git a/streamingvisitors/src/tests/searchvisitor/cfg/generate.sh b/streamingvisitors/src/tests/searchvisitor/cfg/generate.sh new file mode 100755 index 00000000000..1eade461060 --- /dev/null +++ b/streamingvisitors/src/tests/searchvisitor/cfg/generate.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# Use this script to generate config files based on test.sd: +# ./generate.sh + +# Generate config files: +cd $HOME/git/vespa +WORK_DIR=$HOME/git/vespa/streamingvisitors/src/tests/searchvisitor/cfg +mvn test -Dtest=SchemaToDerivedConfigExporter -Dschema.exporter.path=$WORK_DIR -pl config-model + +cd $WORK_DIR +# Delete files not relevant for streaming: +rm attributes.cfg +rm ilscripts.cfg +rm imported-fields.cfg +rm index-info.cfg +rm indexschema.cfg +rm onnx-models.cfg +rm schema-info.cfg + +# Add search cluster name as part of file name: +mv juniperrc.cfg juniperrc.mycl.cfg +mv rank-profiles.cfg rank-profiles.mycl.cfg +mv summary.cfg summary.mycl.cfg +mv vsmfields.cfg vsmfields.mycl.cfg +mv vsmsummary.cfg vsmsummary.mycl.cfg diff --git a/streamingvisitors/src/tests/searchvisitor/cfg/juniperrc.aaa.cfg b/streamingvisitors/src/tests/searchvisitor/cfg/juniperrc.mycl.cfg index 09e13ed0f8b..f2116782b7a 100644 --- a/streamingvisitors/src/tests/searchvisitor/cfg/juniperrc.aaa.cfg +++ b/streamingvisitors/src/tests/searchvisitor/cfg/juniperrc.mycl.cfg @@ -1,11 +1,10 @@ length 256 -max_match_candidates 1000 max_matches 3 min_length 128 prefix true -stem_max_extend 3 -stem_min_length 5 surround_max 128 winsize 200 winsize_fallback_multiplier 10.0 -override[0] +max_match_candidates 1000 +stem_min_length 5 +stem_max_extend 3 diff --git a/streamingvisitors/src/tests/searchvisitor/cfg/juniperrc.simple.cfg b/streamingvisitors/src/tests/searchvisitor/cfg/juniperrc.simple.cfg deleted file mode 100644 index e69de29bb2d..00000000000 --- a/streamingvisitors/src/tests/searchvisitor/cfg/juniperrc.simple.cfg +++ /dev/null diff --git a/streamingvisitors/src/tests/searchvisitor/cfg/rank-profiles.aaa.cfg b/streamingvisitors/src/tests/searchvisitor/cfg/rank-profiles.aaa.cfg deleted file mode 100644 index cf3ee6a7179..00000000000 --- a/streamingvisitors/src/tests/searchvisitor/cfg/rank-profiles.aaa.cfg +++ /dev/null @@ -1,43 +0,0 @@ -rankprofile[2] -rankprofile[0].name "default" -rankprofile[0].fef.property[15] -rankprofile[0].fef.property[00].name "vespa.summary.feature" -rankprofile[0].fef.property[00].value "fieldTermMatch(m1.key,0).firstPosition" -rankprofile[0].fef.property[01].name "vespa.summary.feature" -rankprofile[0].fef.property[01].value "fieldTermMatch(m1.key,0).occurrences" -rankprofile[0].fef.property[02].name "vespa.summary.feature" -rankprofile[0].fef.property[02].value "fieldLength(m1.key)" -rankprofile[0].fef.property[03].name "vespa.summary.feature" -rankprofile[0].fef.property[03].value "fieldTermMatch(m1.value,0).firstPosition" -rankprofile[0].fef.property[04].name "vespa.summary.feature" -rankprofile[0].fef.property[04].value "fieldTermMatch(m1.value,0).occurrences" -rankprofile[0].fef.property[05].name "vespa.summary.feature" -rankprofile[0].fef.property[05].value "fieldLength(m1.value)" -rankprofile[0].fef.property[06].name "vespa.summary.feature" -rankprofile[0].fef.property[06].value "fieldTermMatch(m2.value.a,0).firstPosition" -rankprofile[0].fef.property[07].name "vespa.summary.feature" -rankprofile[0].fef.property[07].value "fieldTermMatch(m2.value.a,0).occurrences" -rankprofile[0].fef.property[08].name "vespa.summary.feature" -rankprofile[0].fef.property[08].value "fieldLength(m2.value.a)" -rankprofile[0].fef.property[09].name "vespa.summary.feature" -rankprofile[0].fef.property[09].value "fieldTermMatch(m3.value,0).firstPosition" -rankprofile[0].fef.property[10].name "vespa.summary.feature" -rankprofile[0].fef.property[10].value "fieldTermMatch(m3.value,0).occurrences" -rankprofile[0].fef.property[11].name "vespa.summary.feature" -rankprofile[0].fef.property[11].value "fieldLength(m3.value)" -rankprofile[0].fef.property[12].name "vespa.summary.feature" -rankprofile[0].fef.property[12].value "fieldTermMatch(m4.value.value,0).firstPosition" -rankprofile[0].fef.property[13].name "vespa.summary.feature" -rankprofile[0].fef.property[13].value "fieldTermMatch(m4.value.value,0).occurrences" -rankprofile[0].fef.property[14].name "vespa.summary.feature" -rankprofile[0].fef.property[14].value "fieldLength(m4.value.value)" -rankprofile[1].name "unranked" -rankprofile[1].fef.property[4] -rankprofile[1].fef.property[0].name "vespa.rank.firstphase" -rankprofile[1].fef.property[0].value "value(0)" -rankprofile[1].fef.property[1].name "vespa.hitcollector.heapsize" -rankprofile[1].fef.property[1].value "0" -rankprofile[1].fef.property[2].name "vespa.hitcollector.arraysize" -rankprofile[1].fef.property[2].value "0" -rankprofile[1].fef.property[3].name "vespa.dump.ignoredefaultfeatures" -rankprofile[1].fef.property[3].value "true" diff --git a/streamingvisitors/src/tests/searchvisitor/cfg/rank-profiles.mycl.cfg b/streamingvisitors/src/tests/searchvisitor/cfg/rank-profiles.mycl.cfg new file mode 100644 index 00000000000..d3132a8ff25 --- /dev/null +++ b/streamingvisitors/src/tests/searchvisitor/cfg/rank-profiles.mycl.cfg @@ -0,0 +1,15 @@ +rankprofile[0].name "default" +rankprofile[1].name "unranked" +rankprofile[1].fef.property[0].name "vespa.rank.firstphase" +rankprofile[1].fef.property[0].value "value(0)" +rankprofile[1].fef.property[1].name "vespa.hitcollector.heapsize" +rankprofile[1].fef.property[1].value "0" +rankprofile[1].fef.property[2].name "vespa.hitcollector.arraysize" +rankprofile[1].fef.property[2].value "0" +rankprofile[1].fef.property[3].name "vespa.dump.ignoredefaultfeatures" +rankprofile[1].fef.property[3].value "true" +rankprofile[2].name "myrank" +rankprofile[2].fef.property[0].name "vespa.rank.firstphase" +rankprofile[2].fef.property[0].value "rankingExpression(firstphase)" +rankprofile[2].fef.property[1].name "rankingExpression(firstphase).rankingScript" +rankprofile[2].fef.property[1].value "attribute(id) + 10" diff --git a/streamingvisitors/src/tests/searchvisitor/cfg/rank-profiles.simple.cfg b/streamingvisitors/src/tests/searchvisitor/cfg/rank-profiles.simple.cfg deleted file mode 100644 index e69de29bb2d..00000000000 --- a/streamingvisitors/src/tests/searchvisitor/cfg/rank-profiles.simple.cfg +++ /dev/null diff --git a/streamingvisitors/src/tests/searchvisitor/cfg/summary.aaa.cfg b/streamingvisitors/src/tests/searchvisitor/cfg/summary.aaa.cfg deleted file mode 100644 index 8cc32ed5ea4..00000000000 --- a/streamingvisitors/src/tests/searchvisitor/cfg/summary.aaa.cfg +++ /dev/null @@ -1,30 +0,0 @@ -defaultsummaryid 197963550 -classes[2] -classes[0].id 190911431 -classes[0].name "attributeprefetch" -classes[0].fields[3] -classes[0].fields[0].name "name" -classes[0].fields[0].type "longstring" -classes[0].fields[1].name "rankfeatures" -classes[0].fields[1].type "longstring" -classes[0].fields[2].name "summaryfeatures" -classes[0].fields[2].type "longstring" -classes[1].id 197963550 -classes[1].name "maptest" -classes[1].fields[8] -classes[1].fields[0].name "documentid" -classes[1].fields[0].type "longstring" -classes[1].fields[1].name "m1" -classes[1].fields[1].type "jsonstring" -classes[1].fields[2].name "m2" -classes[1].fields[2].type "jsonstring" -classes[1].fields[3].name "m3" -classes[1].fields[3].type "jsonstring" -classes[1].fields[4].name "m4" -classes[1].fields[4].type "jsonstring" -classes[1].fields[5].name "name" -classes[1].fields[5].type "longstring" -classes[1].fields[6].name "rankfeatures" -classes[1].fields[6].type "longstring" -classes[1].fields[7].name "summaryfeatures" -classes[1].fields[7].type "longstring" diff --git a/streamingvisitors/src/tests/searchvisitor/cfg/summary.mycl.cfg b/streamingvisitors/src/tests/searchvisitor/cfg/summary.mycl.cfg new file mode 100644 index 00000000000..5983609ac9d --- /dev/null +++ b/streamingvisitors/src/tests/searchvisitor/cfg/summary.mycl.cfg @@ -0,0 +1,41 @@ +defaultsummaryid 793791073 +usev8geopositions true +classes[0].id 793791073 +classes[0].name "default" +classes[0].omitsummaryfeatures false +classes[0].fields[0].name "id" +classes[0].fields[0].command "attribute" +classes[0].fields[0].source "id" +classes[0].fields[1].name "rankfeatures" +classes[0].fields[1].command "rankfeatures" +classes[0].fields[1].source "" +classes[0].fields[2].name "summaryfeatures" +classes[0].fields[2].command "summaryfeatures" +classes[0].fields[2].source "" +classes[0].fields[3].name "documentid" +classes[0].fields[3].command "documentid" +classes[0].fields[3].source "" +classes[1].id 1473683981 +classes[1].name "mysum" +classes[1].omitsummaryfeatures false +classes[1].fields[0].name "id" +classes[1].fields[0].command "attribute" +classes[1].fields[0].source "id" +classes[1].fields[1].name "rankfeatures" +classes[1].fields[1].command "rankfeatures" +classes[1].fields[1].source "" +classes[1].fields[2].name "summaryfeatures" +classes[1].fields[2].command "summaryfeatures" +classes[1].fields[2].source "" +classes[2].id 1783786855 +classes[2].name "attributeprefetch" +classes[2].omitsummaryfeatures false +classes[2].fields[0].name "id" +classes[2].fields[0].command "attribute" +classes[2].fields[0].source "id" +classes[2].fields[1].name "rankfeatures" +classes[2].fields[1].command "rankfeatures" +classes[2].fields[1].source "" +classes[2].fields[2].name "summaryfeatures" +classes[2].fields[2].command "summaryfeatures" +classes[2].fields[2].source "" diff --git a/streamingvisitors/src/tests/searchvisitor/cfg/summary.simple.cfg b/streamingvisitors/src/tests/searchvisitor/cfg/summary.simple.cfg deleted file mode 100644 index e69de29bb2d..00000000000 --- a/streamingvisitors/src/tests/searchvisitor/cfg/summary.simple.cfg +++ /dev/null diff --git a/streamingvisitors/src/tests/searchvisitor/cfg/summarymap.aaa.cfg b/streamingvisitors/src/tests/searchvisitor/cfg/summarymap.aaa.cfg deleted file mode 100644 index 1c9567431ae..00000000000 --- a/streamingvisitors/src/tests/searchvisitor/cfg/summarymap.aaa.cfg +++ /dev/null @@ -1,11 +0,0 @@ -defaultoutputclass -1 -override[3] -override[0].arguments "name" -override[0].command "attribute" -override[0].field "name" -override[1].arguments "" -override[1].command "rankfeatures" -override[1].field "rankfeatures" -override[2].arguments "" -override[2].command "summaryfeatures" -override[2].field "summaryfeatures" diff --git a/streamingvisitors/src/tests/searchvisitor/cfg/summarymap.simple.cfg b/streamingvisitors/src/tests/searchvisitor/cfg/summarymap.simple.cfg deleted file mode 100644 index e69de29bb2d..00000000000 --- a/streamingvisitors/src/tests/searchvisitor/cfg/summarymap.simple.cfg +++ /dev/null diff --git a/streamingvisitors/src/tests/searchvisitor/cfg/test.sd b/streamingvisitors/src/tests/searchvisitor/cfg/test.sd new file mode 100644 index 00000000000..ad998aa91f6 --- /dev/null +++ b/streamingvisitors/src/tests/searchvisitor/cfg/test.sd @@ -0,0 +1,16 @@ +schema test { + document test { + field id type int { + indexing: attribute | summary + } + } + document-summary mysum { + summary id type int {} + } + rank-profile myrank { + first-phase { + expression: attribute(id) + 10 + } + } +} + diff --git a/streamingvisitors/src/tests/searchvisitor/cfg/vsmfields.aaa.cfg b/streamingvisitors/src/tests/searchvisitor/cfg/vsmfields.aaa.cfg deleted file mode 100644 index 4d976764bd9..00000000000 --- a/streamingvisitors/src/tests/searchvisitor/cfg/vsmfields.aaa.cfg +++ /dev/null @@ -1,98 +0,0 @@ -documentverificationlevel 0 -searchall 1 -documenttype[1] -documenttype[0].name "maptest" -documenttype[0].index[17] -documenttype[0].index[00].name "m1" -documenttype[0].index[00].field[2] -documenttype[0].index[00].field[0].name "m1.key" -documenttype[0].index[00].field[1].name "m1.value" -documenttype[0].index[01].name "m1.key" -documenttype[0].index[01].field[1] -documenttype[0].index[01].field[0].name "m1.key" -documenttype[0].index[02].name "m1.value" -documenttype[0].index[02].field[1] -documenttype[0].index[02].field[0].name "m1.value" -documenttype[0].index[03].name "m2" -documenttype[0].index[03].field[3] -documenttype[0].index[03].field[0].name "m2.key" -documenttype[0].index[03].field[1].name "m2.value.a" -documenttype[0].index[03].field[2].name "m2.value.b" -documenttype[0].index[04].name "m2.key" -documenttype[0].index[04].field[1] -documenttype[0].index[04].field[0].name "m2.key" -documenttype[0].index[05].name "m2.value" -documenttype[0].index[05].field[2] -documenttype[0].index[05].field[0].name "m2.value.a" -documenttype[0].index[05].field[1].name "m2.value.b" -documenttype[0].index[06].name "m2.value.a" -documenttype[0].index[06].field[1] -documenttype[0].index[06].field[0].name "m2.value.a" -documenttype[0].index[07].name "m2.value.b" -documenttype[0].index[07].field[1] -documenttype[0].index[07].field[0].name "m2.value.b" -documenttype[0].index[08].name "m3" -documenttype[0].index[08].field[2] -documenttype[0].index[08].field[0].name "m3.key" -documenttype[0].index[08].field[1].name "m3.value" -documenttype[0].index[09].name "m3.key" -documenttype[0].index[09].field[1] -documenttype[0].index[09].field[0].name "m3.key" -documenttype[0].index[10].name "m3.value" -documenttype[0].index[10].field[1] -documenttype[0].index[10].field[0].name "m3.value" -documenttype[0].index[11].name "m4" -documenttype[0].index[11].field[3] -documenttype[0].index[11].field[0].name "m4.key" -documenttype[0].index[11].field[1].name "m4.value.key" -documenttype[0].index[11].field[2].name "m4.value.value" -documenttype[0].index[12].name "m4.key" -documenttype[0].index[12].field[1] -documenttype[0].index[12].field[0].name "m4.key" -documenttype[0].index[13].name "m4.value" -documenttype[0].index[13].field[2] -documenttype[0].index[13].field[0].name "m4.value.key" -documenttype[0].index[13].field[1].name "m4.value.value" -documenttype[0].index[14].name "m4.value.key" -documenttype[0].index[14].field[1] -documenttype[0].index[14].field[0].name "m4.value.key" -documenttype[0].index[15].name "m4.value.value" -documenttype[0].index[15].field[1] -documenttype[0].index[15].field[0].name "m4.value.value" -documenttype[0].index[16].name "name" -documenttype[0].index[16].field[1] -documenttype[0].index[16].field[0].name "name" -fieldspec[11] -fieldspec[00].arg1 "" -fieldspec[00].name "m1.key" -fieldspec[00].searchmethod AUTOUTF8 -fieldspec[01].arg1 "" -fieldspec[01].name "m1.value" -fieldspec[01].searchmethod AUTOUTF8 -fieldspec[02].arg1 "" -fieldspec[02].name "m2.key" -fieldspec[02].searchmethod AUTOUTF8 -fieldspec[03].arg1 "" -fieldspec[03].name "m2.value.a" -fieldspec[03].searchmethod AUTOUTF8 -fieldspec[04].arg1 "" -fieldspec[04].name "m2.value.b" -fieldspec[04].searchmethod AUTOUTF8 -fieldspec[05].arg1 "" -fieldspec[05].name "m3.key" -fieldspec[05].searchmethod AUTOUTF8 -fieldspec[06].arg1 "" -fieldspec[06].name "m3.value" -fieldspec[06].searchmethod AUTOUTF8 -fieldspec[07].arg1 "" -fieldspec[07].name "m4.key" -fieldspec[07].searchmethod AUTOUTF8 -fieldspec[08].arg1 "" -fieldspec[08].name "m4.value.key" -fieldspec[08].searchmethod AUTOUTF8 -fieldspec[09].arg1 "" -fieldspec[09].name "m4.value.value" -fieldspec[09].searchmethod AUTOUTF8 -fieldspec[10].arg1 "" -fieldspec[10].name "name" -fieldspec[10].searchmethod AUTOUTF8 diff --git a/streamingvisitors/src/tests/searchvisitor/cfg/vsmfields.mycl.cfg b/streamingvisitors/src/tests/searchvisitor/cfg/vsmfields.mycl.cfg new file mode 100644 index 00000000000..78b53fe24fb --- /dev/null +++ b/streamingvisitors/src/tests/searchvisitor/cfg/vsmfields.mycl.cfg @@ -0,0 +1,10 @@ +documentverificationlevel 0 +searchall 1 +fieldspec[0].name "id" +fieldspec[0].searchmethod INT32 +fieldspec[0].arg1 "" +fieldspec[0].maxlength 1048576 +fieldspec[0].fieldtype ATTRIBUTE +documenttype[0].name "test" +documenttype[0].index[0].name "id" +documenttype[0].index[0].field[0].name "id" diff --git a/streamingvisitors/src/tests/searchvisitor/cfg/vsmfields.simple.cfg b/streamingvisitors/src/tests/searchvisitor/cfg/vsmfields.simple.cfg deleted file mode 100644 index e69de29bb2d..00000000000 --- a/streamingvisitors/src/tests/searchvisitor/cfg/vsmfields.simple.cfg +++ /dev/null diff --git a/streamingvisitors/src/tests/searchvisitor/cfg/vsmsummary.aaa.cfg b/streamingvisitors/src/tests/searchvisitor/cfg/vsmsummary.aaa.cfg deleted file mode 100644 index 664778d40a8..00000000000 --- a/streamingvisitors/src/tests/searchvisitor/cfg/vsmsummary.aaa.cfg +++ /dev/null @@ -1,26 +0,0 @@ -outputclass "" -fieldmap[6] -fieldmap[0].command NONE -fieldmap[0].summary "m1" -fieldmap[0].document[1] -fieldmap[0].document[0].field "m1" -fieldmap[1].command NONE -fieldmap[1].summary "m2" -fieldmap[1].document[3] -fieldmap[1].document[0].field "m2.key" -fieldmap[1].document[1].field "m2.value.a" -fieldmap[1].document[2].field "m2.value.b" -fieldmap[2].command NONE -fieldmap[2].summary "m3" -fieldmap[2].document[1] -fieldmap[2].document[0].field "m3" -fieldmap[3].command NONE -fieldmap[3].summary "m4" -fieldmap[3].document[1] -fieldmap[3].document[0].field "m4" -fieldmap[4].command NONE -fieldmap[4].summary "rankfeatures" -fieldmap[4].document[0] -fieldmap[5].command NONE -fieldmap[5].summary "summaryfeatures" -fieldmap[5].document[0] diff --git a/streamingvisitors/src/tests/searchvisitor/cfg/vsmsummary.mycl.cfg b/streamingvisitors/src/tests/searchvisitor/cfg/vsmsummary.mycl.cfg new file mode 100644 index 00000000000..4dc35c8e1ce --- /dev/null +++ b/streamingvisitors/src/tests/searchvisitor/cfg/vsmsummary.mycl.cfg @@ -0,0 +1,8 @@ +outputclass "" +fieldmap[0].summary "id" +fieldmap[0].document[0].field "id" +fieldmap[0].command NONE +fieldmap[1].summary "rankfeatures" +fieldmap[1].command NONE +fieldmap[2].summary "summaryfeatures" +fieldmap[2].command NONE diff --git a/streamingvisitors/src/tests/searchvisitor/cfg/vsmsummary.simple.cfg b/streamingvisitors/src/tests/searchvisitor/cfg/vsmsummary.simple.cfg deleted file mode 100644 index e69de29bb2d..00000000000 --- a/streamingvisitors/src/tests/searchvisitor/cfg/vsmsummary.simple.cfg +++ /dev/null diff --git a/streamingvisitors/src/tests/searchvisitor/searchvisitor_test.cpp b/streamingvisitors/src/tests/searchvisitor/searchvisitor_test.cpp index 5521994e60e..b93a2c25f82 100644 --- a/streamingvisitors/src/tests/searchvisitor/searchvisitor_test.cpp +++ b/streamingvisitors/src/tests/searchvisitor/searchvisitor_test.cpp @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/document/base/testdocrepo.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/persistence/spi/docentry.h> #include <vespa/searchlib/query/tree/querybuilder.h> @@ -22,27 +24,149 @@ using namespace storage; namespace streaming { -class SearchVisitorTest : public testing::Test -{ +vespalib::string get_doc_id(int id) { + return "id:test:test::" + std::to_string(id); +} + +/** + * This class reflects the document type defined in cfg/test.sd. + */ +struct MyDocument { + int id; + MyDocument(int id_in) : id(id_in) {} + std::unique_ptr<Document> to_document(const DocumentTypeRepo& repo, const DataType& doc_type) const { + auto result = std::make_unique<Document>(repo, doc_type, DocumentId(get_doc_id(id))); + result->setValue("id", std::make_unique<IntFieldValue>(id)); + return result; + } +}; + +using DocumentVector = std::vector<MyDocument>; + +struct MyHit { + vespalib::string doc_id; + double rank; + MyHit(int id, double rank_in) noexcept : doc_id(get_doc_id(id)), rank(rank_in) {} + MyHit(int id) noexcept : doc_id(get_doc_id(id)), rank(0.0) {} + MyHit(const vespalib::string& doc_id_in, double rank_in) noexcept : doc_id(doc_id_in), rank(rank_in) {} + bool operator==(const MyHit& rhs) const { + return (doc_id == rhs.doc_id) && + (rank == rhs.rank); + } +}; + +using HitVector = std::vector<MyHit>; + +std::ostream& operator<<(std::ostream& oss, const MyHit& hit) { + oss << "{doc_id=" << hit.doc_id << ",rank=" << hit.rank << "}"; + return oss; +} + +class RequestBuilder { +private: + vdslib::Parameters _params; + QueryBuilder<SimpleQueryNodeTypes> _builder; + int32_t _term_id; + +public: + RequestBuilder() : _params(), _builder(), _term_id(1) + { + search_cluster("mycl"); + rank_profile("myrank"); + summary_class("mysum"); + summary_count(10); + } + RequestBuilder& set_param(const vespalib::string& key, const vespalib::string& value) { + _params.set(key, value); + return *this; + } + RequestBuilder& search_cluster(const vespalib::string& value) { return set_param("searchcluster", value); } + RequestBuilder& rank_profile(const vespalib::string& value) { return set_param("rankprofile", value); } + RequestBuilder& summary_class(const vespalib::string& value) { return set_param("summaryclass", value); } + RequestBuilder& summary_count(uint32_t value) { return set_param("summarycount", std::to_string(value)); } + RequestBuilder& string_term(const vespalib::string& term, const vespalib::string& field) { + _builder.addStringTerm(term, field, _term_id++, Weight(100)); + return *this; + } + RequestBuilder& number_term(const vespalib::string& term, const vespalib::string& field) { + _builder.addNumberTerm(term, field, _term_id++, Weight(100)); + return *this; + } + vdslib::Parameters build() { + auto node = _builder.build(); + vespalib::string query_stack_dump = StackDumpCreator::create(*node); + _params.set("query", query_stack_dump); + return _params; + } +}; + +struct VisitorSession { + std::unique_ptr<SearchVisitor> search_visitor; + Visitor& visitor; + Visitor::HitCounter hit_counter; + VisitorSession(SearchVisitor* sv) + : search_visitor(sv), + visitor(*search_visitor), + hit_counter() + { + } + void handle_documents(Visitor::DocEntryList& docs) { + document::BucketId bucket_id; + visitor.handleDocuments(bucket_id, docs, hit_counter); + } + std::unique_ptr<documentapi::QueryResultMessage> generate_query_result() { + return search_visitor->generate_query_result(hit_counter); + } +}; + +class SearchVisitorTest : public testing::Test { public: framework::defaultimplementation::FakeClock _clock; StorageComponentRegisterImpl _componentRegister; std::unique_ptr<StorageComponent> _component; SearchEnvironment _env; - void testCreateSearchVisitor(const vespalib::string & dir, const vdslib::Parameters & parameters); + SearchVisitorFactory _factory; + std::shared_ptr<DocumentTypeRepo> _repo; + const document::DocumentType* _doc_type; SearchVisitorTest(); ~SearchVisitorTest() override; + + std::unique_ptr<VisitorSession> make_visitor_session(const vdslib::Parameters& params) { + VisitorFactory& factory(_factory); + auto *visitor = factory.makeVisitor(*_component, _env, params); + auto *search_visitor = dynamic_cast<SearchVisitor *>(visitor); + assert(search_visitor != nullptr); + return std::make_unique<VisitorSession>(search_visitor); + } + Visitor::DocEntryList make_documents(const std::vector<MyDocument>& docs) const { + Visitor::DocEntryList result; + for (const auto& doc : docs) { + result.push_back(spi::DocEntry::create(spi::Timestamp(), + doc.to_document(*_repo, *_doc_type))); + } + return result; + } + std::unique_ptr<documentapi::QueryResultMessage> execute_query(const vdslib::Parameters& params, + const DocumentVector& docs) { + auto session = make_visitor_session(params); + auto entries = make_documents(docs); + session->handle_documents(entries); + return session->generate_query_result(); + } }; SearchVisitorTest::SearchVisitorTest() : _componentRegister(), - _env(::config::ConfigUri("dir:cfg")) + _env(::config::ConfigUri("dir:cfg")), + _factory(::config::ConfigUri("dir:cfg")), + _repo(std::make_shared<DocumentTypeRepo>(readDocumenttypesConfig("cfg/documenttypes.cfg"))), + _doc_type(_repo->getDocumentType("test")) { - _componentRegister.setNodeInfo("mycluster", lib::NodeType::STORAGE, 1); + assert(_doc_type != nullptr); + _componentRegister.setNodeInfo("mycl", lib::NodeType::STORAGE, 1); _componentRegister.setClock(_clock); - auto repo = std::make_shared<DocumentTypeRepo>(readDocumenttypesConfig("cfg/documenttypes.cfg")); - _componentRegister.setDocumentTypeRepo(repo); + _componentRegister.setDocumentTypeRepo(_repo); _component = std::make_unique<StorageComponent>(_componentRegister, "storage"); } @@ -51,61 +175,67 @@ SearchVisitorTest::~SearchVisitorTest() _env.clear_thread_local_env_map(); } -Visitor::DocEntryList -createDocuments(const vespalib::string & dir) +TEST_F(SearchVisitorTest, search_environment_is_configured) +{ + EXPECT_TRUE(_env.getVSMAdapter("mycl") != nullptr); + EXPECT_TRUE(_env.getRankManager("mycl") != nullptr); +} + +HitVector +to_hit_vector(vdslib::SearchResult& res) +{ + HitVector result; + const char* doc_id; + double rank; + for (size_t i = 0; i < res.getHitCount(); ++i) { + res.getHit(i, doc_id, rank); + result.emplace_back(vespalib::string(doc_id), rank); + } + return result; +} + +HitVector +to_hit_vector(vdslib::DocumentSummary& sum) { - (void) dir; - Visitor::DocEntryList documents; - spi::Timestamp ts; - auto e = spi::DocEntry::create(ts, std::make_unique<Document>()); - documents.push_back(std::move(e)); - return documents; + HitVector result; + const char* doc_id; + const void* buf; + size_t sz; + for (size_t i = 0; i < sum.getSummaryCount(); ++i) { + sum.getSummary(i, doc_id, buf, sz); + result.emplace_back(vespalib::string(doc_id), 0.0); + } + return result; } void -SearchVisitorTest::testCreateSearchVisitor(const vespalib::string & dir, const vdslib::Parameters & params) +expect_hits(const HitVector& exp_hits, documentapi::QueryResultMessage& res) { - ::config::ConfigUri uri(dir); - SearchVisitorFactory sFactory(uri); - VisitorFactory & factory(sFactory); - std::unique_ptr<Visitor> sv(static_cast<SearchVisitor *>(factory.makeVisitor(*_component, _env, params))); - document::BucketId bucketId; - Visitor::DocEntryList documents(createDocuments(dir)); - Visitor::HitCounter hitCounter; - sv->handleDocuments(bucketId, documents, hitCounter); + EXPECT_EQ(exp_hits.size(), res.getSearchResult().getHitCount()); + EXPECT_EQ(exp_hits, to_hit_vector(res.getSearchResult())); } -TEST_F(SearchVisitorTest, test_search_environment) +void +expect_summary(const HitVector& exp_summary, documentapi::QueryResultMessage& res) { - EXPECT_TRUE(_env.getVSMAdapter("simple") != nullptr); - EXPECT_TRUE(_env.getRankManager("simple") != nullptr); + EXPECT_EQ(exp_summary.size(), res.getDocumentSummary().getSummaryCount()); + EXPECT_EQ(exp_summary, to_hit_vector(res.getDocumentSummary())); } -TEST_F(SearchVisitorTest, test_search_visitor) +TEST_F(SearchVisitorTest, basic_query_execution_in_search_visitor) { - vdslib::Parameters params; - params.set("searchcluster", "aaa"); - params.set("queryflags", "0x40000"); - params.set("summarycount", "3"); - params.set("summaryclass", "petra"); - params.set("rankprofile", "default"); - - QueryBuilder<SimpleQueryNodeTypes> builder; - builder.addStringTerm("maptest", "sddocname", 0, Weight(0)); - Node::UP node = builder.build(); - vespalib::string stackDump = StackDumpCreator::create(*node); - - params.set("query", stackDump); - testCreateSearchVisitor("dir:cfg", params); + auto res = execute_query(RequestBuilder(). + number_term("[5;10]", "id").build(), + {{3},{7},{4},{5},{9}}); + expect_hits({{9,19.0}, {7,17.0}, {5,15.0}}, *res); + expect_summary({{5}, {7}, {9}}, *res); } -TEST_F(SearchVisitorTest, test_only_require_weak_read_consistency) +TEST_F(SearchVisitorTest, visitor_only_require_weak_read_consistency) { - SearchVisitorFactory factory(::config::ConfigUri("dir:cfg")); - VisitorFactory& factoryBase(factory); vdslib::Parameters params; - std::unique_ptr<Visitor> sv(factoryBase.makeVisitor(*_component, _env, params)); - EXPECT_TRUE(sv->getRequiredReadConsistency() == spi::ReadConsistency::WEAK); + auto session = make_visitor_session(params); + EXPECT_TRUE(session->visitor.getRequiredReadConsistency() == spi::ReadConsistency::WEAK); } } diff --git a/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp b/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp index 362d5a26611..babc6fbd697 100644 --- a/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/hitcollector.cpp @@ -13,6 +13,7 @@ LOG_SETUP(".searchvisitor.hitcollector"); using search::fef::MatchData; using vespalib::FeatureSet; +using vespalib::FeatureValues; using vdslib::SearchResult; using FefUtils = search::fef::Utils; @@ -126,7 +127,7 @@ HitCollector::addHit(Hit && hit) } void -HitCollector::fillSearchResult(vdslib::SearchResult & searchResult) +HitCollector::fillSearchResult(vdslib::SearchResult & searchResult, FeatureValues&& match_features) { sortByDocId(); for (const Hit & hit : _hits) { @@ -142,6 +143,13 @@ HitCollector::fillSearchResult(vdslib::SearchResult & searchResult) searchResult.addHit(docId, documentId.c_str(), rank, hit.getSortBlob().c_str(), hit.getSortBlob().size()); } } + searchResult.set_match_features(std::move(match_features)); +} + +void +HitCollector::fillSearchResult(vdslib::SearchResult & searchResult) +{ + fillSearchResult(searchResult, FeatureValues()); } FeatureSet::SP @@ -164,5 +172,28 @@ HitCollector::getFeatureSet(IRankProgram &rankProgram, return retval; } +FeatureValues +HitCollector::get_match_features(IRankProgram& rank_program, + const search::fef::FeatureResolver& resolver, + const search::StringStringMap& feature_rename_map) +{ + FeatureValues match_features; + if (resolver.num_features() == 0 || _hits.empty()) { + return match_features; + } + sortByDocId(); + match_features.names = FefUtils::extract_feature_names(resolver, feature_rename_map); + match_features.values.resize(resolver.num_features() * _hits.size()); + auto f = match_features.values.data(); + for (const Hit & hit : _hits) { + auto docid = hit.getDocId(); + rank_program.run(docid, hit.getMatchData()); + FefUtils::extract_feature_values(resolver, docid, f); + f += resolver.num_features(); + } + assert(f == match_features.values.data() + match_features.values.size()); + return match_features; +} + } // namespace streaming diff --git a/streamingvisitors/src/vespa/searchvisitor/hitcollector.h b/streamingvisitors/src/vespa/searchvisitor/hitcollector.h index 2918f815811..07418b85c75 100644 --- a/streamingvisitors/src/vespa/searchvisitor/hitcollector.h +++ b/streamingvisitors/src/vespa/searchvisitor/hitcollector.h @@ -121,6 +121,7 @@ public: * Fills the given search result with the m best hits from the hit heap. * Invoking this method will destroy the heap property of the hit heap. **/ + void fillSearchResult(vdslib::SearchResult & searchResult, vespalib::FeatureValues&& match_features); void fillSearchResult(vdslib::SearchResult & searchResult); /** @@ -136,6 +137,9 @@ public: const search::fef::FeatureResolver &resolver, const search::StringStringMap &feature_rename_map); + vespalib::FeatureValues get_match_features(IRankProgram& rank_program, + const search::fef::FeatureResolver& resolver, + const search::StringStringMap& feature_rename_map); }; } // namespace streaming diff --git a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp index 3ce137bffe5..55638c3ec44 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp @@ -11,6 +11,7 @@ LOG_SETUP(".searchvisitor.rankprocessor"); using vespalib::FeatureSet; +using vespalib::FeatureValues; using search::fef::FeatureHandle; using search::fef::ITermData; using search::fef::ITermFieldData; @@ -131,6 +132,10 @@ RankProcessor::init(bool forRanking, size_t wantedHitCount) _rankScore = getFeature(*_rankProgram); _summaryProgram = _rankSetup.create_summary_program(); setupRankProgram(*_summaryProgram); + if (_rankSetup.has_match_features()) { + _match_features_program = _rankSetup.create_match_program(); + setupRankProgram(*_match_features_program); + } } else { _rankProgram = _rankSetup.create_dump_program(); setupRankProgram(*_rankProgram); @@ -157,7 +162,8 @@ RankProcessor::RankProcessor(RankManager::Snapshot::SP snapshot, _summaryProgram(), _zeroScore(), _rankScore(&_zeroScore), - _hitCollector() + _hitCollector(), + _match_features_program() { } @@ -222,10 +228,21 @@ RankProcessor::calculateFeatureSet() return sf; } +FeatureValues +RankProcessor::calculate_match_features() +{ + if (!_match_features_program) { + return FeatureValues(); + } + RankProgramWrapper wrapper(*_match_data); + search::fef::FeatureResolver resolver(_match_features_program->get_seeds(false)); + return _hitCollector->get_match_features(wrapper, resolver, _rankSetup.get_feature_rename_map()); +} + void RankProcessor::fillSearchResult(vdslib::SearchResult & searchResult) { - _hitCollector->fillSearchResult(searchResult); + _hitCollector->fillSearchResult(searchResult, calculate_match_features()); } void diff --git a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h index c74a2d1e3ee..5307b66e1d5 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h +++ b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h @@ -23,6 +23,9 @@ namespace streaming { class RankProcessor { private: + using RankProgram = search::fef::RankProgram; + using FeatureSet = vespalib::FeatureSet; + using FeatureValues = vespalib::FeatureValues; RankManager::Snapshot::SP _rankManagerSnapshot; const search::fef::RankSetup & _rankSetup; QueryWrapper _query; @@ -37,10 +40,12 @@ private: search::fef::NumberOrObject _zeroScore; search::fef::LazyValue _rankScore; HitCollector::UP _hitCollector; + std::unique_ptr<RankProgram> _match_features_program; void initQueryEnvironment(); void initHitCollector(size_t wantedHitCount); void setupRankProgram(search::fef::RankProgram &program); + FeatureValues calculate_match_features(); /** * Initializes this rank processor. diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index 8980bc1f54d..105196a3249 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -238,7 +238,7 @@ void SearchVisitor::GroupingEntry::aggregate(const document::Document & doc, sea } SearchVisitor::~SearchVisitor() { - if (! isCompletedCalled()) { + if (!isCompletedCalled() && _queryResult) { HitCounter hc; completedVisitingInternal(hc); } @@ -1126,6 +1126,13 @@ void SearchVisitor::completedBucket(const document::BucketId&, HitCounter&) LOG(debug, "Completed bucket"); } +std::unique_ptr<documentapi::QueryResultMessage> +SearchVisitor::generate_query_result(HitCounter& counter) +{ + completedVisitingInternal(counter); + return std::move(_queryResult); +} + void SearchVisitor::completedVisitingInternal(HitCounter& hitCounter) { if (_vsmAdapter == nullptr) { diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h index 80df69f756e..d7c68162246 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h @@ -46,6 +46,10 @@ public: const vdslib::Parameters & params); ~SearchVisitor() override; + + // This should only be used by unit tests. + std::unique_ptr<documentapi::QueryResultMessage> generate_query_result(HitCounter& counter); + private: /** * This struct wraps an attribute vector. diff --git a/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.cpp index db4ee12438e..d4aee8ad652 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.cpp @@ -21,7 +21,6 @@ using search::attribute::Config; using search::fef::QueryValue; using search::tensor::DistanceCalculator; using search::tensor::TensorExtAttribute; -using search::tensor::make_distance_function; using vespalib::eval::ValueType; namespace { diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java index a695e10a29c..33991ef1a3b 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapper.java @@ -22,6 +22,7 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.time.Instant; import java.util.Base64; +import java.util.List; import java.util.Optional; import static com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId.fromDottedString; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocument.java index 577584db185..c7517ef8adb 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocument.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/IdentityDocument.java @@ -6,7 +6,9 @@ import com.yahoo.vespa.athenz.api.AthenzService; import java.time.Instant; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; /** diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentEntity.java index 946eacc67eb..194854cfc3b 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/IdentityDocumentEntity.java @@ -9,6 +9,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import java.time.Instant; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/RoleListEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/RoleListEntity.java new file mode 100644 index 00000000000..f785f19f8ea --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/RoleListEntity.java @@ -0,0 +1,12 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.identityprovider.api.bindings; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +public record RoleListEntity ( + @JsonProperty("roles")List<String> roles) { +} diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java index 513fb4cdbd3..a58debdb32f 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/api/EntityBindingsMapperTest.java @@ -7,9 +7,11 @@ import org.junit.jupiter.api.Test; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Base64; +import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -84,4 +86,6 @@ class EntityBindingsMapperTest { assertEquals(EntityBindingsMapper.mapper.readTree(originalJson), EntityBindingsMapper.mapper.readTree(json)); } + + }
\ No newline at end of file diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java index acb0905700f..276815f263d 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentSignerTest.java @@ -19,6 +19,7 @@ import java.security.KeyPair; import java.time.Instant; import java.util.Arrays; import java.util.HashSet; +import java.util.List; import static com.yahoo.vespa.athenz.identityprovider.api.IdentityType.TENANT; import static com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument.LEGACY_DEFAULT_DOCUMENT_VERSION; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index 1281bece66e..5d8d4576bad 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -1371,15 +1371,15 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { int status = Response.Status.BAD_GATEWAY; switch (code) { - case TIMEOUT: - if ( ! hasVisitedAnyBuckets() && parameters.getVisitInconsistentBuckets()) { + case TIMEOUT: // Intentional fallthrough. + case ABORTED: + if (error.get() == null && ! hasVisitedAnyBuckets() && parameters.getVisitInconsistentBuckets()) { response.writeMessage("No buckets visited within timeout of " + parameters.getSessionTimeoutMs() + "ms (request timeout -5s)"); status = Response.Status.GATEWAY_TIMEOUT; break; } - case SUCCESS: // Intentional fallthrough. - case ABORTED: // Intentional fallthrough. + case SUCCESS: if (error.get() == null) { ProgressToken progress = getProgress() != null ? getProgress() : parameters.getResumeToken(); if (progress != null && ! progress.isFinished()) diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java index 8f3fc27488f..dec546b4294 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java @@ -193,17 +193,18 @@ public class DocumentV1ApiTest { List<AckToken> tokens = List.of(new AckToken(null), new AckToken(null), new AckToken(null)); // GET at non-existent path returns 404 with available paths var response = driver.sendRequest("http://localhost/document/v1/not-found"); - assertSameJson("{" + - " \"pathId\": \"/document/v1/not-found\"," + - " \"message\": \"Nothing at '/document/v1/not-found'. Available paths are:\\n" + - "/document/v1/\\n" + - "/document/v1/{namespace}/{documentType}/docid/\\n" + - "/document/v1/{namespace}/{documentType}/group/{group}/\\n" + - "/document/v1/{namespace}/{documentType}/number/{number}/\\n" + - "/document/v1/{namespace}/{documentType}/docid/{*}\\n" + - "/document/v1/{namespace}/{documentType}/group/{group}/{*}\\n" + - "/document/v1/{namespace}/{documentType}/number/{number}/{*}\"" + - "}", response.readAll()); + assertSameJson(""" + { + "pathId": "/document/v1/not-found", + "message": "Nothing at '/document/v1/not-found'. Available paths are: + /document/v1/ + /document/v1/{namespace}/{documentType}/docid/ + /document/v1/{namespace}/{documentType}/group/{group}/ + /document/v1/{namespace}/{documentType}/number/{number}/ + /document/v1/{namespace}/{documentType}/docid/{*} + /document/v1/{namespace}/{documentType}/group/{group}/{*} + /document/v1/{namespace}/{documentType}/number/{number}/{*}" + }""", response.readAll()); assertEquals("application/json; charset=UTF-8", response.getResponse().headers().getFirst("Content-Type")); assertEquals(404, response.getStatus()); @@ -302,26 +303,27 @@ public class DocumentV1ApiTest { }); response = driver.sendRequest("http://localhost/document/v1?cluster=content&bucketSpace=default&wantedDocumentCount=1025&concurrency=123" + "&selection=all%20the%20things&fieldSet=[id]&timeout=6&stream=true&slices=4&sliceId=1"); - assertSameJson("{" + - " \"pathId\": \"/document/v1\"," + - " \"documents\": [" + - " {" + - " \"id\": \"id:space:music::one\"," + - " \"fields\": {" + - " \"artist\": \"Tom Waits\"," + - " \"embedding\": { \"type\": \"tensor(x[3])\", \"values\": [1.0,2.0,3.0] } " + - " }" + - " }," + - " {" + - " \"id\": \"id:space:music:n=1:two\"," + - " \"fields\": {" + - " \"artist\": \"Asa-Chan & Jun-Ray\"," + - " \"embedding\": { \"type\": \"tensor(x[3])\", \"values\": [4.0,5.0,6.0] } " + - " }" + - " }" + - " ]," + - " \"documentCount\": 2" + - "}", response.readAll()); + assertSameJson(""" + { + "pathId": "/document/v1", + "documents": [ + { + "id": "id:space:music::one", + "fields": { + "artist": "Tom Waits", + "embedding": { "type": "tensor(x[3])", "values": [1.0,2.0,3.0] } + } + }, + { + "id": "id:space:music:n=1:two", + "fields": { + "artist": "Asa-Chan & Jun-Ray", + "embedding": { "type": "tensor(x[3])", "values": [4.0,5.0,6.0] } + } + } + ], + "documentCount": 2 + }""", response.readAll()); assertEquals(200, response.getStatus()); // GET with namespace and document type is a restricted visit. @@ -334,10 +336,12 @@ public class DocumentV1ApiTest { throw new IllegalArgumentException("parse failure"); }); response = driver.sendRequest("http://localhost/document/v1/space/music/docid?continuation=" + progress.serializeToString()); - assertSameJson("{" + - " \"pathId\": \"/document/v1/space/music/docid\"," + - " \"message\": \"parse failure\"" + - "}", response.readAll()); + assertSameJson(""" + { + "pathId": "/document/v1/space/music/docid", + "message": "parse failure" + }""", + response.readAll()); assertEquals(400, response.getStatus()); // GET when a streamed visit returns status code 200 also when errors occur. @@ -347,12 +351,12 @@ public class DocumentV1ApiTest { parameters.getControlHandler().onDone(VisitorControlHandler.CompletionCode.FAILURE, "failure?"); }); response = driver.sendRequest("http://localhost/document/v1/space/music/docid?stream=true"); - assertSameJson("{" + - " \"pathId\": \"/document/v1/space/music/docid\"," + - " \"documents\": []," + - //" \"continuation\": \"" + progress.serializeToString() + "\"," + - " \"message\": \"failure?\"" + - "}", response.readAll()); + assertSameJson(""" + { + "pathId": "/document/v1/space/music/docid", + "documents": [], + "message": "failure?" + }""", response.readAll()); assertEquals(200, response.getStatus()); assertNull(response.getResponse().headers().get("X-Vespa-Ignored-Fields")); @@ -361,10 +365,12 @@ public class DocumentV1ApiTest { fail("Not supposed to run"); }); response = driver.sendRequest("http://localhost/document/v1/space/music/docid", POST); - assertSameJson("{" + - " \"pathId\": \"/document/v1/space/music/docid\"," + - " \"message\": \"Must specify 'destinationCluster' at '/document/v1/space/music/docid'\"" + - "}", response.readAll()); + assertSameJson(""" + { + "pathId": "/document/v1/space/music/docid", + "message": "Must specify 'destinationCluster' at '/document/v1/space/music/docid'" + }""", + response.readAll()); assertEquals(400, response.getStatus()); // POST with namespace and document type is a restricted visit with a required destination cluster ("destinationCluster") @@ -375,9 +381,11 @@ public class DocumentV1ApiTest { parameters.getControlHandler().onDone(VisitorControlHandler.CompletionCode.SUCCESS, "We made it!"); }); response = driver.sendRequest("http://localhost/document/v1/space/music/docid?destinationCluster=content&selection=true&cluster=content&timeout=60", POST); - assertSameJson("{" + - " \"pathId\": \"/document/v1/space/music/docid\"" + - "}", response.readAll()); + assertSameJson(""" + { + "pathId": "/document/v1/space/music/docid" + }""", + response.readAll()); assertEquals(200, response.getStatus()); // PUT with namespace and document type is a restricted visit with a required partial update to apply to visited documents. @@ -399,15 +407,18 @@ public class DocumentV1ApiTest { return new Result(); }); response = driver.sendRequest("http://localhost/document/v1/space/music/docid?selection=true&cluster=content&timeChunk=10", PUT, - "{" + - " \"fields\": {" + - " \"artist\": { \"assign\": \"Lisa Ekdahl\" }, " + - " \"nonexisting\": { \"assign\": \"Ignored\" }" + - " }" + - "}"); - assertSameJson("{" + - " \"pathId\": \"/document/v1/space/music/docid\"" + - "}", response.readAll()); + """ + { + "fields": { + "artist": { "assign": "Lisa Ekdahl" }, + "nonexisting": { "assign": "Ignored" } + } + }"""); + assertSameJson(""" + { + "pathId": "/document/v1/space/music/docid" + }""", + response.readAll()); assertEquals(200, response.getStatus()); assertEquals("true", response.getResponse().headers().get("X-Vespa-Ignored-Fields").get(0).toString()); @@ -416,10 +427,12 @@ public class DocumentV1ApiTest { fail("Not supposed to run"); }); response = driver.sendRequest("http://localhost/document/v1/space/music/group/troupe?selection=false", PUT); - assertSameJson("{" + - " \"pathId\": \"/document/v1/space/music/group/troupe\"," + - " \"message\": \"Must specify 'cluster' at '/document/v1/space/music/group/troupe'\"" + - "}", response.readAll()); + assertSameJson(""" + { + "pathId": "/document/v1/space/music/group/troupe", + "message": "Must specify 'cluster' at '/document/v1/space/music/group/troupe'" + }""", + response.readAll()); assertEquals(400, response.getStatus()); // PUT with namespace, document type and group is also a restricted visit which requires a selection. @@ -427,10 +440,12 @@ public class DocumentV1ApiTest { fail("Not supposed to run"); }); response = driver.sendRequest("http://localhost/document/v1/space/music/group/troupe?cluster=content", PUT); - assertSameJson("{" + - " \"pathId\": \"/document/v1/space/music/group/troupe\"," + - " \"message\": \"Must specify 'selection' at '/document/v1/space/music/group/troupe'\"" + - "}", response.readAll()); + assertSameJson(""" + { + "pathId": "/document/v1/space/music/group/troupe", + "message": "Must specify 'selection' at '/document/v1/space/music/group/troupe'" + }""", + response.readAll()); assertEquals(400, response.getStatus()); // DELETE with namespace and document type is a restricted visit which deletes visited documents. @@ -452,10 +467,12 @@ public class DocumentV1ApiTest { return new Result(); }); response = driver.sendRequest("http://localhost/document/v1/space/music/docid?selection=false&cluster=content", DELETE); - assertSameJson("{" + - " \"pathId\": \"/document/v1/space/music/docid\"," + - " \"message\": \"boom\"" + - "}", response.readAll()); + assertSameJson(""" + { + "pathId": "/document/v1/space/music/docid", + "message": "boom" + }""", + response.readAll()); assertEquals(502, response.getStatus()); // DELETE at the root is also a deletion visit. These also require a selection. @@ -463,10 +480,12 @@ public class DocumentV1ApiTest { fail("Not supposed to run"); }); response = driver.sendRequest("http://localhost/document/v1/", DELETE); - assertSameJson("{" + - " \"pathId\": \"/document/v1/\"," + - " \"message\": \"Must specify 'selection' at '/document/v1/'\"" + - "}", response.readAll()); + assertSameJson(""" + { + "pathId": "/document/v1/", + "message": "Must specify 'selection' at '/document/v1/'" + }""", + response.readAll()); assertEquals(400, response.getStatus()); // DELETE at the root is also a deletion visit. These also require a cluster. @@ -474,10 +493,12 @@ public class DocumentV1ApiTest { fail("Not supposed to run"); }); response = driver.sendRequest("http://localhost/document/v1/?selection=true", DELETE); - assertSameJson("{" + - " \"pathId\": \"/document/v1/\"," + - " \"message\": \"Must specify 'cluster' at '/document/v1/'\"" + - "}", response.readAll()); + assertSameJson(""" + { + "pathId": "/document/v1/", + "message": "Must specify 'cluster' at '/document/v1/'" + }""", + response.readAll()); assertEquals(400, response.getStatus()); // GET with namespace, document type and group is a restricted visit. @@ -486,32 +507,43 @@ public class DocumentV1ApiTest { parameters.getControlHandler().onDone(VisitorControlHandler.CompletionCode.FAILURE, "error"); }); response = driver.sendRequest("http://localhost/document/v1/space/music/group/best%27"); - assertSameJson("{" + - " \"pathId\": \"/document/v1/space/music/group/best%27\"," + - " \"documents\": []," + - " \"message\": \"error\"" + - "}", response.readAll()); + assertSameJson(""" + { + "pathId": "/document/v1/space/music/group/best%27", + "documents": [], + "message": "error" + }""", + response.readAll()); assertEquals(502, response.getStatus()); // GET with namespace, document type and number is a restricted visit. access.expect(parameters -> { assertEquals("(music) and (id.namespace=='space') and (id.user==123)", parameters.getDocumentSelection()); + VisitorStatistics statistics = new VisitorStatistics(); + statistics.setBucketsVisited(1); + statistics.setDocumentsVisited(0); + parameters.getControlHandler().onVisitorStatistics(statistics); parameters.getControlHandler().onDone(VisitorControlHandler.CompletionCode.ABORTED, "aborted"); }); response = driver.sendRequest("http://localhost/document/v1/space/music/number/123"); - assertSameJson("{" + - " \"pathId\": \"/document/v1/space/music/number/123\"," + - " \"documents\": []" + - "}", response.readAll()); + assertSameJson(""" + { + "pathId": "/document/v1/space/music/number/123", + "documents": [ ], + "documentCount": 0 + }""", + response.readAll()); assertEquals(200, response.getStatus()); // GET with from timestamp > to timestamp is an error access.expect(parameters -> { fail("unreachable"); }); response = driver.sendRequest("http://localhost/document/v1/?cluster=content&fromTimestamp=100&toTimestamp=99"); - assertSameJson("{" + - " \"pathId\": \"/document/v1/\"," + - " \"message\": \"toTimestamp must be greater than, or equal to, fromTimestamp\"" + - "}", response.readAll()); + assertSameJson(""" + { + "pathId": "/document/v1/", + "message": "toTimestamp must be greater than, or equal to, fromTimestamp" + }""", + response.readAll()); assertEquals(400, response.getStatus()); // GET with full document ID is a document get operation which returns 404 when no document is found @@ -522,10 +554,12 @@ public class DocumentV1ApiTest { return new Result(); }); response = driver.sendRequest("http://localhost/document/v1/space/music/docid/one?cluster=content&fieldSet=go&timeout=123"); - assertSameJson("{" + - " \"pathId\": \"/document/v1/space/music/docid/one\"," + - " \"id\": \"id:space:music::one\"" + - "}", response.readAll()); + assertSameJson(""" + { + "pathId": "/document/v1/space/music/docid/one", + "id": "id:space:music::one" + }""", + response.readAll()); assertEquals(404, response.getStatus()); // GET with full document ID is a document get operation. diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/MatchFeatureData.java b/vespajlib/src/main/java/com/yahoo/data/access/helpers/MatchFeatureData.java index 7781a01b781..f7d72ade20d 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/MatchFeatureData.java +++ b/vespajlib/src/main/java/com/yahoo/data/access/helpers/MatchFeatureData.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.dispatch.rpc; +package com.yahoo.data.access.helpers; import com.yahoo.collections.Hashlet; @@ -20,11 +20,11 @@ import java.util.Map; * for the field names. * @author arnej */ -class MatchFeatureData { +public class MatchFeatureData { private final Hashlet<String,Integer> hashlet; - MatchFeatureData(List<String> keys) { + public MatchFeatureData(List<String> keys) { this.hashlet = new Hashlet<>(); hashlet.reserve(keys.size()); int i = 0; @@ -33,7 +33,7 @@ class MatchFeatureData { } } - static class HitValue extends Value { + public static class HitValue extends Value { private final Hashlet<String,Integer> hashlet; private final byte[][] dataValues; private final double[] doubleValues; @@ -72,10 +72,10 @@ class MatchFeatureData { this.doubleValues = new double[hashlet.size()]; } - void set(int index, byte[] data) { + public void set(int index, byte[] data) { dataValues[index] = data; } - void set(int index, double value) { + public void set(int index, double value) { doubleValues[index] = value; } @@ -87,7 +87,7 @@ class MatchFeatureData { } } - HitValue addHit() { + public HitValue addHit() { return new HitValue(hashlet); } diff --git a/vespajlib/src/main/java/com/yahoo/data/access/helpers/package-info.java b/vespajlib/src/main/java/com/yahoo/data/access/helpers/package-info.java new file mode 100644 index 00000000000..96c12b63c69 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/data/access/helpers/package-info.java @@ -0,0 +1,5 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.data.access.helpers; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MatchFeatureDataTest.java b/vespajlib/src/test/java/com/yahoo/data/access/helpers/MatchFeatureDataTest.java index 861ed4d165d..22f71107ec3 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/rpc/MatchFeatureDataTest.java +++ b/vespajlib/src/test/java/com/yahoo/data/access/helpers/MatchFeatureDataTest.java @@ -1,6 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.dispatch.rpc; +package com.yahoo.data.access.helpers; import com.yahoo.data.access.ArrayTraverser; import com.yahoo.data.access.Inspector; |