diff options
60 files changed, 1953 insertions, 526 deletions
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java index 158df654439..7b8deb19831 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java @@ -19,12 +19,15 @@ import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3; import java.util.Arrays; import java.util.Iterator; +import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; +import static com.yahoo.vespa.config.ErrorCode.INTERNAL_ERROR; + /** * An RPC server that handles config and file distribution requests. * @@ -247,25 +250,26 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher { private void getConfigImpl(JRTServerConfigRequest request) { ResponseHandler responseHandler = new ResponseHandler(); request.getRequestTrace().trace(TRACELEVEL, "Config proxy getConfig()"); - log.log(Level.FINE, () ->"getConfig: " + request.getShortDescription() + ",config checksums=" + request.getRequestConfigChecksums()); + log.log(Level.FINE, () ->"getConfig: " + request); if (!request.validateParameters()) { - // Error code is set in verifyParameters if parameters are not OK. - log.log(Level.WARNING, "Parameters for request " + request + " did not validate: " + request.errorCode() + " : " + request.errorMessage()); - responseHandler.returnErrorResponse(request, request.errorCode(), "Parameters for request " + request.getShortDescription() + " did not validate: " + request.errorMessage()); + // Error code is set in validateParameters if parameters are not OK. + log.log(Level.WARNING, "Invalid parameters for request " + request + ": " + request.errorCode() + " : " + request.errorMessage()); + responseHandler.returnErrorResponse(request, request.errorCode(), "Invalid parameters for request " + request + + ": " + request.errorMessage()); return; } + try { - RawConfig config = proxyServer.resolveConfig(request); - if (config == null) { - log.log(Level.FINEST, () -> "No config received yet for " + request.getShortDescription() + ", not sending response"); - } else if (ProxyServer.configOrGenerationHasChanged(config, request)) { - responseHandler.returnOkResponse(request, config); - } else { - log.log(Level.FINEST, () -> "No new config for " + request.getShortDescription() + ", not sending response"); - } + Optional<RawConfig> config = proxyServer.resolveConfig(request); + if (config.isEmpty()) + log.log(Level.FINEST, () -> "No config received yet for " + request + ", not sending response"); + else if (ProxyServer.configOrGenerationHasChanged(config.get(), request)) + responseHandler.returnOkResponse(request, config.get()); + else + log.log(Level.FINEST, () -> "No new config for " + request + ", not sending response"); } catch (Exception e) { - e.printStackTrace(); - responseHandler.returnErrorResponse(request, com.yahoo.vespa.config.ErrorCode.INTERNAL_ERROR, e.getMessage()); + log.log(Level.WARNING, "Resolving config " + request + " failed", e); + responseHandler.returnErrorResponse(request, INTERNAL_ERROR, e.getMessage()); } } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java index dae732e56ec..d771d440078 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java @@ -5,6 +5,7 @@ import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import java.util.List; +import java.util.Optional; /** * A client to a config source, which could be an RPC config server or some other backing for @@ -14,7 +15,7 @@ import java.util.List; */ interface ConfigSourceClient { - RawConfig getConfig(RawConfig input, JRTServerConfigRequest request); + Optional<RawConfig> getConfig(RawConfig input, JRTServerConfigRequest request); void shutdown(); diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java index 0e8ebe0d9c9..cae70b41c8c 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import com.yahoo.yolean.Exceptions; +import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; @@ -47,9 +48,9 @@ public class DelayedResponseHandler implements Runnable { while ((response = delayedResponses.responses().poll()) != null) { JRTServerConfigRequest request = response.getRequest(); ConfigCacheKey cacheKey = new ConfigCacheKey(request.getConfigKey(), request.getRequestDefMd5()); - RawConfig config = memoryCache.get(cacheKey); - if (config != null) { - responseHandler.returnOkResponse(request, config); + Optional<RawConfig> config = memoryCache.get(cacheKey); + if (config.isPresent()) { + responseHandler.returnOkResponse(request, config.get()); sentResponses.incrementAndGet(); } else { log.log(Level.WARNING, "Timed out (timeout " + request.getTimeout() + ") getting config " + diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCache.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCache.java index e2ab5de40e9..ff10adf88e9 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCache.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCache.java @@ -14,6 +14,7 @@ import java.io.IOException; import java.io.Writer; import java.nio.file.Files; import java.util.Collection; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; import java.util.logging.Logger; @@ -31,8 +32,8 @@ public class MemoryCache { private final ConcurrentHashMap<ConfigCacheKey, RawConfig> cache = new ConcurrentHashMap<>(500, 0.75f); - public RawConfig get(ConfigCacheKey key) { - return cache.get(key); + public Optional<RawConfig> get(ConfigCacheKey key) { + return Optional.ofNullable(cache.get(key)); } /** diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java index f1be03f07d4..d207fcce639 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; @@ -33,16 +34,13 @@ class MemoryCacheConfigClient implements ConfigSourceClient { * @return A Config with a payload. */ @Override - public RawConfig getConfig(RawConfig input, JRTServerConfigRequest request) { - log.log(Level.FINE, () -> "Getting config from cache"); + public Optional<RawConfig> getConfig(RawConfig input, JRTServerConfigRequest request) { ConfigKey<?> key = input.getKey(); - RawConfig cached = cache.get(new ConfigCacheKey(key, input.getDefMd5())); - if (cached != null) { + + Optional<RawConfig> cached = cache.get(new ConfigCacheKey(key, input.getDefMd5())); + if (cached.isPresent()) log.log(Level.FINE, () -> "Found config " + key + " in cache"); - return cached; - } else { - return null; - } + return cached; } @Override diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java index 9340363def8..a559f3025ed 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java @@ -15,6 +15,7 @@ import com.yahoo.yolean.system.CatchSignals; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -71,11 +72,11 @@ public class ProxyServer implements Runnable { } } - RawConfig resolveConfig(JRTServerConfigRequest req) { + Optional<RawConfig> resolveConfig(JRTServerConfigRequest req) { // Calling getConfig() will either return with an answer immediately or // create a background thread that retrieves config from the server and - // calls updateSubscribers when new config is returned from the config source. - // In the last case the method below will return null. + // calls updateSubscribers when new config is returned from the config server. + // In the last case the method below will return empty. return configClient.getConfig(RawConfig.createFromServerRequest(req), req); } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java index f022ed11f3d..8f844fdd3ee 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java @@ -110,7 +110,7 @@ class RpcConfigSourceClient implements ConfigSourceClient, Runnable { * @return A Config with a payload. */ @Override - public RawConfig getConfig(RawConfig input, JRTServerConfigRequest request) { + public Optional<RawConfig> getConfig(RawConfig input, JRTServerConfigRequest request) { // Always add to delayed responses (we remove instead if we find config in cache) // This is to avoid a race where we might end up not adding to delayed responses // nor subscribing to config if another request for the same config @@ -119,29 +119,29 @@ class RpcConfigSourceClient implements ConfigSourceClient, Runnable { delayedResponses.add(delayedResponse); ConfigCacheKey configCacheKey = new ConfigCacheKey(input.getKey(), input.getDefMd5()); - RawConfig cachedConfig = memoryCache.get(configCacheKey); + Optional<RawConfig> cachedConfig = memoryCache.get(configCacheKey); boolean needToGetConfig = true; - RawConfig ret = null; - if (cachedConfig != null) { - log.log(Level.FINE, () -> "Found config " + configCacheKey + " in cache, generation=" + cachedConfig.getGeneration() + - ",config checksums=" + cachedConfig.getPayloadChecksums()); - log.log(Level.FINEST, () -> "input config=" + input + ",cached config=" + cachedConfig); - if (ProxyServer.configOrGenerationHasChanged(cachedConfig, request)) { + if (cachedConfig.isPresent()) { + RawConfig config = cachedConfig.get(); + log.log(Level.FINE, () -> "Found config " + configCacheKey + " in cache, generation=" + config.getGeneration() + + ",config checksums=" + config.getPayloadChecksums()); + log.log(Level.FINEST, () -> "input config=" + input + ",cached config=" + config); + if (ProxyServer.configOrGenerationHasChanged(config, request)) { log.log(Level.FINEST, () -> "Cached config is not equal to requested, will return it"); if (delayedResponses.remove(delayedResponse)) { // unless another thread already did it - ret = cachedConfig; + return cachedConfig; } } - if (!cachedConfig.isError() && cachedConfig.getGeneration() > 0) { + if (!config.isError() && config.getGeneration() > 0) { needToGetConfig = false; } } if (needToGetConfig) { subscribeToConfig(input, configCacheKey); } - return ret; + return Optional.empty(); } private void subscribeToConfig(RawConfig input, ConfigCacheKey configCacheKey) { diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClientTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClientTest.java index f84d447e3a1..d243d9c6dff 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClientTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClientTest.java @@ -6,7 +6,7 @@ import org.junit.Test; import java.util.List; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; /** * @author hmusum @@ -17,8 +17,8 @@ public class MemoryCacheConfigClientTest { public void basic() { MemoryCacheConfigClient client = new MemoryCacheConfigClient(new MemoryCache()); client.memoryCache().update(ConfigTester.fooConfig); - assertEquals(ConfigTester.fooConfig, client.getConfig(ConfigTester.fooConfig, null)); - assertNull(client.getConfig(ConfigTester.barConfig, null)); + assertEquals(ConfigTester.fooConfig, client.getConfig(ConfigTester.fooConfig, null).orElseThrow()); + assertTrue(client.getConfig(ConfigTester.barConfig, null).isEmpty()); assertEquals("N/A", client.getActiveSourceConnection()); assertEquals(List.of("N/A"), client.getSourceConnections()); diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MemoryCacheTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MemoryCacheTest.java index 1a919ad3988..b0cba728a0c 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MemoryCacheTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MemoryCacheTest.java @@ -15,7 +15,6 @@ import java.util.ArrayList; import java.util.Optional; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; /** @@ -84,14 +83,12 @@ public class MemoryCacheTest { assertTrue(cache.containsKey(cacheKey)); assertTrue(cache.containsKey(cacheKey2)); - RawConfig response = cache.get(cacheKey); - assertNotNull(response); + RawConfig response = cache.get(cacheKey).orElseThrow(); assertEquals(defName, response.getName()); assertEquals(payload.toString(), response.getPayload().toString()); assertEquals(generation, response.getGeneration()); - response = cache.get(cacheKey2); - assertNotNull(response); + response = cache.get(cacheKey2).orElseThrow(); assertEquals(defName2, response.getName()); assertEquals(payload2.toString(), response.getPayload().toString()); assertEquals(generation, response.getGeneration()); @@ -108,14 +105,12 @@ public class MemoryCacheTest { assertEquals(2, cache.size()); assertTrue(cache.containsKey(cacheKey)); - RawConfig response = cache.get(cacheKey); - assertNotNull(response); + RawConfig response = cache.get(cacheKey).orElseThrow(); assertEquals(defName, response.getName()); assertEquals(payload.getData(), response.getPayload().getData()); assertEquals(generation, response.getGeneration()); - response = cache.get(cacheKeyDifferentMd5); - assertNotNull(response); + response = cache.get(cacheKeyDifferentMd5).orElseThrow(); assertEquals(defName, response.getName()); assertEquals(payloadDifferentMd5.getData(), response.getPayload().getData()); assertEquals(generation, response.getGeneration()); diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java index d0724b9dbd0..e02ca22d0a0 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import java.util.Collections; import java.util.List; +import java.util.Optional; /** * Mock client that always returns with config immediately @@ -24,10 +25,10 @@ public class MockConfigSourceClient implements ConfigSourceClient{ } @Override - public RawConfig getConfig(RawConfig input, JRTServerConfigRequest request) { - final RawConfig config = getConfig(input.getKey()); + public Optional<RawConfig> getConfig(RawConfig input, JRTServerConfigRequest request) { + RawConfig config = getConfig(input.getKey()); memoryCache.update(config); - return config; + return Optional.of(config); } private RawConfig getConfig(ConfigKey<?> configKey) { diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java index 09eae6a297d..1bcd1562a85 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java @@ -62,11 +62,10 @@ public class ProxyServerTest { ConfigTester tester = new ConfigTester(); MemoryCache memoryCache = proxy.memoryCache(); assertEquals(0, memoryCache.size()); - RawConfig res = proxy.resolveConfig(tester.createRequest(fooConfig)); - assertNotNull(res); + RawConfig res = proxy.resolveConfig(tester.createRequest(fooConfig)).orElseThrow(); assertEquals(ConfigTester.fooPayload.toString(), res.getPayload().toString()); assertEquals(1, memoryCache.size()); - assertEquals(res, memoryCache.get(new ConfigCacheKey(fooConfig.getKey(), fooConfig.getDefMd5()))); + assertEquals(res, memoryCache.get(new ConfigCacheKey(fooConfig.getKey(), fooConfig.getDefMd5())).orElseThrow()); } /** @@ -111,15 +110,14 @@ public class ProxyServerTest { ConfigTester tester = new ConfigTester(); MemoryCache memoryCache = proxy.memoryCache(); assertEquals(0, memoryCache.size()); - RawConfig res = proxy.resolveConfig(tester.createRequest(fooConfig)); - assertNotNull(res); + RawConfig res = proxy.resolveConfig(tester.createRequest(fooConfig)).orElseThrow(); assertEquals(ConfigTester.fooPayload.toString(), res.getPayload().toString()); assertEquals(1, memoryCache.size()); - assertEquals(res, memoryCache.get(new ConfigCacheKey(fooConfig.getKey(), fooConfig.getDefMd5()))); + assertEquals(res, memoryCache.get(new ConfigCacheKey(fooConfig.getKey(), fooConfig.getDefMd5())).orElseThrow()); // Trying same config again JRTServerConfigRequest newRequestBasedOnResponse = tester.createRequest(res); - RawConfig res2 = proxy.resolveConfig(newRequestBasedOnResponse); + RawConfig res2 = proxy.resolveConfig(newRequestBasedOnResponse).orElseThrow(); assertFalse(ProxyServer.configOrGenerationHasChanged(res2, newRequestBasedOnResponse)); assertEquals(1, memoryCache.size()); } @@ -137,8 +135,7 @@ public class ProxyServerTest { MemoryCache memoryCache = proxy.memoryCache(); assertEquals(0, memoryCache.size()); - RawConfig res = proxy.resolveConfig(tester.createRequest(fooConfig)); - assertNotNull(res); + RawConfig res = proxy.resolveConfig(tester.createRequest(fooConfig)).orElseThrow(); assertNotNull(res.getPayload()); assertTrue(res.isError()); assertEquals(0, memoryCache.size()); @@ -148,14 +145,13 @@ public class ProxyServerTest { source.put(fooConfig.getKey(), createConfigWithNextConfigGeneration(fooConfig, 0)); // Verify that we get the config now and that it is cached - res = proxy.resolveConfig(tester.createRequest(fooConfig)); - assertNotNull(res); + res = proxy.resolveConfig(tester.createRequest(fooConfig)).orElseThrow(); assertNotNull(res.getPayload().getData()); assertEquals(ConfigTester.fooPayload.toString(), res.getPayload().toString()); assertEquals(1, memoryCache.size()); JRTServerConfigRequest newRequestBasedOnResponse = tester.createRequest(res); - RawConfig res2 = proxy.resolveConfig(newRequestBasedOnResponse); + RawConfig res2 = proxy.resolveConfig(newRequestBasedOnResponse).orElseThrow(); assertFalse(ProxyServer.configOrGenerationHasChanged(res2, newRequestBasedOnResponse)); assertEquals(1, memoryCache.size()); } @@ -172,7 +168,7 @@ public class ProxyServerTest { MemoryCache cache = proxy.memoryCache(); assertEquals(0, cache.size()); - RawConfig res = proxy.resolveConfig(tester.createRequest(fooConfig)); + RawConfig res = proxy.resolveConfig(tester.createRequest(fooConfig)).orElseThrow(); assertNotNull(res); assertEquals(ConfigTester.fooPayload.toString(), res.getPayload().toString()); assertEquals(1, cache.size()); @@ -183,8 +179,7 @@ public class ProxyServerTest { 0, fooConfig.getDefContent(), Optional.empty()); source.put(fooConfig.getKey(), emptyConfig); - res = proxy.resolveConfig(tester.createRequest(fooConfig)); - assertNotNull(res.getPayload()); + res = proxy.resolveConfig(tester.createRequest(fooConfig)).orElseThrow(); assertEquals(emptyConfig.getPayload().toString(), res.getPayload().toString()); assertEquals(0, cache.size()); @@ -193,7 +188,7 @@ public class ProxyServerTest { source.put(fooConfig.getKey(), createConfigWithNextConfigGeneration(fooConfig, 0)); // Verify that we get the config now and that it is cached - res = proxy.resolveConfig(tester.createRequest(fooConfig)); + res = proxy.resolveConfig(tester.createRequest(fooConfig)).orElseThrow(); assertNotNull(res.getPayload().getData()); assertEquals(ConfigTester.fooPayload.toString(), res.getPayload().toString()); assertEquals(1, cache.size()); @@ -202,15 +197,14 @@ public class ProxyServerTest { @Test public void testReconfiguration() { ConfigTester tester = new ConfigTester(); - RawConfig res = proxy.resolveConfig(tester.createRequest(fooConfig)); - assertNotNull(res); + RawConfig res = proxy.resolveConfig(tester.createRequest(fooConfig)).orElseThrow(); assertEquals(ConfigTester.fooPayload.toString(), res.getPayload().toString()); // Simulate deployment, add config with new config generation long previousGeneration = res.getGeneration(); source.put(fooConfig.getKey(), createConfigWithNextConfigGeneration(res, 0)); JRTServerConfigRequest newRequestBasedOnResponse = tester.createRequest(res); - RawConfig res2 = proxy.resolveConfig(newRequestBasedOnResponse); + RawConfig res2 = proxy.resolveConfig(newRequestBasedOnResponse).orElseThrow(); assertEquals(previousGeneration + 1, res2.getGeneration()); assertTrue(ProxyServer.configOrGenerationHasChanged(res2, newRequestBasedOnResponse)); } diff --git a/config/src/tests/payload_converter/payload_converter.cpp b/config/src/tests/payload_converter/payload_converter.cpp index d5212048b2e..a36702434c4 100644 --- a/config/src/tests/payload_converter/payload_converter.cpp +++ b/config/src/tests/payload_converter/payload_converter.cpp @@ -3,6 +3,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/common/payload_converter.h> #include <vespa/vespalib/data/slime/slime.h> +#include <algorithm> #include <vespa/log/log.h> LOG_SETUP("payload_converter"); diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index bb8317c298b..78a1c044d35 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -3193,7 +3193,8 @@ "public int hashCode()", "public boolean equals(java.lang.Object)", "public java.lang.String toString()", - "public java.lang.String getLowerCasedName()" + "public java.lang.String getLowerCasedName()", + "public static com.yahoo.processing.request.CompoundName from(java.lang.String)" ], "fields": [ "public static final com.yahoo.processing.request.CompoundName empty" diff --git a/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java b/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java index d571bf583d6..efed58f4ab0 100644 --- a/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java +++ b/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java @@ -285,4 +285,6 @@ public final class CompoundName { return b.length()==0 ? "" : b.substring(0, b.length()-1); } + public static CompoundName from(String name) { return new CompoundName(name); } + } diff --git a/container-core/src/main/java/com/yahoo/processing/request/Properties.java b/container-core/src/main/java/com/yahoo/processing/request/Properties.java index 08072e83ce4..b4a619f11fb 100644 --- a/container-core/src/main/java/com/yahoo/processing/request/Properties.java +++ b/container-core/src/main/java/com/yahoo/processing/request/Properties.java @@ -43,56 +43,42 @@ public class Properties implements Cloneable { return chained.getInstance(propertyClass); } - /** - * Lists all properties of this with no context, by delegating to listProperties("") - */ + /** Lists all properties of this with no context, by delegating to listProperties(""). */ public final Map<String, Object> listProperties() { return listProperties(CompoundName.empty); } - /** - * Returns a snapshot of all properties of this - same as listProperties("",context) - */ + /** Returns a snapshot of all properties of this - same as listProperties("", context). */ public final Map<String, Object> listProperties(Map<String, String> context) { return listProperties(CompoundName.empty, context, this); } - /** - * Returns a snapshot of all properties by calling listProperties(path,null) - */ + /** Returns a snapshot of all properties by calling listProperties(path, null). */ public final Map<String, Object> listProperties(CompoundName path) { return listProperties(path, null, this); } - /** - * Returns a snapshot of all properties by calling listProperties(path,null) - */ + /** Returns a snapshot of all properties by calling listProperties(path, null). */ public final Map<String, Object> listProperties(String path) { return listProperties(new CompoundName(path), null, this); } - /** - * Returns a snapshot of all properties by calling listProperties(path,null) - */ + /** Returns a snapshot of all properties by calling listProperties(path, null). */ public final Map<String, Object> listProperties(CompoundName path, Map<String, String> context) { return listProperties(path, context, this); } - /** - * Returns a snapshot of all properties by calling listProperties(path,null) - */ + /** Returns a snapshot of all properties by calling listProperties(path, null). */ public final Map<String, Object> listProperties(String path, Map<String, String> context) { return listProperties(new CompoundName(path), context, this); } /** * Returns a snapshot of all properties of this having a given path prefix - * <p> - * Some sources of properties may not be list-able (e.g those using reflection) - * and will not be included in this snapshot. - * + * Some sources of properties may not be list-able and will not be included in this snapshot. * - * @param path the prefix (up to a ".") of the properties to return, or null or the empty string to return all properties + * @param path the prefix (up to a ".") of the properties to return, or null or the empty string + * to return all properties * @param context the context used to resolve the properties, or null if none * @param substitution the properties which will be used to do string substitution in the values added to the map */ @@ -107,10 +93,7 @@ public class Properties implements Cloneable { /** * Returns a snapshot of all properties of this having a given path prefix - * <p> - * Some sources of properties may not be list-able (e.g those using reflection) - * and will not be included in this snapshot. - * + * Some sources of properties may not be list-able and will not be included in this snapshot. * * @param path the prefix (up to a ".") of the properties to return, or null or the empty string to return all properties * @param context the context used to resolve the properties, or null if none @@ -133,7 +116,7 @@ public class Properties implements Cloneable { } /** - * Gets a named value which (if necessary) is resolved using a property context + * Gets a named value which (if necessary) is resolved using a property context. * * @param name the name of the property to return * @param context the variant resolution context, or null if none @@ -143,30 +126,22 @@ public class Properties implements Cloneable { return get(new CompoundName(name), context, substitution); } - /** - * Gets a named value from the first chained instance which has one by calling get(name,context,this) - */ + /** Gets a named value from the first chained instance which has one by calling get(name,context,this). */ public final Object get(CompoundName name, Map<String, String> context) { return get(name, context, this); } - /** - * Gets a named value from the first chained instance which has one by calling get(name,context,this) - */ + /** Gets a named value from the first chained instance which has one by calling get(name,context,this). */ public final Object get(String name, Map<String, String> context) { return get(new CompoundName(name), context, this); } - /** - * Gets a named value from the first chained instance which has one by calling get(name,null,this) - */ + /** Gets a named value from the first chained instance which has one by calling get(name,null,this). */ public final Object get(CompoundName name) { return get(name, null, this); } - /** - * Gets a named value from the first chained instance which has one by calling get(name,null,this) - */ + /** Gets a named value from the first chained instance which has one by calling get(name,null,this). */ public final Object get(String name) { return get(new CompoundName(name), null, this); } @@ -174,9 +149,8 @@ public class Properties implements Cloneable { /** * Gets a named value from the first chained instance which has one, * or the default value if no value is set, or if the first value encountered is explicitly set to null. - * <p> - * This default implementation simply forwards to the chained instance, or returns the default if none * + * This default implementation simply forwards to the chained instance, or returns the default if none. * * @param name the name of the property to return * @param defaultValue the default value returned if the value returned is null @@ -190,8 +164,8 @@ public class Properties implements Cloneable { /** * Gets a named value from the first chained instance which has one, * or the default value if no value is set, or if the first value encountered is explicitly set to null. - * <p> - * This default implementation simply forwards to the chained instance, or returns the default if none + * + * This default implementation simply forwards to the chained instance, or returns the default if none. * * @param name the name of the property to return * @param defaultValue the default value returned if the value returned is null @@ -202,7 +176,7 @@ public class Properties implements Cloneable { /** * Sets a value to the first chained instance which accepts it. - * <p> + * * This default implementation forwards to the chained instance or throws * a RuntimeException if there is not chained instance. * @@ -219,7 +193,7 @@ public class Properties implements Cloneable { /** * Sets a value to the first chained instance which accepts it. - * <p> + * * This default implementation forwards to the chained instance or throws * a RuntimeException if there is not chained instance. * @@ -233,7 +207,7 @@ public class Properties implements Cloneable { } /** - * Sets a value to the first chained instance which accepts it by calling set(name,value,null). + * Sets a value to the first chained instance which accepts it by calling set(name, value, null). * * @param name the name of the property * @param value the value to set. Setting a property to null clears it. @@ -244,7 +218,7 @@ public class Properties implements Cloneable { } /** - * Sets a value to the first chained instance which accepts it by calling set(name,value,null). + * Sets a value to the first chained instance which accepts it by calling set(name, value, null). * * @param name the name of the property * @param value the value to set. Setting a property to null clears it. @@ -364,7 +338,7 @@ public class Properties implements Cloneable { } /** - * Returns this property as a string + * Returns this property as a string. * * @return this property as a string, or null if the property is null */ @@ -373,7 +347,7 @@ public class Properties implements Cloneable { } /** - * Returns this property as a string + * Returns this property as a string. * * @return this property as a string, or null if the property is null */ @@ -382,7 +356,7 @@ public class Properties implements Cloneable { } /** - * Returns this property as a string + * Returns this property as a string. * * @param key the property key * @param defaultValue the value to return if this property is null @@ -393,7 +367,7 @@ public class Properties implements Cloneable { } /** - * Returns this property as a string + * Returns this property as a string. * * @param key the property key * @param defaultValue the value to return if this property is null @@ -409,7 +383,7 @@ public class Properties implements Cloneable { } /** - * Returns a property as an Integer + * Returns a property as an Integer. * * @return the integer value of the name, or null if the property is null * @throws NumberFormatException if the given parameter exists but @@ -420,7 +394,7 @@ public class Properties implements Cloneable { } /** - * Returns a property as an Integer + * Returns a property as an Integer. * * @return the integer value of the name, or null if the property is null * @throws NumberFormatException if the given parameter exists but @@ -431,7 +405,7 @@ public class Properties implements Cloneable { } /** - * Returns a property as an Integer + * Returns a property as an Integer. * * @param name the property name * @param defaultValue the value to return if this property is null @@ -444,7 +418,7 @@ public class Properties implements Cloneable { } /** - * Returns a property as an Integer + * Returns a property as an Integer. * * @param name the property name * @param defaultValue the value to return if this property is null @@ -475,7 +449,7 @@ public class Properties implements Cloneable { } /** - * Returns a property as a Long + * Returns a property as a Long. * * @return the long value of the name, or null if the property is null * @throws NumberFormatException if the given parameter exists but have a value which @@ -486,7 +460,7 @@ public class Properties implements Cloneable { } /** - * Returns a property as a Long + * Returns a property as a Long. * * @return the long value of the name, or null if the property is null * @throws NumberFormatException if the given parameter exists but have a value which @@ -497,7 +471,7 @@ public class Properties implements Cloneable { } /** - * Returns a property as a Long + * Returns a property as a Long. * * @param name the property name * @param defaultValue the value to return if this property is null @@ -510,7 +484,7 @@ public class Properties implements Cloneable { } /** - * Returns a property as a Long + * Returns a property as a Long. * * @param name the property name * @param defaultValue the value to return if this property is null @@ -541,7 +515,7 @@ public class Properties implements Cloneable { } /** - * Returns a property as a Double + * Returns a property as a Double. * * @return the double value of the name, or null if the property is null * @throws NumberFormatException if the given parameter exists but have a value which @@ -552,7 +526,7 @@ public class Properties implements Cloneable { } /** - * Returns a property as a Double + * Returns a property as a Double. * * @return the double value of the name, or null if the property is null * @throws NumberFormatException if the given parameter exists but have a value which @@ -563,7 +537,7 @@ public class Properties implements Cloneable { } /** - * Returns a property as a Double + * Returns a property as a Double. * * @param name the property name * @param defaultValue the value to return if this property is null @@ -576,7 +550,7 @@ public class Properties implements Cloneable { } /** - * Returns a property as a Double + * Returns a property as a Double. * * @param name the property name * @param defaultValue the value to return if this property is null @@ -626,7 +600,7 @@ public class Properties implements Cloneable { return cloneHelper.cloneMap(map); } - /** Clones this object if it is clonable, and the clone is public. Returns null if not */ + /** Clones this object if it is clonable, and the clone is public. Returns null if not. */ public static Object clone(Object object) { return cloneHelper.clone(object); } diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 67bc553c478..2ab51f975a4 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -1905,6 +1905,8 @@ "public java.util.Map getEmbedders()", "public com.yahoo.search.Query$Builder setZoneInfo(ai.vespa.cloud.ZoneInfo)", "public ai.vespa.cloud.ZoneInfo getZoneInfo()", + "public com.yahoo.search.Query$Builder setSchemaInfo(com.yahoo.search.config.SchemaInfo)", + "public com.yahoo.search.config.SchemaInfo getSchemaInfo()", "public com.yahoo.search.Query build()" ], "fields": [] @@ -2288,6 +2290,580 @@ ], "fields": [] }, + "com.yahoo.search.config.ClusterConfig$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigInstance$Builder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.search.config.ClusterConfig)", + "public com.yahoo.search.config.ClusterConfig$Builder clusterId(int)", + "public com.yahoo.search.config.ClusterConfig$Builder cacheSize(int)", + "public com.yahoo.search.config.ClusterConfig$Builder cacheTimeout(double)", + "public com.yahoo.search.config.ClusterConfig$Builder failoverToRemote(boolean)", + "public com.yahoo.search.config.ClusterConfig$Builder clusterName(java.lang.String)", + "public com.yahoo.search.config.ClusterConfig$Builder maxQueryTimeout(double)", + "public com.yahoo.search.config.ClusterConfig$Builder maxQueryCacheTimeout(double)", + "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", + "public final java.lang.String getDefMd5()", + "public final java.lang.String getDefName()", + "public final java.lang.String getDefNamespace()", + "public final boolean getApplyOnRestart()", + "public final void setApplyOnRestart(boolean)", + "public com.yahoo.search.config.ClusterConfig build()" + ], + "fields": [] + }, + "com.yahoo.search.config.ClusterConfig$Producer": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigInstance$Producer" + ], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract void getConfig(com.yahoo.search.config.ClusterConfig$Builder)" + ], + "fields": [] + }, + "com.yahoo.search.config.ClusterConfig": { + "superClass": "com.yahoo.config.ConfigInstance", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public static java.lang.String getDefMd5()", + "public static java.lang.String getDefName()", + "public static java.lang.String getDefNamespace()", + "public static java.lang.String getDefVersion()", + "public void <init>(com.yahoo.search.config.ClusterConfig$Builder)", + "public int clusterId()", + "public int cacheSize()", + "public double cacheTimeout()", + "public boolean failoverToRemote()", + "public java.lang.String clusterName()", + "public double maxQueryTimeout()", + "public double maxQueryCacheTimeout()" + ], + "fields": [ + "public static final java.lang.String CONFIG_DEF_MD5", + "public static final java.lang.String CONFIG_DEF_NAME", + "public static final java.lang.String CONFIG_DEF_NAMESPACE", + "public static final java.lang.String CONFIG_DEF_VERSION", + "public static final java.lang.String[] CONFIG_DEF_SCHEMA" + ] + }, + "com.yahoo.search.config.IndexInfoConfig$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigInstance$Builder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.search.config.IndexInfoConfig)", + "public com.yahoo.search.config.IndexInfoConfig$Builder indexinfo(com.yahoo.search.config.IndexInfoConfig$Indexinfo$Builder)", + "public com.yahoo.search.config.IndexInfoConfig$Builder indexinfo(java.util.function.Consumer)", + "public com.yahoo.search.config.IndexInfoConfig$Builder indexinfo(java.util.List)", + "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", + "public final java.lang.String getDefMd5()", + "public final java.lang.String getDefName()", + "public final java.lang.String getDefNamespace()", + "public final boolean getApplyOnRestart()", + "public final void setApplyOnRestart(boolean)", + "public com.yahoo.search.config.IndexInfoConfig build()" + ], + "fields": [ + "public java.util.List indexinfo" + ] + }, + "com.yahoo.search.config.IndexInfoConfig$Indexinfo$Alias$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigBuilder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.search.config.IndexInfoConfig$Indexinfo$Alias)", + "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Alias$Builder alias(java.lang.String)", + "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Alias$Builder indexname(java.lang.String)", + "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Alias build()" + ], + "fields": [] + }, + "com.yahoo.search.config.IndexInfoConfig$Indexinfo$Alias": { + "superClass": "com.yahoo.config.InnerNode", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(com.yahoo.search.config.IndexInfoConfig$Indexinfo$Alias$Builder)", + "public java.lang.String alias()", + "public java.lang.String indexname()" + ], + "fields": [] + }, + "com.yahoo.search.config.IndexInfoConfig$Indexinfo$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigBuilder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.search.config.IndexInfoConfig$Indexinfo)", + "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Builder name(java.lang.String)", + "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Builder command(com.yahoo.search.config.IndexInfoConfig$Indexinfo$Command$Builder)", + "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Builder command(java.util.function.Consumer)", + "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Builder command(java.util.List)", + "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Builder alias(com.yahoo.search.config.IndexInfoConfig$Indexinfo$Alias$Builder)", + "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Builder alias(java.util.function.Consumer)", + "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Builder alias(java.util.List)", + "public com.yahoo.search.config.IndexInfoConfig$Indexinfo build()" + ], + "fields": [ + "public java.util.List command", + "public java.util.List alias" + ] + }, + "com.yahoo.search.config.IndexInfoConfig$Indexinfo$Command$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigBuilder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.search.config.IndexInfoConfig$Indexinfo$Command)", + "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Command$Builder indexname(java.lang.String)", + "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Command$Builder command(java.lang.String)", + "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Command build()" + ], + "fields": [] + }, + "com.yahoo.search.config.IndexInfoConfig$Indexinfo$Command": { + "superClass": "com.yahoo.config.InnerNode", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(com.yahoo.search.config.IndexInfoConfig$Indexinfo$Command$Builder)", + "public java.lang.String indexname()", + "public java.lang.String command()" + ], + "fields": [] + }, + "com.yahoo.search.config.IndexInfoConfig$Indexinfo": { + "superClass": "com.yahoo.config.InnerNode", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(com.yahoo.search.config.IndexInfoConfig$Indexinfo$Builder)", + "public java.lang.String name()", + "public java.util.List command()", + "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Command command(int)", + "public java.util.List alias()", + "public com.yahoo.search.config.IndexInfoConfig$Indexinfo$Alias alias(int)" + ], + "fields": [] + }, + "com.yahoo.search.config.IndexInfoConfig$Producer": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigInstance$Producer" + ], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract void getConfig(com.yahoo.search.config.IndexInfoConfig$Builder)" + ], + "fields": [] + }, + "com.yahoo.search.config.IndexInfoConfig": { + "superClass": "com.yahoo.config.ConfigInstance", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public static java.lang.String getDefMd5()", + "public static java.lang.String getDefName()", + "public static java.lang.String getDefNamespace()", + "public static java.lang.String getDefVersion()", + "public void <init>(com.yahoo.search.config.IndexInfoConfig$Builder)", + "public java.util.List indexinfo()", + "public com.yahoo.search.config.IndexInfoConfig$Indexinfo indexinfo(int)" + ], + "fields": [ + "public static final java.lang.String CONFIG_DEF_MD5", + "public static final java.lang.String CONFIG_DEF_NAME", + "public static final java.lang.String CONFIG_DEF_NAMESPACE", + "public static final java.lang.String CONFIG_DEF_VERSION", + "public static final java.lang.String[] CONFIG_DEF_SCHEMA" + ] + }, + "com.yahoo.search.config.QrStartConfig$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigInstance$Builder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.search.config.QrStartConfig)", + "public com.yahoo.search.config.QrStartConfig$Builder jvm(com.yahoo.search.config.QrStartConfig$Jvm$Builder)", + "public com.yahoo.search.config.QrStartConfig$Builder jvm(java.util.function.Consumer)", + "public com.yahoo.search.config.QrStartConfig$Builder qrs(com.yahoo.search.config.QrStartConfig$Qrs$Builder)", + "public com.yahoo.search.config.QrStartConfig$Builder qrs(java.util.function.Consumer)", + "public com.yahoo.search.config.QrStartConfig$Builder ulimitv(java.lang.String)", + "public com.yahoo.search.config.QrStartConfig$Builder jdisc(com.yahoo.search.config.QrStartConfig$Jdisc$Builder)", + "public com.yahoo.search.config.QrStartConfig$Builder jdisc(java.util.function.Consumer)", + "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", + "public final java.lang.String getDefMd5()", + "public final java.lang.String getDefName()", + "public final java.lang.String getDefNamespace()", + "public final boolean getApplyOnRestart()", + "public final void setApplyOnRestart(boolean)", + "public com.yahoo.search.config.QrStartConfig build()" + ], + "fields": [ + "public com.yahoo.search.config.QrStartConfig$Jvm$Builder jvm", + "public com.yahoo.search.config.QrStartConfig$Qrs$Builder qrs", + "public com.yahoo.search.config.QrStartConfig$Jdisc$Builder jdisc" + ] + }, + "com.yahoo.search.config.QrStartConfig$Jdisc$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigBuilder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.search.config.QrStartConfig$Jdisc)", + "public com.yahoo.search.config.QrStartConfig$Jdisc$Builder classpath_extra(java.lang.String)", + "public com.yahoo.search.config.QrStartConfig$Jdisc$Builder export_packages(java.lang.String)", + "public com.yahoo.search.config.QrStartConfig$Jdisc build()" + ], + "fields": [] + }, + "com.yahoo.search.config.QrStartConfig$Jdisc": { + "superClass": "com.yahoo.config.InnerNode", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(com.yahoo.search.config.QrStartConfig$Jdisc$Builder)", + "public java.lang.String classpath_extra()", + "public java.lang.String export_packages()" + ], + "fields": [] + }, + "com.yahoo.search.config.QrStartConfig$Jvm$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigBuilder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.search.config.QrStartConfig$Jvm)", + "public com.yahoo.search.config.QrStartConfig$Jvm$Builder server(boolean)", + "public com.yahoo.search.config.QrStartConfig$Jvm$Builder verbosegc(boolean)", + "public com.yahoo.search.config.QrStartConfig$Jvm$Builder gcopts(java.lang.String)", + "public com.yahoo.search.config.QrStartConfig$Jvm$Builder heapsize(int)", + "public com.yahoo.search.config.QrStartConfig$Jvm$Builder minHeapsize(int)", + "public com.yahoo.search.config.QrStartConfig$Jvm$Builder stacksize(int)", + "public com.yahoo.search.config.QrStartConfig$Jvm$Builder compressedClassSpaceSize(int)", + "public com.yahoo.search.config.QrStartConfig$Jvm$Builder baseMaxDirectMemorySize(int)", + "public com.yahoo.search.config.QrStartConfig$Jvm$Builder directMemorySizeCache(int)", + "public com.yahoo.search.config.QrStartConfig$Jvm$Builder heapSizeAsPercentageOfPhysicalMemory(int)", + "public com.yahoo.search.config.QrStartConfig$Jvm$Builder availableProcessors(int)", + "public com.yahoo.search.config.QrStartConfig$Jvm build()" + ], + "fields": [] + }, + "com.yahoo.search.config.QrStartConfig$Jvm": { + "superClass": "com.yahoo.config.InnerNode", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(com.yahoo.search.config.QrStartConfig$Jvm$Builder)", + "public boolean server()", + "public boolean verbosegc()", + "public java.lang.String gcopts()", + "public int heapsize()", + "public int minHeapsize()", + "public int stacksize()", + "public int compressedClassSpaceSize()", + "public int baseMaxDirectMemorySize()", + "public int directMemorySizeCache()", + "public int heapSizeAsPercentageOfPhysicalMemory()", + "public int availableProcessors()" + ], + "fields": [] + }, + "com.yahoo.search.config.QrStartConfig$Producer": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigInstance$Producer" + ], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract void getConfig(com.yahoo.search.config.QrStartConfig$Builder)" + ], + "fields": [] + }, + "com.yahoo.search.config.QrStartConfig$Qrs$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigBuilder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.search.config.QrStartConfig$Qrs)", + "public com.yahoo.search.config.QrStartConfig$Qrs$Builder env(java.lang.String)", + "public com.yahoo.search.config.QrStartConfig$Qrs build()" + ], + "fields": [] + }, + "com.yahoo.search.config.QrStartConfig$Qrs": { + "superClass": "com.yahoo.config.InnerNode", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(com.yahoo.search.config.QrStartConfig$Qrs$Builder)", + "public java.lang.String env()" + ], + "fields": [] + }, + "com.yahoo.search.config.QrStartConfig": { + "superClass": "com.yahoo.config.ConfigInstance", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public static java.lang.String getDefMd5()", + "public static java.lang.String getDefName()", + "public static java.lang.String getDefNamespace()", + "public static java.lang.String getDefVersion()", + "public void <init>(com.yahoo.search.config.QrStartConfig$Builder)", + "public com.yahoo.search.config.QrStartConfig$Jvm jvm()", + "public com.yahoo.search.config.QrStartConfig$Qrs qrs()", + "public java.lang.String ulimitv()", + "public com.yahoo.search.config.QrStartConfig$Jdisc jdisc()" + ], + "fields": [ + "public static final java.lang.String CONFIG_DEF_MD5", + "public static final java.lang.String CONFIG_DEF_NAME", + "public static final java.lang.String CONFIG_DEF_NAMESPACE", + "public static final java.lang.String CONFIG_DEF_VERSION", + "public static final java.lang.String[] CONFIG_DEF_SCHEMA" + ] + }, + "com.yahoo.search.config.RankProfile$Builder": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String)", + "public com.yahoo.search.config.RankProfile$Builder setHasSummaryFeatures(boolean)", + "public com.yahoo.search.config.RankProfile$Builder setHasRankFeatures(boolean)", + "public com.yahoo.search.config.RankProfile$Builder addInput(java.lang.String, com.yahoo.tensor.TensorType)", + "public com.yahoo.search.config.RankProfile build()" + ], + "fields": [] + }, + "com.yahoo.search.config.RankProfile": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public java.lang.String name()", + "public boolean hasSummaryFeatures()", + "public boolean hasRankFeatures()", + "public java.util.Map inputs()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields": [] + }, + "com.yahoo.search.config.RateLimitingConfig$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigInstance$Builder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.search.config.RateLimitingConfig)", + "public com.yahoo.search.config.RateLimitingConfig$Builder capacityIncrement(double)", + "public com.yahoo.search.config.RateLimitingConfig$Builder maxAvailableCapacity(double)", + "public com.yahoo.search.config.RateLimitingConfig$Builder recheckForCapacityProbability(double)", + "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", + "public final java.lang.String getDefMd5()", + "public final java.lang.String getDefName()", + "public final java.lang.String getDefNamespace()", + "public final boolean getApplyOnRestart()", + "public final void setApplyOnRestart(boolean)", + "public com.yahoo.search.config.RateLimitingConfig build()" + ], + "fields": [] + }, + "com.yahoo.search.config.RateLimitingConfig$Producer": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigInstance$Producer" + ], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract void getConfig(com.yahoo.search.config.RateLimitingConfig$Builder)" + ], + "fields": [] + }, + "com.yahoo.search.config.RateLimitingConfig": { + "superClass": "com.yahoo.config.ConfigInstance", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public static java.lang.String getDefMd5()", + "public static java.lang.String getDefName()", + "public static java.lang.String getDefNamespace()", + "public static java.lang.String getDefVersion()", + "public void <init>(com.yahoo.search.config.RateLimitingConfig$Builder)", + "public double capacityIncrement()", + "public double maxAvailableCapacity()", + "public double recheckForCapacityProbability()" + ], + "fields": [ + "public static final java.lang.String CONFIG_DEF_MD5", + "public static final java.lang.String CONFIG_DEF_NAME", + "public static final java.lang.String CONFIG_DEF_NAMESPACE", + "public static final java.lang.String CONFIG_DEF_VERSION", + "public static final java.lang.String[] CONFIG_DEF_SCHEMA" + ] + }, + "com.yahoo.search.config.Schema$Builder": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(java.lang.String)", + "public com.yahoo.search.config.Schema$Builder add(com.yahoo.search.config.RankProfile)", + "public com.yahoo.search.config.Schema build()" + ], + "fields": [] + }, + "com.yahoo.search.config.Schema": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public java.lang.String name()", + "public java.util.Map rankProfiles()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", + "public java.lang.String toString()" + ], + "fields": [] + }, + "com.yahoo.search.config.SchemaInfo$Session": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public com.yahoo.tensor.TensorType rankProfileInput(java.lang.String, java.lang.String)" + ], + "fields": [] + }, + "com.yahoo.search.config.SchemaInfo": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.search.config.IndexInfoConfig, com.yahoo.prelude.fastsearch.DocumentdbInfoConfig, com.yahoo.container.QrSearchersConfig)", + "public void <init>(java.util.List, java.util.Map)", + "public com.yahoo.search.config.SchemaInfo$Session newSession(com.yahoo.search.Query)", + "public static com.yahoo.search.config.SchemaInfo empty()", + "public boolean equals(java.lang.Object)", + "public int hashCode()" + ], + "fields": [] + }, "com.yahoo.search.federation.selection.FederationTarget": { "superClass": "java.lang.Object", "interfaces": [], @@ -5379,7 +5955,6 @@ "public" ], "methods": [ - "public static java.lang.String lookupRankProfileIn(java.util.Map)", "public static com.yahoo.search.query.profile.types.QueryProfileType getArgumentType()", "public void <init>(com.yahoo.search.Query)", "public boolean hasRankProfile()", @@ -6566,6 +7141,7 @@ "public java.lang.Class getValueClass(java.lang.String)", "public com.yahoo.search.query.profile.types.QueryProfileType getType(java.lang.String)", "public com.yahoo.search.query.profile.types.FieldType getFieldType(com.yahoo.processing.request.CompoundName)", + "public com.yahoo.search.query.profile.types.FieldDescription getField(com.yahoo.processing.request.CompoundName)", "public com.yahoo.search.query.profile.types.FieldDescription getField(java.lang.String)", "public com.yahoo.search.query.profile.types.FieldDescription removeField(java.lang.String)", "public void addField(com.yahoo.search.query.profile.types.FieldDescription)", @@ -6717,7 +7293,8 @@ "public" ], "methods": [ - "public void <init>(com.yahoo.search.Query)", + "public void <init>(com.yahoo.search.config.SchemaInfo, com.yahoo.search.Query, java.util.Map)", + "public void set(com.yahoo.processing.request.CompoundName, java.lang.Object, java.util.Map)", "public void requireSettable(com.yahoo.processing.request.CompoundName, java.lang.Object, java.util.Map)" ], "fields": [] @@ -6898,6 +7475,7 @@ "public com.yahoo.search.query.ranking.RankFeatures clone()", "public com.yahoo.search.query.ranking.RankFeatures cloneFor(com.yahoo.search.query.Ranking)", "public java.lang.String toString()", + "public static boolean isFeatureName(java.lang.String)", "public bridge synthetic java.lang.Object clone()" ], "fields": [] @@ -7995,6 +8573,7 @@ "final" ], "methods": [ + "public void <init>(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts, com.yahoo.search.config.SchemaInfo, com.yahoo.language.process.SpecialTokenRegistry, com.yahoo.search.rendering.RendererRegistry, com.yahoo.language.Linguistics, java.util.concurrent.Executor)", "public void <init>(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts, com.yahoo.language.process.SpecialTokenRegistry, com.yahoo.search.rendering.RendererRegistry, com.yahoo.language.Linguistics, java.util.concurrent.Executor)", "public void <init>(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts, com.yahoo.language.process.SpecialTokenRegistry, com.yahoo.search.rendering.RendererRegistry, com.yahoo.language.Linguistics)", "public static com.yahoo.search.searchchain.Execution$Context createContextStub()", @@ -8003,6 +8582,7 @@ "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts)", "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.prelude.IndexFacts, com.yahoo.language.Linguistics)", "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts, com.yahoo.language.Linguistics)", + "public static com.yahoo.search.searchchain.Execution$Context createContextStub(com.yahoo.search.searchchain.SearchChainRegistry, com.yahoo.prelude.IndexFacts, com.yahoo.search.config.SchemaInfo, com.yahoo.language.Linguistics)", "public void populateFrom(com.yahoo.search.searchchain.Execution$Context)", "public boolean equals(com.yahoo.search.searchchain.Execution$Context)", "public int hashCode()", @@ -8010,6 +8590,7 @@ "public com.yahoo.search.searchchain.Execution$Context shallowCopy()", "public com.yahoo.prelude.IndexFacts getIndexFacts()", "public void setIndexFacts(com.yahoo.prelude.IndexFacts)", + "public com.yahoo.search.config.SchemaInfo schemaInfo()", "public com.yahoo.search.searchchain.SearchChainRegistry searchChainRegistry()", "public com.yahoo.search.rendering.RendererRegistry rendererRegistry()", "public com.yahoo.language.process.SpecialTokenRegistry getTokenRegistry()", @@ -8061,12 +8642,14 @@ "public" ], "methods": [ + "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.prelude.fastsearch.DocumentdbInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.language.Linguistics, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor)", "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.language.Linguistics, com.yahoo.component.provider.ComponentRegistry, java.util.concurrent.Executor)", "public void <init>(com.yahoo.container.core.ChainsConfig, com.yahoo.search.config.IndexInfoConfig, com.yahoo.container.QrSearchersConfig, com.yahoo.component.provider.ComponentRegistry, com.yahoo.vespa.configdefinition.SpecialtokensConfig, com.yahoo.language.Linguistics, com.yahoo.component.provider.ComponentRegistry)", "public com.yahoo.search.searchchain.Execution newExecution(com.yahoo.component.chain.Chain)", "public com.yahoo.search.searchchain.Execution newExecution(java.lang.String)", "public com.yahoo.search.searchchain.SearchChainRegistry searchChainRegistry()", "public com.yahoo.search.rendering.RendererRegistry rendererRegistry()", + "public com.yahoo.search.config.SchemaInfo schemaInfo()", "public void deconstruct()", "public static com.yahoo.search.searchchain.ExecutionFactory empty()" ], diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java index a1613dccdd5..c9a855c2f34 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java @@ -323,11 +323,11 @@ public class IndexFacts { private Session(Collection<String> sources, Collection<String> restrict) { // Assumption: Search definition name equals document name. - documentTypes = ImmutableList.copyOf(resolveDocumentTypes(sources, restrict, searchDefinitions.keySet())); + documentTypes = List.copyOf(resolveDocumentTypes(sources, restrict, searchDefinitions.keySet())); } private Session(Collection<String> sources, Collection<String> restrict, Set<String> candidateDocumentTypes) { - documentTypes = ImmutableList.copyOf(resolveDocumentTypes(sources, restrict, candidateDocumentTypes)); + documentTypes = List.copyOf(resolveDocumentTypes(sources, restrict, candidateDocumentTypes)); } /** diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java index e7018f81de1..57a8d518ed2 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java @@ -24,9 +24,9 @@ public final class IndexModel { private static final Logger log = Logger.getLogger(IndexModel.class.getName()); - private Map<String, List<String>> masterClusters; - private Map<String, SearchDefinition> searchDefinitions; - private SearchDefinition unionSearchDefinition; + private final Map<String, List<String>> masterClusters; + private final Map<String, SearchDefinition> searchDefinitions; + private final SearchDefinition unionSearchDefinition; /** Create an index model for a single search definition */ public IndexModel(SearchDefinition searchDefinition) { @@ -83,7 +83,6 @@ public final class IndexModel { return clusters; } - @SuppressWarnings("deprecation") private static Map<String, SearchDefinition> toSearchDefinitions(IndexInfoConfig c) { Map<String, SearchDefinition> searchDefinitions = new HashMap<>(); diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java index d5a3e8c2786..f35559ad2f4 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocumentDatabase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.fastsearch; +import com.yahoo.search.config.RankProfile; import com.yahoo.tensor.TensorType; import java.util.ArrayList; @@ -10,7 +11,7 @@ import java.util.Map; import java.util.stream.Collectors; /** - * Representation of a back-end document database. + * Representation of a document database realizing a schema in a content cluster. * * @author geirst */ @@ -33,7 +34,7 @@ public class DocumentDatabase { public DocumentDatabase(String name, DocsumDefinitionSet docsumDefinitionSet, Collection<RankProfile> rankProfiles) { this.name = name; this.docsumDefSet = docsumDefinitionSet; - this.rankProfiles = Map.copyOf(rankProfiles.stream().collect(Collectors.toMap(RankProfile::getName, p -> p))); + this.rankProfiles = Map.copyOf(rankProfiles.stream().collect(Collectors.toMap(RankProfile::name, p -> p))); } public String getName() { @@ -49,13 +50,15 @@ public class DocumentDatabase { private static Collection<RankProfile> toRankProfiles(Collection<DocumentdbInfoConfig.Documentdb.Rankprofile> rankProfileConfigList) { List<RankProfile> rankProfiles = new ArrayList<>(); - for (DocumentdbInfoConfig.Documentdb.Rankprofile c : rankProfileConfigList) - rankProfiles.add(new RankProfile(c.name(), c.hasSummaryFeatures(), c.hasRankFeatures(), inputs(c))); + for (var profileConfig : rankProfileConfigList) { + var builder = new RankProfile.Builder(profileConfig.name()); + builder.setHasSummaryFeatures(profileConfig.hasSummaryFeatures()); + builder.setHasRankFeatures(profileConfig.hasRankFeatures()); + for (var inputConfig : profileConfig.input()) + builder.addInput(inputConfig.name(), TensorType.fromSpec(inputConfig.type())); + rankProfiles.add(builder.build()); + } return rankProfiles; } - private static Map<String, TensorType> inputs(DocumentdbInfoConfig.Documentdb.Rankprofile c) { - return c.input().stream().collect(Collectors.toMap(i -> i.name(), i -> TensorType.fromSpec(i.type()))); - } - } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/RankProfile.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/RankProfile.java deleted file mode 100644 index a4248245f2a..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/RankProfile.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.fastsearch; - -import com.yahoo.tensor.TensorType; - -import java.util.Map; - -/** - * Information about a rank profile - * - * @author bratseth - */ -class RankProfile { - - private final String name; - private final boolean hasSummaryFeatures; - private final boolean hasRankFeatures; - private final Map<String, TensorType> inputs; - - public RankProfile(String name, - boolean hasSummaryFeatures, - boolean hasRankFeatures, - Map<String, TensorType> inputs) { - this.name = name; - this.hasSummaryFeatures = hasSummaryFeatures; - this.hasRankFeatures = hasRankFeatures; - this.inputs = Map.copyOf(inputs); - } - - public String getName() { return name; } - - /** Returns true if this rank profile has summary features. */ - public boolean hasSummaryFeatures() { return hasSummaryFeatures; } - - /** Returns true if this rank profile has rank features. */ - public boolean hasRankFeatures() { return hasRankFeatures; } - - /** Returns the inputs explicitly declared in this rank profile. */ - public Map<String, TensorType> inputs() { return inputs; } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index d26791411c5..a6da823d990 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -14,8 +14,8 @@ import com.yahoo.protect.Validator; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.cluster.PingableSearcher; +import com.yahoo.search.config.RankProfile; import com.yahoo.search.grouping.vespa.GroupingExecutor; -import com.yahoo.search.result.ErrorHit; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; @@ -33,7 +33,7 @@ import java.util.logging.Logger; /** * Superclass for backend searchers. * - * @author baldersheim + * @author baldersheim */ public abstract class VespaBackEndSearcher extends PingableSearcher { diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 74091d9978c..354881c763b 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -13,6 +13,7 @@ import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.query.Highlight; import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation; import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.config.SchemaInfo; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.rpc.ProtobufSerialization; import com.yahoo.search.federation.FederationSearcher; @@ -45,6 +46,7 @@ import com.yahoo.search.query.properties.QueryProperties; import com.yahoo.search.query.properties.QueryPropertyAliases; import com.yahoo.search.query.properties.RankProfileInputProperties; import com.yahoo.search.query.properties.RequestContextProperties; +import com.yahoo.search.query.ranking.RankFeatures; import com.yahoo.search.yql.NullItemException; import com.yahoo.search.yql.VespaSerializer; import com.yahoo.search.yql.YqlParser; @@ -58,6 +60,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.logging.Level; @@ -154,11 +157,10 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** The timeout of the query, in milliseconds */ private long timeout = defaultTimeout; - /** Whether this query is forbidden to access cached information */ private boolean noCache = false; - /** Whether or not grouping should use a session cache */ + /** Whether grouping should use a session cache */ private boolean groupingSessionCache = true; //-------------- Generic property containers -------------------------------- @@ -177,7 +179,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** The ranking requested in this query */ private Ranking ranking = new Ranking(this); - /** The query query and/or query program declaration */ + /** The query and/or query program declaration */ private Model model = new Model(this); /** How results of this query should be presented */ @@ -212,6 +214,8 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { argumentType = new QueryProfileType("native"); argumentType.setBuiltin(true); + // Note: Order here matters as fields are set in this order, and rank feature conversion depends + // on other fields already being set (see RankProfileInputProperties) argumentType.addField(new FieldDescription(OFFSET.toString(), "integer", "offset start")); argumentType.addField(new FieldDescription(HITS.toString(), "integer", "hits count")); argumentType.addField(new FieldDescription(QUERY_PROFILE.toString(), "string")); @@ -223,11 +227,11 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { argumentType.addField(new FieldDescription(TIMEOUT.toString(), "string", "timeout")); argumentType.addField(new FieldDescription(FederationSearcher.SOURCENAME.toString(),"string")); argumentType.addField(new FieldDescription(FederationSearcher.PROVIDERNAME.toString(),"string")); - argumentType.addField(new FieldDescription(Presentation.PRESENTATION, new QueryProfileFieldType(Presentation.getArgumentType()))); - argumentType.addField(new FieldDescription(Ranking.RANKING, new QueryProfileFieldType(Ranking.getArgumentType()))); argumentType.addField(new FieldDescription(Model.MODEL, new QueryProfileFieldType(Model.getArgumentType()))); argumentType.addField(new FieldDescription(Select.SELECT, new QueryProfileFieldType(Select.getArgumentType()))); argumentType.addField(new FieldDescription(Dispatcher.DISPATCH, new QueryProfileFieldType(Dispatcher.getArgumentType()))); + argumentType.addField(new FieldDescription(Ranking.RANKING, new QueryProfileFieldType(Ranking.getArgumentType()))); + argumentType.addField(new FieldDescription(Presentation.PRESENTATION, new QueryProfileFieldType(Presentation.getArgumentType()))); argumentType.freeze(); } public static QueryProfileType getArgumentType() { return argumentType; } @@ -259,9 +263,9 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { public static void addNativeQueryProfileTypesTo(QueryProfileTypeRegistry registry) { // Add modifiable copies to allow query profile types in this to add to these registry.register(Query.getArgumentType().unfrozen()); - registry.register(Ranking.getArgumentType().unfrozen()); registry.register(Model.getArgumentType().unfrozen()); registry.register(Select.getArgumentType().unfrozen()); + registry.register(Ranking.getArgumentType().unfrozen()); registry.register(Presentation.getArgumentType().unfrozen()); registry.register(DefaultProperties.argumentType.unfrozen()); } @@ -271,7 +275,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { ImmutableList.copyOf(namesUnder(CompoundName.empty, Query.getArgumentType())); private static List<CompoundName> namesUnder(CompoundName prefix, QueryProfileType type) { - if ( type == null) return Collections.emptyList(); // Names not known statically + if (type == null) return Collections.emptyList(); // Names not known statically List<CompoundName> names = new ArrayList<>(); for (Map.Entry<String, FieldDescription> field : type.fields().entrySet()) { if (field.getValue().getType() instanceof QueryProfileFieldType) { @@ -339,7 +343,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { public Query(HttpRequest request, Map<String, String> requestMap, CompiledQueryProfile queryProfile) { super(new QueryPropertyAliases(propertyAliases)); this.httpRequest = request; - init(requestMap, queryProfile, Embedder.throwsOnUse.asMap(), ZoneInfo.defaultInfo()); + init(requestMap, queryProfile, Embedder.throwsOnUse.asMap(), ZoneInfo.defaultInfo(), SchemaInfo.empty()); } // TODO: Deprecate most constructors above here @@ -349,23 +353,26 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { builder.getRequestMap(), builder.getQueryProfile(), builder.getEmbedders(), - builder.getZoneInfo()); + builder.getZoneInfo(), + builder.getSchemaInfo()); } private Query(HttpRequest request, Map<String, String> requestMap, CompiledQueryProfile queryProfile, Map<String, Embedder> embedders, - ZoneInfo zoneInfo) { + ZoneInfo zoneInfo, + SchemaInfo schemaInfo) { super(new QueryPropertyAliases(propertyAliases)); this.httpRequest = request; - init(requestMap, queryProfile, embedders, zoneInfo); + init(requestMap, queryProfile, embedders, zoneInfo, schemaInfo); } private void init(Map<String, String> requestMap, CompiledQueryProfile queryProfile, Map<String, Embedder> embedders, - ZoneInfo zoneInfo) { + ZoneInfo zoneInfo, + SchemaInfo schemaInfo) { startTime = httpRequest.getJDiscRequest().creationTime(TimeUnit.MILLISECONDS); if (queryProfile != null) { // Move all request parameters to the query profile @@ -374,7 +381,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { setPropertiesFromRequestMap(requestMap, properties(), true); // Create the full chain - properties().chain(new RankProfileInputProperties(this)) + properties().chain(new RankProfileInputProperties(schemaInfo, this, embedders)) .chain(new QueryProperties(this, queryProfile.getRegistry(), embedders)) .chain(new ModelObjectMap()) .chain(new RequestContextProperties(requestMap, zoneInfo)) @@ -395,7 +402,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { } else { // bypass these complications if there is no query profile to get values from and validate against properties(). - chain(new RankProfileInputProperties(this)). + chain(new RankProfileInputProperties(schemaInfo, this, embedders)). chain(new QueryProperties(this, CompiledQueryProfileRegistry.empty, embedders)). chain(new PropertyMap()). chain(new DefaultProperties()); @@ -462,13 +469,13 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** Calls properties.set on all entries in requestMap */ private void setPropertiesFromRequestMap(Map<String, String> requestMap, Properties properties, boolean ignoreSelect) { - // Set rank profile first because it contains type information in inputs which impacts other values set - String rankProfile = Ranking.lookupRankProfileIn(requestMap); - if (rankProfile != null) - properties.set(Ranking.RANKING + "." + Ranking.PROFILE, rankProfile, requestMap); - for (var entry : requestMap.entrySet()) { if (ignoreSelect && entry.getKey().equals(Select.SELECT)) continue; + if (RankFeatures.isFeatureName(entry.getKey())) continue; // Set these last + properties.set(entry.getKey(), entry.getValue(), requestMap); + } + for (var entry : requestMap.entrySet()) { + if ( ! RankFeatures.isFeatureName(entry.getKey())) continue; properties.set(entry.getKey(), entry.getValue(), requestMap); } } @@ -1142,6 +1149,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { private CompiledQueryProfile queryProfile = null; private Map<String, Embedder> embedders = Embedder.throwsOnUse.asMap(); private ZoneInfo zoneInfo = ZoneInfo.defaultInfo(); + private SchemaInfo schemaInfo = SchemaInfo.empty(); public Builder setRequest(String query) { request = HttpRequest.createTestRequest(query, com.yahoo.jdisc.http.HttpRequest.Method.GET); @@ -1204,6 +1212,13 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { public ZoneInfo getZoneInfo() { return zoneInfo; } + public Builder setSchemaInfo(SchemaInfo schemaInfo) { + this.schemaInfo = schemaInfo; + return this; + } + + public SchemaInfo getSchemaInfo() { return schemaInfo; } + /** Creates a new query from this builder. No properties are required to before calling this. */ public Query build() { return new Query(this); } diff --git a/container-search/src/main/java/com/yahoo/search/config/RankProfile.java b/container-search/src/main/java/com/yahoo/search/config/RankProfile.java new file mode 100644 index 00000000000..944a23f2964 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/config/RankProfile.java @@ -0,0 +1,94 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.config; + +import com.yahoo.tensor.TensorType; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Information about a rank profile + * + * @author bratseth + */ +public class RankProfile { + + private final String name; + private final boolean hasSummaryFeatures; + private final boolean hasRankFeatures; + private final Map<String, TensorType> inputs; + + private RankProfile(Builder builder) { + this.name = builder.name; + this.hasSummaryFeatures = builder.hasSummaryFeatures; + this.hasRankFeatures = builder.hasRankFeatures; + this.inputs = Map.copyOf(builder.inputs); + } + + public String name() { return name; } + + /** Returns true if this rank profile has summary features. */ + public boolean hasSummaryFeatures() { return hasSummaryFeatures; } + + /** Returns true if this rank profile has rank features. */ + public boolean hasRankFeatures() { return hasRankFeatures; } + + /** Returns the inputs explicitly declared in this rank profile. */ + public Map<String, TensorType> inputs() { return inputs; } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof RankProfile)) return false; + RankProfile other = (RankProfile)o; + if ( ! other.name.equals(this.name)) return false; + if ( other.hasSummaryFeatures != this.hasSummaryFeatures) return false; + if ( other.hasRankFeatures != this.hasRankFeatures) return false; + if ( ! other.inputs.equals(this.inputs)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(name, hasSummaryFeatures, hasRankFeatures, inputs); + } + + @Override + public String toString() { + return "rank profile '" + name + "'"; + } + + public static class Builder { + + private final String name; + private boolean hasSummaryFeatures = true; + private boolean hasRankFeatures = true; + private final Map<String, TensorType> inputs = new HashMap<>(); + + public Builder(String name) { + this.name = Objects.requireNonNull(name); + } + + public Builder setHasSummaryFeatures(boolean hasSummaryFeatures) { + this.hasSummaryFeatures = hasSummaryFeatures; + return this; + } + + public Builder setHasRankFeatures(boolean hasRankFeatures) { + this.hasRankFeatures = hasRankFeatures; + return this; + } + + public Builder addInput(String name, TensorType type) { + inputs.put(name, type); + return this; + } + + public RankProfile build() { + return new RankProfile(this); + } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/config/Schema.java b/container-search/src/main/java/com/yahoo/search/config/Schema.java new file mode 100644 index 00000000000..57712c731f4 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/config/Schema.java @@ -0,0 +1,71 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.config; + +import com.yahoo.api.annotations.Beta; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Information about a schema which is part of the application running this. + * + * This is immutable. + * + * @author bratseth + */ +@Beta +public class Schema { + + private final String name; + private final Map<String, RankProfile> rankProfiles; + + private Schema(Builder builder) { + this.name = builder.name; + this.rankProfiles = Map.copyOf(builder.rankProfiles); + } + + public String name() { return name; } + public Map<String, RankProfile> rankProfiles() { return rankProfiles; } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof Schema)) return false; + Schema other = (Schema)o; + if ( ! other.name.equals(this.name)) return false; + if ( ! other.rankProfiles.equals(this.rankProfiles)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(name, rankProfiles); + } + + @Override + public String toString() { + return "schema '" + name + "'"; + } + + public static class Builder { + + private final String name; + private final Map<String, RankProfile> rankProfiles = new HashMap<>(); + + public Builder(String name) { + this.name = Objects.requireNonNull(name); + } + + public Builder add(RankProfile profile) { + rankProfiles.put(profile.name(), Objects.requireNonNull(profile)); + return this; + } + + public Schema build() { + return new Schema(this); + } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/config/SchemaInfo.java b/container-search/src/main/java/com/yahoo/search/config/SchemaInfo.java new file mode 100644 index 00000000000..746f1c340f2 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/config/SchemaInfo.java @@ -0,0 +1,149 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.config; + +import com.yahoo.api.annotations.Beta; +import com.yahoo.component.annotation.Inject; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.search.Query; +import com.yahoo.tensor.TensorType; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Information about all the schemas configured in the application this container is a part of. + * + * Usage: + * <code> + * SchemaInfo.Session session = schemaInfo.newSession(query); // once when starting to process a query + * session.get(...) // access information about the schema(s) relevant to the query + * </code> + * + * This is immutable. + * + * @author bratseth + */ +// NOTES: +// This should replace IndexFacts, and probably DocumentDatabase. +// It replicates the schema resolution mechanism in IndexFacts, but does not yet contain any field information. +// To replace IndexFacts, this must accept IndexInfo and expose that information, as well as consolidation +// given a set of possible schemas: The session mechanism is present here to make that efficient when added +// (resolving schema subsets for every field lookup is too expensive). +@Beta +public class SchemaInfo { + + private static final SchemaInfo empty = new SchemaInfo(List.of(), Map.of()); + + private final List<Schema> schemas; + + /** The schemas contained in each content cluster indexed by cluster name */ + private final Map<String, List<String>> clusters; + + @Inject + public SchemaInfo(IndexInfoConfig indexInfo, // will be used in the future + DocumentdbInfoConfig documentdbInfoConfig, + QrSearchersConfig qrSearchersConfig) { + this(SchemaInfoConfigurer.toSchemas(documentdbInfoConfig), SchemaInfoConfigurer.toClusters(qrSearchersConfig)); + } + + public SchemaInfo(List<Schema> schemas, Map<String, List<String>> clusters) { + this.schemas = List.copyOf(schemas); + this.clusters = Map.copyOf(clusters); + } + + public Session newSession(Query query) { + return new Session(query.getModel().getSources(), query.getModel().getRestrict(), clusters, schemas); + } + + public static SchemaInfo empty() { return empty; } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof SchemaInfo)) return false; + SchemaInfo other = (SchemaInfo)o; + if ( ! other.schemas.equals(this.schemas)) return false; + if ( ! other.clusters.equals(this.clusters)) return false; + return true; + } + + @Override + public int hashCode() { return Objects.hash(schemas, clusters); } + + /** The schema information resolved to be relevant to this session. */ + public static class Session { + + private final List<Schema> schemas; + + private Session(Set<String> sources, + Set<String> restrict, + Map<String, List<String>> clusters, + List<Schema> candidates) { + this.schemas = resolveSchemas(sources, restrict, clusters, candidates); + } + + /** + * Given a search list which is a mixture of schemas and cluster + * names, and a restrict list which is a list of schemas, return a + * set of all valid schemas for this combination. + * + * @return the possibly empty list of schemas matching the arguments + */ + private static List<Schema> resolveSchemas(Set<String> sources, + Set<String> restrict, + Map<String, List<String>> clusters, + List<Schema> candidates) { + if (sources.isEmpty()) + return restrict.isEmpty() ? candidates : keep(restrict, candidates); + + Set<String> schemaNames = new HashSet<>(); + for (String source : sources) { + if (clusters.containsKey(source)) // source is a cluster + schemaNames.addAll(clusters.get(source)); + else // source is a schema + schemaNames.add(source); + } + candidates = keep(schemaNames, candidates); + return restrict.isEmpty() ? candidates : keep(restrict, candidates); + } + + private static List<Schema> keep(Set<String> names, List<Schema> schemas) { + return schemas.stream().filter(schema -> names.contains(schema.name())).collect(Collectors.toList()); + } + + /** + * Returns the type of the given rank feature name in the given profile, + * if it can be uniquely determined. + * + * @param rankFeature the rank feature name, a string on the form "query(name)" + * @param rankProfile the name of the rank profile in which to locate the input declaration + * @return the type of the declared input, or null if it is not declared or the rank profile is not found + * @throws IllegalArgumentException if the feature is declared in this rank profile in multiple schemas + * of this session with conflicting types + */ + public TensorType rankProfileInput(String rankFeature, String rankProfile) { + TensorType foundType = null; + Schema declaringSchema = null; + for (Schema schema : schemas) { + RankProfile profile = schema.rankProfiles().get(rankProfile); + if (profile == null) continue; + TensorType newlyFoundType = profile.inputs().get(rankFeature); + if (newlyFoundType == null) continue; + if (foundType != null && ! newlyFoundType.equals(foundType)) + throw new IllegalArgumentException("Conflicting input type declarations for '" + rankFeature + "': " + + "Declared as " + foundType + " in " + profile + " in " + declaringSchema + + ", and as " + newlyFoundType + " in " + profile + " in " + schema); + foundType = newlyFoundType; + declaringSchema = schema; + } + return foundType; + } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/config/SchemaInfoConfigurer.java b/container-search/src/main/java/com/yahoo/search/config/SchemaInfoConfigurer.java new file mode 100644 index 00000000000..ae06babda66 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/config/SchemaInfoConfigurer.java @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.config; + +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.tensor.TensorType; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Translation between schema info configuration and schema objects. + * + * @author bratseth + */ +class SchemaInfoConfigurer { + + static List<Schema> toSchemas(DocumentdbInfoConfig documentdbInfoConfig) { + return documentdbInfoConfig.documentdb().stream().map(config -> toSchema(config)).collect(Collectors.toList()); + } + + static Schema toSchema(DocumentdbInfoConfig.Documentdb documentDbConfig) { + Schema.Builder builder = new Schema.Builder(documentDbConfig.name()); + for (var profileConfig : documentDbConfig.rankprofile()) { + RankProfile.Builder profileBuilder = new RankProfile.Builder(profileConfig.name()); + profileBuilder.setHasSummaryFeatures(profileConfig.hasSummaryFeatures()); + profileBuilder.setHasRankFeatures(profileConfig.hasRankFeatures()); + for (var inputConfig : profileConfig.input()) + profileBuilder.addInput(inputConfig.name(), TensorType.fromSpec(inputConfig.type())); + builder.add(profileBuilder.build()); + } + return builder.build(); + } + + static Map<String, List<String>> toClusters(QrSearchersConfig config) { + Map<String, List<String>> clusters = new HashMap<>(); + for (int i = 0; i < config.searchcluster().size(); ++i) { + List<String> schemas = new ArrayList<>(); + String clusterName = config.searchcluster(i).name(); + for (int j = 0; j < config.searchcluster(i).searchdef().size(); ++j) + schemas.add(config.searchcluster(i).searchdef(j)); + clusters.put(clusterName, schemas); + } + return clusters; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/config/dispatchprototype/package-info.java b/container-search/src/main/java/com/yahoo/search/config/dispatchprototype/package-info.java deleted file mode 100644 index 217fbe80888..00000000000 --- a/container-search/src/main/java/com/yahoo/search/config/dispatchprototype/package-info.java +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Package for dispatchprototype config. - * @author Tony Vaagenes - */ -@ExportPackage -package com.yahoo.search.config.dispatchprototype; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/main/java/com/yahoo/search/config/internal/TensorConverter.java b/container-search/src/main/java/com/yahoo/search/config/internal/TensorConverter.java new file mode 100644 index 00000000000..fbe2ffb8984 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/config/internal/TensorConverter.java @@ -0,0 +1,95 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.config.internal; + +import com.yahoo.language.Language; +import com.yahoo.language.process.Embedder; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A class which knows how to convert an Object value to a tensor of a given type. + * + * @author bratseth + */ +public class TensorConverter { + + private static final Pattern embedderArgumentRegexp = Pattern.compile("^([A-Za-z0-9_\\-.]+),\\s*([\"'].*[\"'])"); + + private final Map<String, Embedder> embedders; + + public TensorConverter(Map<String, Embedder> embedders) { + this.embedders = embedders; + } + + public Tensor convertTo(TensorType type, String key, Object value, Language language) { + var context = new Embedder.Context(key).setLanguage(language); + Tensor tensor = toTensor(type, value, context); + if (tensor == null) return null; + if (! tensor.type().isAssignableTo(type)) + throw new IllegalArgumentException("Require a tensor of type " + type); + return tensor; + } + + private Tensor toTensor(TensorType type, Object value, Embedder.Context context) { + if (value instanceof Tensor) return (Tensor)value; + if (value instanceof String && isEmbed((String)value)) return embed((String)value, type, context); + if (value instanceof String) return Tensor.from(type, (String)value); + return null; + } + + static boolean isEmbed(String value) { + return value.startsWith("embed("); + } + + private Tensor embed(String s, TensorType type, Embedder.Context embedderContext) { + if ( ! s.endsWith(")")) + throw new IllegalArgumentException("Expected any string enclosed in embed(), but the argument does not end by ')'"); + String argument = s.substring("embed(".length(), s.length() - 1); + Embedder embedder; + + // Check if arguments specifies an embedder with the format embed(embedder, "text to encode") + Matcher matcher = embedderArgumentRegexp.matcher(argument); + if (matcher.matches()) { + String embedderId = matcher.group(1); + argument = matcher.group(2); + if ( ! embedders.containsKey(embedderId)) { + throw new IllegalArgumentException("Can't find embedder '" + embedderId + "'. " + + "Valid embedders are " + validEmbedders(embedders)); + } + embedder = embedders.get(embedderId); + } else if (embedders.size() == 0) { + throw new IllegalStateException("No embedders provided"); // should never happen + } else if (embedders.size() > 1) { + throw new IllegalArgumentException("Multiple embedders are provided but no embedder id is given. " + + "Valid embedders are " + validEmbedders(embedders)); + } else { + embedder = embedders.entrySet().stream().findFirst().get().getValue(); + } + + return embedder.embed(removeQuotes(argument), embedderContext, type); + } + + private static String removeQuotes(String s) { + if (s.startsWith("'") && s.endsWith("'")) { + return s.substring(1, s.length() - 1); + } + if (s.startsWith("\"") && s.endsWith("\"")) { + return s.substring(1, s.length() - 1); + } + return s; + } + + private static String validEmbedders(Map<String, Embedder> embedders) { + List<String> embedderIds = new ArrayList<>(); + embedders.forEach((key, value) -> embedderIds.add(key)); + embedderIds.sort(null); + return String.join(",", embedderIds); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/config/package-info.java b/container-search/src/main/java/com/yahoo/search/config/package-info.java index aec055a0972..dd9c7bfcf04 100644 --- a/container-search/src/main/java/com/yahoo/search/config/package-info.java +++ b/container-search/src/main/java/com/yahoo/search/config/package-info.java @@ -1,5 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage +@PublicApi package com.yahoo.search.config; +import com.yahoo.api.annotations.PublicApi; import com.yahoo.osgi.annotation.ExportPackage; + +/** + * Information about the current configuration this is running as a part of. + */
\ No newline at end of file diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 54d8ac40556..025b567ce80 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -360,6 +360,7 @@ public class SearchHandler extends LoggingRequestHandler { .setQueryProfile(queryProfile) .setEmbedders(embedders) .setZoneInfo(zoneInfo) + .setSchemaInfo(executionFactory.schemaInfo()) .build(); boolean benchmarking = VespaHeaders.benchmarkOutput(request); diff --git a/container-search/src/main/java/com/yahoo/search/query/Model.java b/container-search/src/main/java/com/yahoo/search/query/Model.java index 0cc7a50141f..82cda9e8a1b 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Model.java +++ b/container-search/src/main/java/com/yahoo/search/query/Model.java @@ -10,6 +10,7 @@ import com.yahoo.prelude.query.TaggableItem; import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; +import com.yahoo.search.config.SchemaInfo; import com.yahoo.search.query.parser.Parsable; import com.yahoo.search.query.parser.Parser; import com.yahoo.search.query.parser.ParserEnvironment; @@ -91,8 +92,13 @@ public class Model implements Cloneable { private Set<String> restrict = new LinkedHashSet<>(); private String searchPath; private String documentDbName = null; - private Execution execution = new Execution(new Execution.Context(null, null, - null, null, null, Runnable::run)); + private Execution execution = new Execution(new Execution.Context(null, + null, + SchemaInfo.empty(), + null, + null, + null, + Runnable::run)); public Model(Query query) { setParent(query); diff --git a/container-search/src/main/java/com/yahoo/search/query/Ranking.java b/container-search/src/main/java/com/yahoo/search/query/Ranking.java index 9f5b9a77547..fd0cd5a85b7 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Ranking.java +++ b/container-search/src/main/java/com/yahoo/search/query/Ranking.java @@ -52,17 +52,14 @@ public class Ranking implements Cloneable { public static final String FEATURES = "features"; public static final String PROPERTIES = "properties"; - /** For internal use only. */ - public static String lookupRankProfileIn(Map<String, String> properties) { - return Optional.ofNullable(properties.get(RANKING + "." + PROFILE)).orElse(properties.get("ranking")); - } - static { argumentType = new QueryProfileType(RANKING); argumentType.setStrict(true); argumentType.setBuiltin(true); + // Note: Order here matters as fields are set in this order, and rank feature conversion depends + // on other fields already being set (see RankProfileInputProperties) + argumentType.addField(new FieldDescription(PROFILE, "string", "ranking")); argumentType.addField(new FieldDescription(LOCATION, "string", "location")); - argumentType.addField(new FieldDescription(PROFILE, "string", "ranking")); // Alias repeated in lookupRankProfileIn argumentType.addField(new FieldDescription(SORTING, "string", "sorting sortspec")); argumentType.addField(new FieldDescription(LIST_FEATURES, "string", RANKFEATURES.toString())); argumentType.addField(new FieldDescription(FRESHNESS, "string", "datetime")); @@ -72,7 +69,7 @@ public class Ranking implements Cloneable { argumentType.addField(new FieldDescription(DIVERSITY, new QueryProfileFieldType(Diversity.getArgumentType()))); argumentType.addField(new FieldDescription(SOFTTIMEOUT, new QueryProfileFieldType(SoftTimeout.getArgumentType()))); argumentType.addField(new FieldDescription(MATCHING, new QueryProfileFieldType(Matching.getArgumentType()))); - argumentType.addField(new FieldDescription(FEATURES, "query-profile", "rankfeature")); + argumentType.addField(new FieldDescription(FEATURES, "query-profile", "rankfeature")); // Repeated at the end of RankFeatures argumentType.addField(new FieldDescription(PROPERTIES, "query-profile", "rankproperty")); argumentType.freeze(); argumentTypeName = new CompoundName(argumentType.getId().getName()); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java index 8b6470996d5..02a4199d32e 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java @@ -12,6 +12,7 @@ import com.yahoo.search.query.profile.QueryProfile; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -48,7 +49,7 @@ public class QueryProfileType extends FreezableSimpleComponent { } public QueryProfileType(ComponentId id) { - this(id, new HashMap<>(), new ArrayList<>()); + this(id, new LinkedHashMap<>(), new ArrayList<>()); } private QueryProfileType(ComponentId id, Map<String, FieldDescription> fields, List<QueryProfileType> inherited) { @@ -61,7 +62,7 @@ public class QueryProfileType extends FreezableSimpleComponent { private QueryProfileType(ComponentId id, Map<String, FieldDescription> fields, List<QueryProfileType> inherited, boolean strict, boolean matchAsPath, boolean builtin, Map<String,String> aliases) { - this(id, new HashMap<>(fields), new ArrayList<>(inherited)); + this(id, new LinkedHashMap<>(fields), new ArrayList<>(inherited)); this.strict = strict; this.matchAsPath = matchAsPath; this.builtin = builtin; @@ -79,7 +80,7 @@ public class QueryProfileType extends FreezableSimpleComponent { } // Unfreeze nested query profile references - Map<String, FieldDescription> unfrozenFields = new HashMap<>(); + Map<String, FieldDescription> unfrozenFields = new LinkedHashMap<>(); for (Map.Entry<String, FieldDescription> field : fields.entrySet()) { FieldDescription unfrozenFieldValue = field.getValue(); if (field.getValue().getType() instanceof QueryProfileFieldType) { @@ -196,8 +197,8 @@ public class QueryProfileType extends FreezableSimpleComponent { * Default: true (so all non-declared fields returns true) */ public boolean isOverridable(String fieldName) { - FieldDescription field=getField(fieldName); - if (field==null) return true; + FieldDescription field = getField(fieldName); + if (field == null) return true; return field.isOverridable(); } @@ -208,8 +209,8 @@ public class QueryProfileType extends FreezableSimpleComponent { * null if no types are legal (i.e if the name is not legal) */ public Class<?> getValueClass(String name) { - FieldDescription fieldDescription=getField(name); - if (fieldDescription==null) { + FieldDescription fieldDescription = getField(name); + if (fieldDescription == null) { if (strict) return null; // Undefined -> Not legal else @@ -226,7 +227,7 @@ public class QueryProfileType extends FreezableSimpleComponent { return ((QueryProfileFieldType) fieldDescription.getType()).getQueryProfileType(); } - /** Returns the field type of the given name under this, of null if none */ + /** Returns the type of the given name under this, of null if none */ public FieldType getFieldType(CompoundName name) { FieldDescription field = getField(name.first()); if (field == null) return null; @@ -239,18 +240,31 @@ public class QueryProfileType extends FreezableSimpleComponent { return ((QueryProfileFieldType)fieldType).getQueryProfileType().getFieldType(name.rest()); } + /** Returns the description of the given name under this, of null if none */ + public FieldDescription getField(CompoundName globalName) { + FieldDescription field = getField(globalName.first()); + if (field == null) return null; + + if (globalName.size() == 1) return field; + + FieldType fieldType = field.getType(); + if ( ! (fieldType instanceof QueryProfileFieldType)) return null; + + return ((QueryProfileFieldType)fieldType).getQueryProfileType().getField(globalName.rest()); + } + /** * Returns the description of the field with the given name in this type or an inherited type * (depth first left to right search). Returns null if the field is not defined in this or an inherited profile. */ - public FieldDescription getField(String name) { - FieldDescription field = fields.get(name); + public FieldDescription getField(String localName) { + FieldDescription field = fields.get(localName); if ( field != null ) return field; if ( isFrozen() ) return null; // Inherited are collapsed into this for (QueryProfileType inheritedType : this.inherited() ) { - field = inheritedType.getField(name); + field = inheritedType.getField(localName); if (field != null) return field; } @@ -322,7 +336,7 @@ public class QueryProfileType extends FreezableSimpleComponent { // found in registry but not already added in *this* type (getField also checks parents): extend it if (type != null && ! fields.containsKey(name)) { type = new QueryProfileType(registry.createAnonymousId(type.getIdString()), - new HashMap<>(), + new LinkedHashMap<>(), List.of(type)); } @@ -355,7 +369,7 @@ public class QueryProfileType extends FreezableSimpleComponent { if (inherited().size() == 0) return Collections.unmodifiableMap(fields); // Collapse inherited - Map<String, FieldDescription> allFields = new HashMap<>(); + Map<String, FieldDescription> allFields = new LinkedHashMap<>(); for (QueryProfileType inheritedType : inherited) allFields.putAll(inheritedType.fields()); allFields.putAll(fields); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java index cc6b18af820..e0dea744075 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java @@ -1,18 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.types; -import com.yahoo.language.process.Embedder; import com.yahoo.processing.request.Properties; +import com.yahoo.search.config.internal.TensorConverter; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.SubstituteString; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; -import java.util.ArrayList; -import java.util.List; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; /** * A tensor field type in a query profile @@ -21,8 +17,6 @@ import java.util.regex.Pattern; */ public class TensorFieldType extends FieldType { - private static final Pattern embedderArgumentRegexp = Pattern.compile("^([A-Za-z0-9_\\-.]+),\\s*([\"'].*[\"'])"); - private final TensorType type; /** Creates a tensor field type with information about the kind of tensor this will hold */ @@ -54,71 +48,7 @@ public class TensorFieldType extends FieldType { @Override public Object convertFrom(Object o, ConversionContext context) { if (o instanceof SubstituteString) return new SubstituteStringTensor((SubstituteString) o, type); - Tensor tensor = toTensor(o, context); - if (tensor == null) return null; - if (! tensor.type().isAssignableTo(type)) - throw new IllegalArgumentException("Require a tensor of type " + type); - return tensor; - } - - private Tensor toTensor(Object o, ConversionContext context) { - if (o instanceof Tensor) return (Tensor)o; - if (o instanceof String && isEmbed((String)o)) return embed((String)o, type, context); - if (o instanceof String) return Tensor.from(type, (String)o); - return null; - } - - static boolean isEmbed(String value) { - return value.startsWith("embed("); - } - - static Tensor embed(String s, TensorType type, ConversionContext context) { - if ( ! s.endsWith(")")) - throw new IllegalArgumentException("Expected any string enclosed in embed(), but the argument does not end by ')'"); - String argument = s.substring("embed(".length(), s.length() - 1); - Embedder embedder; - - // Check if arguments specifies an embedder with the format embed(embedder, "text to encode") - Matcher matcher = embedderArgumentRegexp.matcher(argument); - if (matcher.matches()) { - String embedderId = matcher.group(1); - argument = matcher.group(2); - if (!context.embedders().containsKey(embedderId)) { - throw new IllegalArgumentException("Can't find embedder '" + embedderId + "'. " + - "Valid embedders are " + validEmbedders(context.embedders())); - } - embedder = context.embedders().get(embedderId); - } else if (context.embedders().size() == 0) { - throw new IllegalStateException("No embedders provided"); // should never happen - } else if (context.embedders().size() > 1) { - throw new IllegalArgumentException("Multiple embedders are provided but no embedder id is given. " + - "Valid embedders are " + validEmbedders(context.embedders())); - } else { - embedder = context.embedders().entrySet().stream().findFirst().get().getValue(); - } - - return embedder.embed(removeQuotes(argument), toEmbedderContext(context), type); - } - - private static String removeQuotes(String s) { - if (s.startsWith("'") && s.endsWith("'")) { - return s.substring(1, s.length() - 1); - } - if (s.startsWith("\"") && s.endsWith("\"")) { - return s.substring(1, s.length() - 1); - } - return s; - } - - private static String validEmbedders(Map<String, Embedder> embedders) { - List<String> embedderIds = new ArrayList<>(); - embedders.forEach((key, value) -> embedderIds.add(key)); - embedderIds.sort(null); - return String.join(",", embedderIds); - } - - private static Embedder.Context toEmbedderContext(ConversionContext context) { - return new Embedder.Context(context.destination()).setLanguage(context.language()); + return new TensorConverter(context.embedders()).convertTo(type, context.destination(), o, context.language()); } public static TensorFieldType fromTypeString(String s) { diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java index 6769f05bb3e..7f4cee07e8c 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java @@ -2,9 +2,16 @@ package com.yahoo.search.query.properties; import com.yahoo.api.annotations.Beta; +import com.yahoo.language.process.Embedder; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; +import com.yahoo.search.config.SchemaInfo; +import com.yahoo.search.config.internal.TensorConverter; import com.yahoo.search.query.Properties; +import com.yahoo.search.query.Ranking; +import com.yahoo.search.query.ranking.RankFeatures; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; import java.util.Map; @@ -16,19 +23,71 @@ import java.util.Map; @Beta public class RankProfileInputProperties extends Properties { + private final SchemaInfo schemaInfo; private final Query query; + private final TensorConverter tensorConverter; - public RankProfileInputProperties(Query query) { + private SchemaInfo.Session session = null; + + public RankProfileInputProperties(SchemaInfo schemaInfo, Query query, Map<String, Embedder> embedders) { + this.schemaInfo = schemaInfo; this.query = query; + this.tensorConverter = new TensorConverter(embedders); + } + + @Override + public void set(CompoundName name, Object value, Map<String, String> context) { + if (RankFeatures.isFeatureName(name.toString())) { + TensorType expectedType = typeOf(name); + if (expectedType != null) { + try { + value = tensorConverter.convertTo(expectedType, + name.last(), + value, + query.getModel().getLanguage()); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Could not set '" + name + "' to '" + value + "'", e); + } + } + } + super.set(name, value, context); } - /** - * Throws IllegalInputException if the given key cannot be set to the given value. - * This default implementation just passes to the chained properties, if any. - */ + @Override public void requireSettable(CompoundName name, Object value, Map<String, String> context) { - if (chained() != null) - chained().requireSettable(name, value, context); + if (RankFeatures.isFeatureName(name.toString())) { + TensorType expectedType = typeOf(name); + if (expectedType != null) + verifyType(name, value, expectedType); + } + super.requireSettable(name, value, context); + } + + private TensorType typeOf(CompoundName name) { + // Session is lazily resolved because order matters: + // model.sources+restrict must be set in the query before this is done + if (session == null) + session = schemaInfo.newSession(query); + // In addition, the rank profile must be set before features + return session.rankProfileInput(name.last(), query.getRanking().getProfile()); + } + + private void verifyType(CompoundName name, Object value, TensorType expectedType) { + if (value instanceof Tensor) { + TensorType valueType = ((Tensor)value).type(); + if ( ! valueType.isAssignableTo(expectedType)) + throwIllegalInput(name, value, expectedType); + } + else if (expectedType.rank() > 0) { // rank 0 tensor may also be represented as a scalar or string + throwIllegalInput(name, value, expectedType); + } + } + + private void throwIllegalInput(CompoundName name, Object value, TensorType expectedType) { + throw new IllegalArgumentException("Could not set '" + name + "' to '" + value + "': " + + "This input is declared in rank profile '" + query.getRanking().getProfile() + + "' as " + expectedType); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java b/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java index ff2e3949ec4..dab824a6fef 100644 --- a/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java @@ -191,4 +191,8 @@ public class RankFeatures implements Cloneable { return JSON.encode(features); } + public static boolean isFeatureName(String fullPropertyName) { + return fullPropertyName.startsWith("ranking.features.") || fullPropertyName.startsWith("rankfeature."); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index 9374027504e..baf9f35c72b 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -16,6 +16,7 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.cluster.PingableSearcher; +import com.yahoo.search.config.SchemaInfo; import com.yahoo.search.rendering.Renderer; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.search.statistics.TimeTracker; @@ -78,6 +79,8 @@ public class Execution extends com.yahoo.processing.execution.Execution { private IndexFacts indexFacts = null; + private SchemaInfo schemaInfo = SchemaInfo.empty(); + /** The current set of special tokens */ private SpecialTokenRegistry tokenRegistry = null; @@ -116,7 +119,7 @@ public class Execution extends com.yahoo.processing.execution.Execution { * This context is never attached to an execution but is used to carry state into * another context. */ - public Context(SearchChainRegistry searchChainRegistry, IndexFacts indexFacts, + public Context(SearchChainRegistry searchChainRegistry, IndexFacts indexFacts, SchemaInfo schemaInfo, SpecialTokenRegistry tokenRegistry, RendererRegistry rendererRegistry, Linguistics linguistics, Executor executor) { owner = null; @@ -125,50 +128,67 @@ public class Execution extends com.yahoo.processing.execution.Execution { // obviously, the most complete constructor. this.searchChainRegistry = searchChainRegistry; this.indexFacts = indexFacts; + this.schemaInfo = Objects.requireNonNull(schemaInfo); this.tokenRegistry = tokenRegistry; this.rendererRegistry = rendererRegistry; this.linguistics = linguistics; this.executor = Objects.requireNonNull(executor, "The executor cannot be null"); } + /** @deprecated pass schemaInfo */ + @Deprecated + public Context(SearchChainRegistry searchChainRegistry, IndexFacts indexFacts, + SpecialTokenRegistry tokenRegistry, RendererRegistry rendererRegistry, Linguistics linguistics, + Executor executor) { + this(searchChainRegistry, indexFacts, SchemaInfo.empty(), tokenRegistry, rendererRegistry, linguistics, Runnable::run); + } + /** @deprecated pass an executor */ @Deprecated // TODO: Remove on Vespa 8 public Context(SearchChainRegistry searchChainRegistry, IndexFacts indexFacts, SpecialTokenRegistry tokenRegistry, RendererRegistry rendererRegistry, Linguistics linguistics) { - this(searchChainRegistry, indexFacts, tokenRegistry, rendererRegistry, linguistics, Runnable::run); + this(searchChainRegistry, indexFacts, SchemaInfo.empty(), tokenRegistry, rendererRegistry, linguistics, Runnable::run); } /** Creates a Context instance where everything except the given arguments is empty. This is for unit testing.*/ public static Context createContextStub() { - return createContextStub(null, null, null); + return createContextStub(null, null, SchemaInfo.empty(), null); } /** Creates a Context instance where everything except the given arguments is empty. This is for unit testing.*/ public static Context createContextStub(SearchChainRegistry searchChainRegistry) { - return createContextStub(searchChainRegistry, null, null); + return createContextStub(searchChainRegistry, null, SchemaInfo.empty(), null); } /** Creates a Context instance where everything except the given arguments is empty. This is for unit testing.*/ public static Context createContextStub(IndexFacts indexFacts) { - return createContextStub(null, indexFacts, null); + return createContextStub(null, indexFacts, SchemaInfo.empty(), null); } /** Creates a Context instance where everything except the given arguments is empty. This is for unit testing.*/ public static Context createContextStub(SearchChainRegistry searchChainRegistry, IndexFacts indexFacts) { - return createContextStub(searchChainRegistry, indexFacts, null); + return createContextStub(searchChainRegistry, indexFacts, SchemaInfo.empty(), null); } /** Creates a Context instance where everything except the given arguments is empty. This is for unit testing.*/ public static Context createContextStub(IndexFacts indexFacts, Linguistics linguistics) { - return createContextStub(null, indexFacts, linguistics); + return createContextStub(null, indexFacts, SchemaInfo.empty(), linguistics); + } + + public static Context createContextStub(SearchChainRegistry searchChainRegistry, + IndexFacts indexFacts, + Linguistics linguistics) { + return createContextStub(searchChainRegistry, indexFacts, SchemaInfo.empty(), linguistics); } /** Creates a Context instance where everything except the given arguments is empty. This is for unit testing.*/ public static Context createContextStub(SearchChainRegistry searchChainRegistry, IndexFacts indexFacts, + SchemaInfo schemaInfo, Linguistics linguistics) { return new Context(searchChainRegistry != null ? searchChainRegistry : new SearchChainRegistry(), indexFacts != null ? indexFacts : new IndexFacts(), + schemaInfo, null, new RendererRegistry(Runnable::run), linguistics != null ? linguistics : new SimpleLinguistics(), @@ -188,6 +208,7 @@ public class Execution extends com.yahoo.processing.execution.Execution { breakdown = sourceContext.breakdown; if (indexFacts == null) indexFacts = sourceContext.indexFacts; + schemaInfo = sourceContext.schemaInfo; if (tokenRegistry == null) tokenRegistry = sourceContext.tokenRegistry; if (searchChainRegistry == null) @@ -207,6 +228,7 @@ public class Execution extends com.yahoo.processing.execution.Execution { void fill(Context other) { searchChainRegistry = other.searchChainRegistry; indexFacts = other.indexFacts; + schemaInfo = other.schemaInfo; tokenRegistry = other.tokenRegistry; rendererRegistry = other.rendererRegistry; detailedDiagnostics = other.detailedDiagnostics; @@ -219,18 +241,20 @@ public class Execution extends com.yahoo.processing.execution.Execution { // equals() needs to be cheap, that's yet another reason we can only // allow immutables and frozen objects in the context return other.indexFacts == indexFacts - && other.rendererRegistry == rendererRegistry - && other.tokenRegistry == tokenRegistry - && other.searchChainRegistry == searchChainRegistry - && other.detailedDiagnostics == detailedDiagnostics - && other.breakdown == breakdown - && other.linguistics == linguistics - && other.executor == executor; + && other.schemaInfo == schemaInfo + && other.rendererRegistry == rendererRegistry + && other.tokenRegistry == tokenRegistry + && other.searchChainRegistry == searchChainRegistry + && other.detailedDiagnostics == detailedDiagnostics + && other.breakdown == breakdown + && other.linguistics == linguistics + && other.executor == executor; } @Override public int hashCode() { return java.util.Objects.hash(indexFacts, + schemaInfo, rendererRegistry, tokenRegistry, searchChainRegistry, detailedDiagnostics, breakdown, linguistics, @@ -293,28 +317,25 @@ public class Execution extends com.yahoo.processing.execution.Execution { this.indexFacts = indexFacts; } + /** Returns information about the schemas specified in this application. This is never null. */ + public SchemaInfo schemaInfo() { return schemaInfo; } + /** * Returns the search chain registry to use with this execution. This is * a snapshot taken at creation of this execution, use * Context.shallowCopy() to get a correctly instantiated Context if * making a custom Context instance. */ - public SearchChainRegistry searchChainRegistry() { - return searchChainRegistry; - } + public SearchChainRegistry searchChainRegistry() { return searchChainRegistry; } /** * Returns the template registry to use with this execution. This is * a snapshot taken at creation of this execution. */ - public RendererRegistry rendererRegistry() { - return rendererRegistry; - } + public RendererRegistry rendererRegistry() { return rendererRegistry; } /** Returns the current set of special strings for the query tokenizer */ - public SpecialTokenRegistry getTokenRegistry() { - return tokenRegistry; - } + public SpecialTokenRegistry getTokenRegistry() { return tokenRegistry; } /** * Wrapping the incoming special token registry and then setting the @@ -324,13 +345,9 @@ public class Execution extends com.yahoo.processing.execution.Execution { * * @param tokenRegistry a new registry for overriding behavior of following searchers */ - public void setTokenRegistry(SpecialTokenRegistry tokenRegistry) { - this.tokenRegistry = tokenRegistry; - } + public void setTokenRegistry(SpecialTokenRegistry tokenRegistry) { this.tokenRegistry = tokenRegistry; } - public void setDetailedDiagnostics(boolean breakdown) { - this.detailedDiagnostics = breakdown; - } + public void setDetailedDiagnostics(boolean breakdown) { this.detailedDiagnostics = breakdown; } /** * The container has some internal diagnostics mechanisms which may be @@ -342,9 +359,7 @@ public class Execution extends com.yahoo.processing.execution.Execution { * @return whether components exposing different level of diagnostics * should go for the most detailed level */ - public boolean getDetailedDiagnostics() { - return detailedDiagnostics; - } + public boolean getDetailedDiagnostics() { return detailedDiagnostics; } /** * If too many queries time out, the search handler will assume the @@ -352,27 +367,17 @@ public class Execution extends com.yahoo.processing.execution.Execution { * * @return whether the system is assumed to be in a breakdown state */ - public boolean getBreakdown() { - return breakdown; - } + public boolean getBreakdown() { return breakdown; } - public void setBreakdown(boolean breakdown) { - this.breakdown = breakdown; - } + public void setBreakdown(boolean breakdown) { this.breakdown = breakdown; } /** * Returns the {@link Linguistics} object assigned to this Context. This object provides access to all the * linguistic-related APIs, and comes pre-configured with the Execution given. - * - * @return The current Linguistics. */ - public Linguistics getLinguistics() { - return linguistics; - } + public Linguistics getLinguistics() { return linguistics; } - public void setLinguistics(Linguistics linguistics) { - this.linguistics = linguistics; - } + public void setLinguistics(Linguistics linguistics) { this.linguistics = linguistics; } /** * Returns the executor that should be used to execute tasks as part of this execution. @@ -474,15 +479,10 @@ public class Execution extends com.yahoo.processing.execution.Execution { * to ensure only searchChain or searcher is null (and because it's long and * cumbersome). * - * @param searchChain - * the search chain to execute, must be null if searcher is set - * @param context - * execution context for the search - * @param searcherIndex - * index of the first searcher to invoke, see - * Execution(Execution) - * @throws IllegalArgumentException - * if searchChain is null + * @param searchChain the search chain to execute, must be null if searcher is set + * @param context execution context for the search + * @param searcherIndex index of the first searcher to invoke, see Execution(Execution) + * @throws IllegalArgumentException if searchChain is null */ @SuppressWarnings("unchecked") private Execution(Chain<? extends Processor> searchChain, Context context, int searcherIndex) { @@ -493,7 +493,7 @@ public class Execution extends com.yahoo.processing.execution.Execution { super(searchChain, searcherIndex, context.createChildTrace(), context.createChildEnvironment()); this.context.fill(context); contextCache = new Context[searchChain.components().size()]; - entryIndex=searcherIndex; + entryIndex = searcherIndex; timer = new TimeTracker(searchChain, searcherIndex); } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java index 28c8ed8f3cf..06814a4c436 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/ExecutionFactory.java @@ -16,9 +16,11 @@ import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.language.process.SpecialTokenRegistry; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.processing.rendering.Renderer; import com.yahoo.search.Searcher; import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.search.config.SchemaInfo; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.vespa.configdefinition.SpecialtokensConfig; @@ -39,24 +41,17 @@ public class ExecutionFactory extends AbstractComponent { private final SearchChainRegistry searchChainRegistry; private final IndexFacts indexFacts; + private final SchemaInfo schemaInfo; private final SpecialTokenRegistry specialTokens; private final Linguistics linguistics; private final ThreadPoolExecutor renderingExecutor; private final RendererRegistry rendererRegistry; private final Executor executor; - private static ThreadPoolExecutor createRenderingExecutor() { - int threadCount = Runtime.getRuntime().availableProcessors(); - ThreadPoolExecutor executor = new ThreadPoolExecutor(threadCount, threadCount, 1L, TimeUnit.SECONDS, - new LinkedBlockingQueue<>(), - ThreadFactoryFactory.getThreadFactory("common-rendering")); - executor.prestartAllCoreThreads(); - return executor; - } - @Inject public ExecutionFactory(ChainsConfig chainsConfig, IndexInfoConfig indexInfo, + DocumentdbInfoConfig documentdbInfo, QrSearchersConfig clusters, ComponentRegistry<Searcher> searchers, SpecialtokensConfig specialTokens, @@ -65,6 +60,7 @@ public class ExecutionFactory extends AbstractComponent { Executor executor) { this.searchChainRegistry = createSearchChainRegistry(searchers, chainsConfig); this.indexFacts = new IndexFacts(new IndexModel(indexInfo, clusters)).freeze(); + this.schemaInfo = new SchemaInfo(indexInfo, documentdbInfo, clusters); this.specialTokens = new SpecialTokenRegistry(specialTokens); this.linguistics = linguistics; this.renderingExecutor = createRenderingExecutor(); @@ -72,6 +68,19 @@ public class ExecutionFactory extends AbstractComponent { this.executor = executor != null ? executor : Executors.newSingleThreadExecutor(); } + /** @deprecated pass documentDbInfo */ + @Deprecated + public ExecutionFactory(ChainsConfig chainsConfig, + IndexInfoConfig indexInfo, + QrSearchersConfig clusters, + ComponentRegistry<Searcher> searchers, + SpecialtokensConfig specialTokens, + Linguistics linguistics, + ComponentRegistry<Renderer> renderers, + Executor executor) { + this(chainsConfig, indexInfo, new DocumentdbInfoConfig.Builder().build(), clusters, searchers, specialTokens, linguistics, renderers, executor); + } + /** @deprecated pass the container threadpool */ @Deprecated // TODO: Remove on Vespa 8 public ExecutionFactory(ChainsConfig chainsConfig, @@ -81,10 +90,11 @@ public class ExecutionFactory extends AbstractComponent { SpecialtokensConfig specialTokens, Linguistics linguistics, ComponentRegistry<Renderer> renderers) { - this(chainsConfig, indexInfo, clusters, searchers, specialTokens, linguistics, renderers, null); + this(chainsConfig, indexInfo, new DocumentdbInfoConfig.Builder().build(), clusters, searchers, specialTokens, linguistics, renderers, null); } - private SearchChainRegistry createSearchChainRegistry(ComponentRegistry<Searcher> searchers, ChainsConfig chainsConfig) { + private SearchChainRegistry createSearchChainRegistry(ComponentRegistry<Searcher> searchers, + ChainsConfig chainsConfig) { SearchChainRegistry searchChainRegistry = new SearchChainRegistry(searchers); ChainsModel chainsModel = ChainsModelBuilder.buildFromConfig(chainsConfig); ChainsConfigurer.prepareChainRegistry(searchChainRegistry, chainsModel, searchers); @@ -98,7 +108,7 @@ public class ExecutionFactory extends AbstractComponent { */ public Execution newExecution(Chain<? extends Searcher> searchChain) { return new Execution(searchChain, - new Execution.Context(searchChainRegistry, indexFacts, specialTokens, rendererRegistry, linguistics, executor)); + new Execution.Context(searchChainRegistry, indexFacts, schemaInfo, specialTokens, rendererRegistry, linguistics, executor)); } /** @@ -107,7 +117,7 @@ public class ExecutionFactory extends AbstractComponent { */ public Execution newExecution(String searchChainId) { return new Execution(searchChainRegistry().getChain(searchChainId), - new Execution.Context(searchChainRegistry, indexFacts, specialTokens, rendererRegistry, linguistics, executor)); + new Execution.Context(searchChainRegistry, indexFacts, schemaInfo, specialTokens, rendererRegistry, linguistics, executor)); } /** Returns the search chain registry used by this */ @@ -116,6 +126,8 @@ public class ExecutionFactory extends AbstractComponent { /** Returns the renderers known to this */ public RendererRegistry rendererRegistry() { return rendererRegistry; } + public SchemaInfo schemaInfo() { return schemaInfo; } + @Override public void deconstruct() { rendererRegistry.deconstruct(); @@ -132,6 +144,7 @@ public class ExecutionFactory extends AbstractComponent { public static ExecutionFactory empty() { return new ExecutionFactory(new ChainsConfig.Builder().build(), new IndexInfoConfig.Builder().build(), + new DocumentdbInfoConfig.Builder().build(), new QrSearchersConfig.Builder().build(), new ComponentRegistry<>(), new SpecialtokensConfig.Builder().build(), @@ -140,4 +153,13 @@ public class ExecutionFactory extends AbstractComponent { null); } + private static ThreadPoolExecutor createRenderingExecutor() { + int threadCount = Runtime.getRuntime().availableProcessors(); + ThreadPoolExecutor executor = new ThreadPoolExecutor(threadCount, threadCount, 1L, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(), + ThreadFactoryFactory.getThreadFactory("common-rendering")); + executor.prestartAllCoreThreads(); + return executor; + } + } diff --git a/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java index 94b7c140b0f..25d0e184588 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java @@ -2,8 +2,6 @@ package com.yahoo.search.searchers; -import com.yahoo.api.annotations.Beta; - import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.NearestNeighborItem; import com.yahoo.prelude.query.ToolBox; @@ -38,7 +36,7 @@ public class ValidateNearestNeighborSearcher extends Searcher { public ValidateNearestNeighborSearcher(AttributesConfig attributesConfig) { for (AttributesConfig.Attribute a : attributesConfig.attribute()) { if (! validAttributes.containsKey(a.name())) { - validAttributes.put(a.name(), new ArrayList<TensorType>()); + validAttributes.put(a.name(), new ArrayList<>()); } if (a.datatype() == AttributesConfig.Attribute.Datatype.TENSOR) { TensorType tt = TensorType.fromSpec(a.tensortype()); diff --git a/container-search/src/test/java/com/yahoo/search/config/SchemaInfoTest.java b/container-search/src/test/java/com/yahoo/search/config/SchemaInfoTest.java new file mode 100644 index 00000000000..728ebbf8f7f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/config/SchemaInfoTest.java @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.config; + +import com.yahoo.tensor.TensorType; +import com.yahoo.yolean.Exceptions; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class SchemaInfoTest { + + @Test + public void testSchemaInfoConfiguration() { + assertEquals(SchemaInfoTester.createSchemaInfoFromConfig(), SchemaInfoTester.createSchemaInfo()); + } + + @Test + public void testInputResolution() { + var tester = new SchemaInfoTester(); + tester.assertInput(TensorType.fromSpec("tensor(a{},b{})"), + "", "", "commonProfile", "query(myTensor1)"); + tester.assertInput(TensorType.fromSpec("tensor(a{},b{})"), + "ab", "", "commonProfile", "query(myTensor1)"); + tester.assertInput(TensorType.fromSpec("tensor(a{},b{})"), + "a", "", "commonProfile", "query(myTensor1)"); + tester.assertInput(TensorType.fromSpec("tensor(a{},b{})"), + "b", "", "commonProfile", "query(myTensor1)"); + + tester.assertInputConflict(TensorType.fromSpec("tensor(a{},b{})"), + "", "", "inconsistent", "query(myTensor1)"); + tester.assertInputConflict(TensorType.fromSpec("tensor(a{},b{})"), + "ab", "", "inconsistent", "query(myTensor1)"); + tester.assertInput(TensorType.fromSpec("tensor(a{},b{})"), + "ab", "a", "inconsistent", "query(myTensor1)"); + tester.assertInput(TensorType.fromSpec("tensor(x[10])"), + "ab", "b", "inconsistent", "query(myTensor1)"); + tester.assertInput(TensorType.fromSpec("tensor(a{},b{})"), + "a", "", "inconsistent", "query(myTensor1)"); + tester.assertInput(TensorType.fromSpec("tensor(x[10])"), + "b", "", "inconsistent", "query(myTensor1)"); + tester.assertInput(null, + "a", "", "bOnly", "query(myTensor1)"); + tester.assertInput(TensorType.fromSpec("tensor(a{},b{})"), + "ab", "", "bOnly", "query(myTensor1)"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/config/SchemaInfoTester.java b/container-search/src/test/java/com/yahoo/search/config/SchemaInfoTester.java new file mode 100644 index 00000000000..d5b4522f3aa --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/config/SchemaInfoTester.java @@ -0,0 +1,133 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.config; + +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.search.Query; +import com.yahoo.tensor.TensorType; +import com.yahoo.yolean.Exceptions; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class SchemaInfoTester { + + private final SchemaInfo schemaInfo; + + SchemaInfoTester() { + this.schemaInfo = createSchemaInfo(); + } + + SchemaInfo schemaInfo() { return schemaInfo; } + + Query query(String sources, String restrict) { + Map<String, String> params = new HashMap<>(); + if ( ! sources.isEmpty()) + params.put("sources", sources); + if ( ! restrict.isEmpty()) + params.put("restrict", restrict); + return new Query.Builder().setSchemaInfo(schemaInfo) + .setRequestMap(params) + .build(); + } + + void assertInput(TensorType expectedType, String sources, String restrict, String rankProfile, String feature) { + assertEquals(expectedType, + schemaInfo.newSession(query(sources, restrict)).rankProfileInput(feature, rankProfile)); + } + + void assertInputConflict(TensorType expectedType, String sources, String restrict, String rankProfile, String feature) { + try { + assertInput(expectedType, sources, restrict, rankProfile, feature); + } + catch (IllegalArgumentException e) { + assertEquals("Conflicting input type declarations for '" + feature + "'", + e.getMessage().split(":")[0]); + } + } + + static SchemaInfo createSchemaInfo() { + List<Schema> schemas = new ArrayList<>(); + RankProfile common = new RankProfile.Builder("commonProfile") + .addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})")) + .addInput("query(myTensor2)", TensorType.fromSpec("tensor(x[2],y[2])")) + .addInput("query(myTensor3)", TensorType.fromSpec("tensor(x[2],y[2])")) + .addInput("query(myTensor4)", TensorType.fromSpec("tensor<float>(x[5])")) + .build(); + schemas.add(new Schema.Builder("a") + .add(common) + .add(new RankProfile.Builder("inconsistent") + .addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})")) + .build()) + .build()); + schemas.add(new Schema.Builder("b") + .add(common) + .add(new RankProfile.Builder("inconsistent") + .addInput("query(myTensor1)", TensorType.fromSpec("tensor(x[10])")) + .build()) + .add(new RankProfile.Builder("bOnly") + .addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})")) + .build()) + .build()); + Map<String, List<String>> clusters = new HashMap<>(); + clusters.put("ab", List.of("a", "b")); + clusters.put("a", List.of("a")); + return new SchemaInfo(schemas, clusters); + } + + /** Creates the same schema info as createSchemaInfo from config objects. */ + static SchemaInfo createSchemaInfoFromConfig() { + var indexInfoConfig = new IndexInfoConfig.Builder(); + + var rankProfileCommon = new DocumentdbInfoConfig.Documentdb.Rankprofile.Builder(); + rankProfileCommon.name("commonProfile"); + rankProfileCommon.input(new DocumentdbInfoConfig.Documentdb.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(a{},b{})")); + rankProfileCommon.input(new DocumentdbInfoConfig.Documentdb.Rankprofile.Input.Builder().name("query(myTensor2)").type("tensor(x[2],y[2])")); + rankProfileCommon.input(new DocumentdbInfoConfig.Documentdb.Rankprofile.Input.Builder().name("query(myTensor3)").type("tensor(x[2],y[2])")); + rankProfileCommon.input(new DocumentdbInfoConfig.Documentdb.Rankprofile.Input.Builder().name("query(myTensor4)").type("tensor<float>(x[5])")); + + var documentDbInfoInfoConfig = new DocumentdbInfoConfig.Builder(); + + var documentDbA = new DocumentdbInfoConfig.Documentdb.Builder(); + documentDbA.name("a"); + documentDbA.rankprofile(rankProfileCommon); + var rankProfileInconsistentA = new DocumentdbInfoConfig.Documentdb.Rankprofile.Builder(); + rankProfileInconsistentA.name("inconsistent"); + rankProfileInconsistentA.input(new DocumentdbInfoConfig.Documentdb.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(a{},b{})")); + documentDbA.rankprofile(rankProfileInconsistentA); + documentDbInfoInfoConfig.documentdb(documentDbA); + + var documentDbB = new DocumentdbInfoConfig.Documentdb.Builder(); + documentDbB.name("b"); + documentDbB.rankprofile(rankProfileCommon); + var rankProfileInconsistentB = new DocumentdbInfoConfig.Documentdb.Rankprofile.Builder(); + rankProfileInconsistentB.name("inconsistent"); + rankProfileInconsistentB.input(new DocumentdbInfoConfig.Documentdb.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(x[10])")); + documentDbB.rankprofile(rankProfileInconsistentB); + var rankProfileBOnly = new DocumentdbInfoConfig.Documentdb.Rankprofile.Builder(); + rankProfileBOnly.name("bOnly"); + rankProfileBOnly.input(new DocumentdbInfoConfig.Documentdb.Rankprofile.Input.Builder().name("query(myTensor1)").type("tensor(a{},b{})")); + documentDbB.rankprofile(rankProfileBOnly); + documentDbInfoInfoConfig.documentdb(documentDbB); + + var qrSearchersConfig = new QrSearchersConfig.Builder(); + var clusterAB = new QrSearchersConfig.Searchcluster.Builder(); + clusterAB.name("ab"); + clusterAB.searchdef("a").searchdef("b"); + qrSearchersConfig.searchcluster(clusterAB); + var clusterA = new QrSearchersConfig.Searchcluster.Builder(); + clusterA.name("a"); + clusterA.searchdef("a"); + qrSearchersConfig.searchcluster(clusterA); + + return new SchemaInfo(indexInfoConfig.build(), documentDbInfoInfoConfig.build(), qrSearchersConfig.build()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java b/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java new file mode 100644 index 00000000000..1b10e4cd0ba --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/RankProfileInputTest.java @@ -0,0 +1,300 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.language.Language; +import com.yahoo.language.process.Embedder; +import com.yahoo.search.Query; +import com.yahoo.search.config.RankProfile; +import com.yahoo.search.config.Schema; +import com.yahoo.search.config.SchemaInfo; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; +import com.yahoo.yolean.Exceptions; +import org.junit.Test; + +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.fail; + +/** + * Tests queries towards rank profiles using input declarations. + * + * @author bratseth + */ +public class RankProfileInputTest { + + @Test + public void testTensorRankFeatureInRequest() { + String tensorString = "{{a:a1, b:b1}:1.0, {a:a2, b:b1}:2.0}}"; + + { + Query query = createTensor1Query(tensorString, "commonProfile", ""); + assertEquals(0, query.errors().size()); + assertEquals(Tensor.from(tensorString), query.properties().get("ranking.features.query(myTensor1)")); + assertEquals(Tensor.from(tensorString), query.getRanking().getFeatures().getTensor("query(myTensor1)").get()); + } + + { // Partial resolution is sufficient + Query query = createTensor1Query(tensorString, "bOnly", ""); + assertEquals(0, query.errors().size()); + assertEquals(Tensor.from(tensorString), query.properties().get("ranking.features.query(myTensor1)")); + assertEquals(Tensor.from(tensorString), query.getRanking().getFeatures().getTensor("query(myTensor1)").get()); + } + + { // Resolution is limited to the correct sources + Query query = createTensor1Query(tensorString, "bOnly", "sources=a"); + assertEquals(0, query.errors().size()); + assertEquals("Not converted to tensor", + tensorString, query.properties().get("ranking.features.query(myTensor1)")); + } + } + + @Test + public void testTensorRankFeatureInRequestInconsistentInput() { + String tensorString = "{{a:a1, b:b1}:1.0, {a:a2, b:b1}:2.0}}"; + try { + createTensor1Query(tensorString, "inconsistent", ""); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("Conflicting input type declarations for 'query(myTensor1)': " + + "Declared as tensor(a{},b{}) in rank profile 'inconsistent' in schema 'a', " + + "and as tensor(x[10]) in rank profile 'inconsistent' in schema 'b'", + Exceptions.toMessageString(e)); + } + } + + @Test + public void testTensorRankFeatureWithSourceResolution() { + String tensorString = "{{a:a1, b:b1}:1.0, {a:a2, b:b1}:2.0}}"; + + { + createTensor1Query(tensorString, "inconsistent", "sources=a"); + // Success: No exception + } + + try { + createTensor1Query(tensorString, "inconsistent", "sources=ab"); + fail("Excpected exception"); + } + catch (IllegalArgumentException e) { + // success + } + + { + createTensor1Query(tensorString, "inconsistent", "sources=a&restrict=a"); + // Success: No exception + } + } + + @Test + public void testTensorRankFeatureSetProgrammatically() { + String tensorString = "{{a:a1, b:b1}:1.0, {a:a2, b:b1}:2.0}}"; + Query query = new Query.Builder() + .setSchemaInfo(createSchemaInfo()) + .setQueryProfile(createQueryProfile()) // Use the instantiation path with query profiles + .setRequest(HttpRequest.createTestRequest("?" + + "&ranking=commonProfile", + com.yahoo.jdisc.http.HttpRequest.Method.GET)) + .build(); + + query.properties().set("ranking.features.query(myTensor1)", Tensor.from(tensorString)); + assertEquals(Tensor.from(tensorString), query.getRanking().getFeatures().getTensor("query(myTensor1)").get()); + } + + @Test + public void testTensorRankFeatureSetProgrammaticallyWithWrongType() { + Query query = new Query.Builder() + .setSchemaInfo(createSchemaInfo()) + .setQueryProfile(createQueryProfile()) // Use the instantiation path with query profiles + .setRequest(HttpRequest.createTestRequest("?" + + "&ranking=commonProfile", + com.yahoo.jdisc.http.HttpRequest.Method.GET)) + .build(); + + String tensorString = "tensor(x[3]):[0.1, 0.2, 0.3]"; + try { + query.getRanking().getFeatures().put("query(myTensor1)",Tensor.from(tensorString)); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("Could not set 'ranking.features.query(myTensor1)' to 'tensor(x[3]):[0.1, 0.2, 0.3]': " + + "This input is declared in rank profile 'commonProfile' as tensor(a{},b{})", + Exceptions.toMessageString(e)); + } + try { + query.properties().set("ranking.features.query(myTensor1)", Tensor.from(tensorString)); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("Could not set 'ranking.features.query(myTensor1)' to 'tensor(x[3]):[0.1, 0.2, 0.3]': " + + "Require a tensor of type tensor(a{},b{})", + Exceptions.toMessageString(e)); + } + } + + @Test + public void testUnembeddedTensorRankFeatureInRequest() { + String text = "text to embed into a tensor"; + Tensor embedding1 = Tensor.from("tensor<float>(x[5]):[3,7,4,0,0]]"); + Tensor embedding2 = Tensor.from("tensor<float>(x[5]):[1,2,3,4,0]]"); + + Map<String, Embedder> embedders = Map.of( + "emb1", new MockEmbedder(text, Language.UNKNOWN, embedding1) + ); + assertEmbedQuery("embed(" + text + ")", embedding1, embedders); + assertEmbedQuery("embed('" + text + "')", embedding1, embedders); + assertEmbedQuery("embed(\"" + text + "\")", embedding1, embedders); + assertEmbedQuery("embed(emb1, '" + text + "')", embedding1, embedders); + assertEmbedQuery("embed(emb1, \"" + text + "\")", embedding1, embedders); + assertEmbedQueryFails("embed(emb2, \"" + text + "\")", embedding1, embedders, + "Can't find embedder 'emb2'. Valid embedders are emb1"); + + embedders = Map.of( + "emb1", new MockEmbedder(text, Language.UNKNOWN, embedding1), + "emb2", new MockEmbedder(text, Language.UNKNOWN, embedding2) + ); + assertEmbedQuery("embed(emb1, '" + text + "')", embedding1, embedders); + assertEmbedQuery("embed(emb2, '" + text + "')", embedding2, embedders); + assertEmbedQueryFails("embed(emb3, \"" + text + "\")", embedding1, embedders, + "Can't find embedder 'emb3'. Valid embedders are emb1,emb2"); + + // And with specified language + embedders = Map.of( + "emb1", new MockEmbedder(text, Language.ENGLISH, embedding1) + ); + assertEmbedQuery("embed(" + text + ")", embedding1, embedders, Language.ENGLISH.languageCode()); + + embedders = Map.of( + "emb1", new MockEmbedder(text, Language.ENGLISH, embedding1), + "emb2", new MockEmbedder(text, Language.UNKNOWN, embedding2) + ); + assertEmbedQuery("embed(emb1, '" + text + "')", embedding1, embedders, Language.ENGLISH.languageCode()); + assertEmbedQuery("embed(emb2, '" + text + "')", embedding2, embedders, Language.UNKNOWN.languageCode()); + } + + private Query createTensor1Query(String tensorString, String profile, String additionalParams) { + return new Query.Builder() + .setSchemaInfo(createSchemaInfo()) + .setQueryProfile(createQueryProfile()) // Use the instantiation path with query profiles + .setRequest(HttpRequest.createTestRequest("?" + urlEncode("ranking.features.query(myTensor1)") + + "=" + urlEncode(tensorString) + + "&ranking=" + profile + + "&" + additionalParams, + com.yahoo.jdisc.http.HttpRequest.Method.GET)) + .build(); + } + + private String urlEncode(String s) { + return URLEncoder.encode(s, StandardCharsets.UTF_8); + } + + private void assertEmbedQuery(String embed, Tensor expected, Map<String, Embedder> embedders) { + assertEmbedQuery(embed, expected, embedders, null); + } + + private void assertEmbedQuery(String embed, Tensor expected, Map<String, Embedder> embedders, String language) { + String languageParam = language == null ? "" : "&language=" + language; + String destination = "query(myTensor4)"; + + Query query = new Query.Builder().setRequest(HttpRequest.createTestRequest( + "?" + urlEncode("ranking.features." + destination) + + "=" + urlEncode(embed) + + "&ranking=commonProfile" + + languageParam, + com.yahoo.jdisc.http.HttpRequest.Method.GET)) + .setSchemaInfo(createSchemaInfo()) + .setQueryProfile(createQueryProfile()) + .setEmbedders(embedders) + .build(); + assertEquals(0, query.errors().size()); + assertEquals(expected, query.properties().get("ranking.features." + destination)); + assertEquals(expected, query.getRanking().getFeatures().getTensor(destination).get()); + } + + private void assertEmbedQueryFails(String embed, Tensor expected, Map<String, Embedder> embedders, String errMsg) { + Throwable t = assertThrows(IllegalArgumentException.class, () -> assertEmbedQuery(embed, expected, embedders)); + while (t != null) { + if (t.getMessage().equals(errMsg)) return; + t = t.getCause(); + } + fail("Error '" + errMsg + "' not thrown"); + } + + private CompiledQueryProfile createQueryProfile() { + var registry = new QueryProfileRegistry(); + registry.register(new QueryProfile("test")); + return registry.compile().findQueryProfile("test"); + } + + private SchemaInfo createSchemaInfo() { + List<Schema> schemas = new ArrayList<>(); + RankProfile common = new RankProfile.Builder("commonProfile") + .addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})")) + .addInput("query(myTensor2)", TensorType.fromSpec("tensor(x[2],y[2])")) + .addInput("query(myTensor3)", TensorType.fromSpec("tensor(x[2],y[2])")) + .addInput("query(myTensor4)", TensorType.fromSpec("tensor<float>(x[5])")) + .build(); + schemas.add(new Schema.Builder("a") + .add(common) + .add(new RankProfile.Builder("inconsistent") + .addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})")) + .build()) + .build()); + schemas.add(new Schema.Builder("b") + .add(common) + .add(new RankProfile.Builder("inconsistent") + .addInput("query(myTensor1)", TensorType.fromSpec("tensor(x[10])")) + .build()) + .add(new RankProfile.Builder("bOnly") + .addInput("query(myTensor1)", TensorType.fromSpec("tensor(a{},b{})")) + .build()) + .build()); + Map<String, List<String>> clusters = new HashMap<>(); + clusters.put("ab", List.of("a", "b")); + clusters.put("a", List.of("a")); + return new SchemaInfo(schemas, clusters); + } + + private static final class MockEmbedder implements Embedder { + + private final String expectedText; + private final Language expectedLanguage; + private final Tensor tensorToReturn; + + public MockEmbedder(String expectedText, + Language expectedLanguage, + Tensor tensorToReturn) { + this.expectedText = expectedText; + this.expectedLanguage = expectedLanguage; + this.tensorToReturn = tensorToReturn; + } + + @Override + public List<Integer> embed(String text, Embedder.Context context) { + fail("Unexpected call"); + return null; + } + + @Override + public Tensor embed(String text, Embedder.Context context, TensorType tensorType) { + assertEquals(expectedText, text); + assertEquals(expectedLanguage, context.getLanguage()); + assertEquals(tensorToReturn.type(), tensorType); + return tensorToReturn; + } + + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 16398fb1137..c3c68f7596f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -15,12 +15,8 @@ import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.curator.MultiplePathsLock; -import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; @@ -28,7 +24,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCe import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; import com.yahoo.vespa.hosted.controller.deployment.RetriggerEntry; @@ -42,12 +37,10 @@ import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy; import com.yahoo.vespa.hosted.controller.support.access.SupportAccess; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; - import java.io.IOException; import java.io.UncheckedIOException; import java.nio.ByteBuffer; @@ -121,7 +114,6 @@ public class CuratorDb { private final Curator curator; private final Duration tryLockTimeout; - private final StringFlag lockScheme; // For each application id (path), store the ZK node version and its deserialised data - update when version changes. // This will grow to keep all applications in memory, but this should be OK @@ -131,14 +123,13 @@ public class CuratorDb { private final Map<Path, Pair<Integer, NavigableMap<RunId, Run>>> cachedHistoricRuns = new ConcurrentHashMap<>(); @Inject - public CuratorDb(Curator curator, FlagSource flagSource, ServiceRegistry services) { - this(curator, defaultTryLockTimeout, flagSource, services.zoneRegistry().system()); + public CuratorDb(Curator curator, ServiceRegistry services) { + this(curator, defaultTryLockTimeout, services.zoneRegistry().system()); } - CuratorDb(Curator curator, Duration tryLockTimeout, FlagSource flagSource, SystemName system) { + CuratorDb(Curator curator, Duration tryLockTimeout, SystemName system) { this.curator = curator; this.tryLockTimeout = tryLockTimeout; - this.lockScheme = Flags.CONTROLLER_LOCK_SCHEME.bindTo(flagSource); } /** Returns all hostnames configured to be part of this ZooKeeper cluster */ @@ -156,58 +147,19 @@ public class CuratorDb { } public Mutex lock(TenantAndApplicationId id) { - switch (lockScheme.value()) { - case "BOTH": - return new MultiplePathsLock(curator.lock(legacyLockPath(id), defaultLockTimeout.multipliedBy(2)), - curator.lock(lockPath(id), defaultLockTimeout.multipliedBy(2))); - case "OLD": - return curator.lock(legacyLockPath(id), defaultLockTimeout.multipliedBy(2)); - case "NEW": - return curator.lock(lockPath(id), defaultLockTimeout.multipliedBy(2)); - default: - throw new IllegalArgumentException("Unknown lock scheme " + lockScheme.value()); - } + return curator.lock(lockPath(id), defaultLockTimeout.multipliedBy(2)); } public Mutex lockForDeployment(ApplicationId id, ZoneId zone) { - switch (lockScheme.value()) { - case "BOTH": - return new MultiplePathsLock(curator.lock(legacyLockPath(id, zone), defaultLockTimeout), - curator.lock(lockPath(id, zone), defaultLockTimeout)); - case "OLD": - return curator.lock(legacyLockPath(id, zone), deployLockTimeout); - case "NEW": - return curator.lock(lockPath(id, zone), deployLockTimeout); - default: - throw new IllegalArgumentException("Unknown lock scheme " + lockScheme.value()); - } + return curator.lock(lockPath(id, zone), deployLockTimeout); } public Mutex lock(ApplicationId id, JobType type) { - switch (lockScheme.value()) { - case "BOTH": - return new MultiplePathsLock(curator.lock(legacyLockPath(id, type), defaultLockTimeout), - curator.lock(lockPath(id, type), defaultLockTimeout)); - case "OLD": - return curator.lock(legacyLockPath(id, type), defaultLockTimeout); - case "NEW": - return curator.lock(lockPath(id, type), defaultLockTimeout); - default: - throw new IllegalArgumentException("Unknown lock scheme " + lockScheme.value()); - } + return curator.lock(lockPath(id, type), defaultLockTimeout); } public Mutex lock(ApplicationId id, JobType type, Step step) throws TimeoutException { - switch (lockScheme.value()) { - case "BOTH": - return tryLock(legacyLockPath(id, type, step), lockPath(id, type, step)); - case "OLD": - return tryLock(legacyLockPath(id, type, step)); - case "NEW": - return tryLock(lockPath(id, type, step)); - default: - throw new IllegalArgumentException("Unknown lock scheme " + lockScheme.value()); - } + return tryLock(lockPath(id, type, step)); } public Mutex lockRotations() { @@ -289,19 +241,6 @@ public class CuratorDb { } } - /** Try locking with a low timeout, meaning it is OK to fail lock acquisition. - * - * Useful for maintenance jobs, where there is no point in running the jobs back to back. - */ - private Mutex tryLock(Path path, Path path2) throws TimeoutException { - try { - return new MultiplePathsLock(curator.lock(path, tryLockTimeout), curator.lock(path2, tryLockTimeout)); - } - catch (UncheckedTimeoutException e) { - throw new TimeoutException(e.getMessage()); - } - } - private <T> Optional<T> read(Path path, Function<byte[], T> mapper) { return curator.getData(path).filter(data -> data.length > 0).map(mapper); } @@ -730,32 +669,6 @@ public class CuratorDb { .append(tenant.value()); } - private Path legacyLockPath(TenantAndApplicationId application) { - return lockPath(application.tenant()) - .append(application.application().value()); - } - - private Path legacyLockPath(ApplicationId instance) { - return legacyLockPath(TenantAndApplicationId.from(instance)) - .append(instance.instance().value()); - } - - private Path legacyLockPath(ApplicationId instance, ZoneId zone) { - return legacyLockPath(instance) - .append(zone.environment().value()) - .append(zone.region().value()); - } - - private Path legacyLockPath(ApplicationId instance, JobType type) { - return legacyLockPath(instance) - .append(type.jobName()); - } - - private Path legacyLockPath(ApplicationId instance, JobType type, Step step) { - return legacyLockPath(instance, type) - .append(step.name()); - } - private Path lockPath(TenantAndApplicationId application) { return lockRoot.append(application.tenant().value() + ":" + application.application().value()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java index d7391c69bf6..21414339a87 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java @@ -5,8 +5,6 @@ import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.flags.InMemoryFlagSource; - import java.time.Duration; /** @@ -34,7 +32,7 @@ public class MockCuratorDb extends CuratorDb { } public MockCuratorDb(MockCurator curator, SystemName system) { - super(curator, Duration.ofMillis(100), new InMemoryFlagSource(), system); + super(curator, Duration.ofMillis(100), system); this.curator = curator; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 3ce7aca971b..dff379a2249 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -298,16 +298,18 @@ class JobControllerApiHandlerHelper { || deployments.stream().anyMatch(deployment -> deployment.version().isBefore(latestPlatform.versionNumber()))); Cursor availableArray = latestPlatformObject.setArray("available"); + boolean isUpgrade = true; for (VespaVersion available : availablePlatforms) { if ( deployments.stream().anyMatch(deployment -> deployment.version().isAfter(available.versionNumber())) || deployments.stream().noneMatch(deployment -> deployment.version().isBefore(available.versionNumber())) && ! deployments.isEmpty() || status.hasCompleted(stepStatus.instance(), Change.of(available.versionNumber())) - || change.platform().map(available.versionNumber()::compareTo).orElse(1) <= 0) - break; + || change.platform().map(available.versionNumber()::compareTo).orElse(1) < 0) + isUpgrade = false; - availableArray.addObject().setString("platform", available.versionNumber().toFullString()); + Cursor platformObject = availableArray.addObject(); + platformObject.setString("platform", available.versionNumber().toFullString()); + platformObject.setBool("upgrade", isUpgrade || change.platform().map(available.versionNumber()::equals).orElse(false)); } - change.platform().ifPresent(version -> availableArray.addObject().setString("platform", version.toFullString())); toSlime(latestPlatformObject.setArray("blockers"), blockers.stream().filter(ChangeBlocker::blocksVersions)); } List<ApplicationVersion> availableApplications = new ArrayList<>(application.revisions().deployable(false)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json index 3c6e55ae195..481997102d2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json @@ -24,7 +24,12 @@ "upgrade": true, "available": [ { - "platform": "7.1.0" + "platform": "7.1.0", + "upgrade": true + }, + { + "platform": "6.1.0", + "upgrade": false } ], "blockers": [ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index 037a7085eaf..81d363aa3e8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -24,7 +24,8 @@ "upgrade": true, "available": [ { - "platform": "6.1.0" + "platform": "6.1.0", + "upgrade": true } ], "blockers": [ ] @@ -563,7 +564,8 @@ "upgrade": true, "available": [ { - "platform": "6.1.0" + "platform": "6.1.0", + "upgrade": true } ], "blockers": [ ] diff --git a/default_build_settings.cmake b/default_build_settings.cmake index 1925e5f42b6..6f711a72353 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -83,13 +83,13 @@ endfunction() function(setup_vespa_default_build_settings_fedora_36) message("-- Setting up default build settings for fedora 36") set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) - set(DEFAULT_VESPA_LLVM_VERSION "13" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "14" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_fedora_37) message("-- Setting up default build settings for fedora 37") set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) - set(DEFAULT_VESPA_LLVM_VERSION "13" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "14" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_amzn_2) diff --git a/dist/vespa.spec b/dist/vespa.spec index 7e7a3e6ab58..0154467f90e 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -177,14 +177,14 @@ BuildRequires: gmock-devel %endif %if 0%{?fc36} BuildRequires: protobuf-devel -BuildRequires: llvm-devel >= 13.0.1 +BuildRequires: llvm-devel >= 14.0.0 BuildRequires: boost-devel >= 1.76 BuildRequires: gtest-devel BuildRequires: gmock-devel %endif %if 0%{?fc37} BuildRequires: protobuf-devel -BuildRequires: llvm-devel >= 13.0.1 +BuildRequires: llvm-devel >= 14.0.0 BuildRequires: boost-devel >= 1.76 BuildRequires: gtest-devel BuildRequires: gmock-devel @@ -321,10 +321,10 @@ Requires: gtest %define _vespa_llvm_version 13 %endif %if 0%{?fc36} -%define _vespa_llvm_version 13 +%define _vespa_llvm_version 14 %endif %if 0%{?fc37} -%define _vespa_llvm_version 13 +%define _vespa_llvm_version 14 %endif %define _extra_link_directory %{_vespa_deps_prefix}/lib64 %define _extra_include_directory %{_vespa_deps_prefix}/include;/usr/include/openblas @@ -444,10 +444,10 @@ Requires: llvm-libs >= 12.0.0 Requires: llvm-libs >= 13.0.0 %endif %if 0%{?fc36} -Requires: llvm-libs >= 13.0.1 +Requires: llvm-libs >= 14.0.0 %endif %if 0%{?fc37} -Requires: llvm-libs >= 13.0.1 +Requires: llvm-libs >= 14.0.0 %endif %endif Requires: vespa-onnxruntime = 1.11.0 diff --git a/documentapi/src/main/java/com/yahoo/documentapi/Parameters.java b/documentapi/src/main/java/com/yahoo/documentapi/Parameters.java index 082f47f45d4..f51ab4415f8 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/Parameters.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/Parameters.java @@ -11,7 +11,9 @@ import com.yahoo.messagebus.ThrottlePolicy; * @author bratseth */ public class Parameters { + ThrottlePolicy throttlePolicy; + public void setThrottlePolicy(ThrottlePolicy throttlePolicy) { this.throttlePolicy = throttlePolicy; } @@ -19,4 +21,5 @@ public class Parameters { public ThrottlePolicy getThrottlePolicy() { return throttlePolicy; } + } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java index d030c8d0f04..c76a25f5c22 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorParameters.java @@ -41,7 +41,7 @@ public class VisitorParameters extends Parameters { private VisitorControlHandler controlHandler; private Map<String, byte []> libraryParameters = new TreeMap<>(); private Route visitRoute = null; - private float weight = 1; + private final float weight = 1; private long maxFirstPassHits = -1; private long maxTotalHits = -1; private int maxBucketsPerVisitor = 1; diff --git a/linguistics/src/main/java/com/yahoo/language/process/Embedder.java b/linguistics/src/main/java/com/yahoo/language/process/Embedder.java index 238698e898a..c8ba3395c3c 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/Embedder.java +++ b/linguistics/src/main/java/com/yahoo/language/process/Embedder.java @@ -67,7 +67,7 @@ public interface Embedder { /** Sets the language of the text, or UNKNOWN to use language independent embedding */ public Context setLanguage(Language language) { - this.language = language; + this.language = language != null ? language : Language.UNKNOWN; return this; } diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp index 5663ae07686..ed841ae24b3 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp @@ -65,7 +65,6 @@ RankSetup::RankSetup(const BlueprintFactory &factory, const IIndexEnvironment &i _softTimeoutEnabled(false), _softTimeoutTailCost(0.1), _softTimeoutFactor(0.5), - _nearest_neighbor_brute_force_limit(0.05), _global_filter_lower_limit(0.0), _global_filter_upper_limit(1.0), _mutateOnMatch(), diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.h b/searchlib/src/vespa/searchlib/fef/ranksetup.h index bce2af8fa24..6a2871827ab 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.h +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.h @@ -74,7 +74,6 @@ private: bool _softTimeoutEnabled; double _softTimeoutTailCost; double _softTimeoutFactor; - double _nearest_neighbor_brute_force_limit; double _global_filter_lower_limit; double _global_filter_upper_limit; MutateOperation _mutateOnMatch; diff --git a/vespa-application-maven-plugin/src/main/java/com/yahoo/container/plugin/mojo/ApplicationMojo.java b/vespa-application-maven-plugin/src/main/java/com/yahoo/container/plugin/mojo/ApplicationMojo.java index 6fdcf6efe87..b3fd107631b 100644 --- a/vespa-application-maven-plugin/src/main/java/com/yahoo/container/plugin/mojo/ApplicationMojo.java +++ b/vespa-application-maven-plugin/src/main/java/com/yahoo/container/plugin/mojo/ApplicationMojo.java @@ -68,19 +68,18 @@ public class ApplicationMojo extends AbstractMojo { while (current.getParent() != null && current.getParent().getParentArtifact() != null) current = current.getParent(); - boolean hasVespaParent = false; + Version parentVersion = null; Artifact parentArtifact = current.getParentArtifact(); if (parentArtifact != null && (parentArtifact.getGroupId().startsWith("com.yahoo.vespa.") || parentArtifact.getGroupId().startsWith("ai.vespa."))) { - hasVespaParent = true; - Version parentVersion = Version.from(parentArtifact.getVersion()); + parentVersion = Version.from(parentArtifact.getVersion()); if (parentVersion.compareTo(compileVersion) < 0) throw new IllegalArgumentException("compile version (" + compileVersion + ") cannot be higher than parent version (" + parentVersion + ")"); } - String metaData = String.format("{\n \"compileVersion\": \"%s\",\n \"buildTime\": %d,\n \"hasVespaParent\": %b\n}", + String metaData = String.format("{\n \"compileVersion\": \"%s\",\n \"buildTime\": %d,\n \"parentVersion\": %s\n}", compileVersion, System.currentTimeMillis(), - hasVespaParent); + parentVersion); try { Files.write(applicationDestination.toPath().resolve("build-meta.json"), metaData.getBytes(StandardCharsets.UTF_8)); diff --git a/vespalib/src/tests/nice/nice_test.cpp b/vespalib/src/tests/nice/nice_test.cpp index ec1d25be3bd..86fb3be0667 100644 --- a/vespalib/src/tests/nice/nice_test.cpp +++ b/vespalib/src/tests/nice/nice_test.cpp @@ -50,7 +50,7 @@ TEST("require that nice value is tracked per thread") { for (int i = 0; i < 5; ++i) { threads.push_back(run_with_init([my_barrier = &barrier, i] { - nice(i); + [[maybe_unused]] auto nice_result = nice(i); (*my_barrier)(); EXPECT_EQUAL(nice(0), i); })); diff --git a/vespalib/src/vespa/vespalib/stllike/hashtable.h b/vespalib/src/vespa/vespalib/stllike/hashtable.h index e8921498a9f..c064690296f 100644 --- a/vespalib/src/vespa/vespalib/stllike/hashtable.h +++ b/vespalib/src/vespa/vespalib/stllike/hashtable.h @@ -6,6 +6,7 @@ #include <vespa/vespalib/util/alloc.h> #include <algorithm> #include <iterator> +#include <vector> namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/util/nice.cpp b/vespalib/src/vespa/vespalib/util/nice.cpp index cefaaa0347b..4b3b6a266e6 100644 --- a/vespalib/src/vespa/vespalib/util/nice.cpp +++ b/vespalib/src/vespa/vespalib/util/nice.cpp @@ -15,7 +15,7 @@ void set_nice_value(double how_nice) { int now = nice(0); int max = 19; int max_inc = (max - now); - nice(std::min(max_inc, int(how_nice * (max_inc + 1)))); + [[maybe_unused]] auto nice_result = nice(std::min(max_inc, int(how_nice * (max_inc + 1)))); #endif } } diff --git a/vespamalloc/src/vespamalloc/malloc/datasegment.cpp b/vespamalloc/src/vespamalloc/malloc/datasegment.cpp index 4bb36eade43..4f10739c8f9 100644 --- a/vespamalloc/src/vespamalloc/malloc/datasegment.cpp +++ b/vespamalloc/src/vespamalloc/malloc/datasegment.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "datasegment.h" +#include <algorithm> #include <unistd.h> namespace vespamalloc::segment { diff --git a/vespamalloc/src/vespamalloc/malloc/mmappool.cpp b/vespamalloc/src/vespamalloc/malloc/mmappool.cpp index 1d4cb5a6b5e..eadd5aee337 100644 --- a/vespamalloc/src/vespamalloc/malloc/mmappool.cpp +++ b/vespamalloc/src/vespamalloc/malloc/mmappool.cpp @@ -2,6 +2,7 @@ #include "mmappool.h" #include "common.h" #include <sys/mman.h> +#include <algorithm> #include <unistd.h> namespace vespamalloc { |