diff options
63 files changed, 608 insertions, 261 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 49e3bff50b7..9ee36831d6a 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -10,7 +10,6 @@ import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.Zone; import java.io.File; @@ -70,7 +69,6 @@ public interface ModelContext { * - Remove all flag data files from hosted-feature-flag repository */ interface FeatureFlags { - @ModelFeatureFlag(owners = {"jonmv"}, removeAfter = "7.457") default Optional<NodeResources> dedicatedClusterControllerFlavor() { return Optional.of(new NodeResources(0.25, 1, 10, 0.3, NodeResources.DiskSpeed.any, NodeResources.StorageType.any)); } @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Revisit in May or June 2021") default double defaultTermwiseLimit() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"vekterli"}) default boolean useThreePhaseUpdates() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select sequencer type use while feeding") default String feedSequencerType() { throw new UnsupportedOperationException("TODO specify default value"); } @@ -93,7 +91,6 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default List<String> allowedAthenzProxyIdentities() { return List.of(); } @ModelFeatureFlag(owners = {"vekterli"}) default int maxActivationInhibitedOutOfSyncGroups() { return 0; } @ModelFeatureFlag(owners = {"hmusum"}) default String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return ""; } - @ModelFeatureFlag(owners = {"tokle", "bjorncs"}, removeAfter = "7.450") default boolean enableCustomAclMapping() { return true; } @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default int numDistributorStripes() { return 0; } @ModelFeatureFlag(owners = {"arnej"}) default boolean requireConnectivityCheck() { return true; } @ModelFeatureFlag(owners = {"hmusum"}, removeAfter = "7.470") default boolean throwIfResourceLimitsSpecified() { return true; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java index f640af71a59..f84e5c6c3a7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java @@ -18,7 +18,6 @@ import org.w3c.dom.Element; public class Distributor extends ContentNode implements StorDistributormanagerConfig.Producer { PersistenceEngine provider; - private final int numDistributorStripesFlag; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<Distributor> { ModelElement clusterXml; @@ -41,7 +40,6 @@ public class Distributor extends ContentNode implements StorDistributormanagerCo StorageNode.rootFolder + parent.getClusterName() + "/distributor/" + distributionKey, distributionKey); this.provider = provider; - this.numDistributorStripesFlag = properties.featureFlags().numDistributorStripes(); if (distributorBasePort != null) { setBasePort(distributorBasePort); @@ -49,23 +47,21 @@ public class Distributor extends ContentNode implements StorDistributormanagerCo } private int tuneNumDistributorStripes() { - if (numDistributorStripesFlag == -1) { - if (getHostResource() != null) { - int cores = (int)getHostResource().realResources().vcpu(); - // This should match the calculation used when node flavor is not available: - // storage/src/vespa/storage/common/bucket_stripe_utils.cpp - if (cores <= 16) { - return 1; - } else if (cores <= 64) { - return 2; - } else { - return 4; - } - } else { + if (getHostResource() != null && + !getHostResource().realResources().isUnspecified()) { + int cores = (int)getHostResource().realResources().vcpu(); + // This should match the calculation used when node flavor is not available: + // storage/src/vespa/storage/common/bucket_stripe_utils.cpp + if (cores <= 16) { return 1; + } else if (cores <= 64) { + return 2; + } else { + return 4; } + } else { + return 0; } - return numDistributorStripesFlag; } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java index 7689fffe440..de5eaa2278e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java @@ -45,7 +45,7 @@ public class FileSender implements Serializable { public static void send(FileReference fileReference, Collection<? extends AbstractService> services) { if (services.isEmpty()) { throw new IllegalStateException("No service instances. Probably a standalone cluster setting up <nodes> " + - "using 'count' instead of <node> tags."); + "using 'count' instead of <node> tags."); } for (AbstractService service : services) { @@ -57,8 +57,7 @@ public class FileSender implements Serializable { /** * Sends all user configured files for a producer to all given services. */ - public <PRODUCER extends AbstractConfigProducer<?>> - void sendUserConfiguredFiles(PRODUCER producer) { + public <PRODUCER extends AbstractConfigProducer<?>> void sendUserConfiguredFiles(PRODUCER producer) { if (services.isEmpty()) return; @@ -69,7 +68,7 @@ public class FileSender implements Serializable { try { sendUserConfiguredFiles(builder, sentFiles, key); } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Unable to send files for " + key, e); + throw new IllegalArgumentException("Unable to send file specified in " + key, e); } } } @@ -78,7 +77,8 @@ public class FileSender implements Serializable { ConfigDefinition configDefinition = builder.getConfigDefinition(); if (configDefinition == null) { // TODO: throw new IllegalArgumentException("Not able to find config definition for " + builder); - logger.logApplicationPackage(Level.FINE, "Not able to find config definition for " + key + ". Will not send files for this config"); + logger.logApplicationPackage(Level.FINE, "Not able to find config definition for " + key + + ". Will not send files for this config"); return; } // Inspect fields at this level @@ -133,7 +133,7 @@ public class FileSender implements Serializable { for (String name : entries.keySet()) { ConfigPayloadBuilder fileEntry = builder.getObject(name); if (fileEntry.getValue() == null) { - throw new IllegalArgumentException("Unable to send file for field '" + name + "'. Invalid config value " + fileEntry.getValue()); + throw new IllegalArgumentException("Unable to send file for field '" + name + "': Invalid config value " + fileEntry.getValue()); } sendFileEntry(fileEntry, sentFiles); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index c7c02c581f9..0b686db6801 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1057,30 +1057,22 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(2, resolveMaxInhibitedGroupsConfigWithFeatureFlag(2)); } - private int resolveNumDistributorStripesConfigWithFeatureFlag(TestProperties props, Optional<Flavor> flavor) throws Exception { - var cc = createOneNodeCluster(props, flavor); + private int resolveNumDistributorStripesConfig(Optional<Flavor> flavor) throws Exception { + var cc = createOneNodeCluster(new TestProperties(), flavor); var builder = new StorDistributormanagerConfig.Builder(); cc.getDistributorNodes().getChildren().get("0").getConfig(builder); return (new StorDistributormanagerConfig(builder)).num_distributor_stripes(); } - private int resolveNumDistributorStripesConfigWithFeatureFlag(int numStripes) throws Exception { - return resolveNumDistributorStripesConfigWithFeatureFlag(new TestProperties().setNumDistributorStripes(numStripes), Optional.empty()); - } - private int resolveTunedNumDistributorStripesConfig(int numCpuCores) throws Exception { var flavor = new Flavor(new FlavorsConfig.Flavor(new FlavorsConfig.Flavor.Builder().name("test").minCpuCores(numCpuCores))); - return resolveNumDistributorStripesConfigWithFeatureFlag(new TestProperties().setNumDistributorStripes(-1), - Optional.of(flavor)); + return resolveNumDistributorStripesConfig(Optional.of(flavor)); } @Test - public void num_distributor_stripes_config_controlled_by_properties() throws Exception { - assertEquals(0, resolveNumDistributorStripesConfigWithFeatureFlag(new TestProperties(), Optional.empty())); - assertEquals(0, resolveNumDistributorStripesConfigWithFeatureFlag(0)); - assertEquals(1, resolveNumDistributorStripesConfigWithFeatureFlag(1)); - assertEquals(1, resolveNumDistributorStripesConfigWithFeatureFlag(-1)); - assertEquals(4, resolveNumDistributorStripesConfigWithFeatureFlag(4)); + public void num_distributor_stripes_config_defaults_to_zero() throws Exception { + // This triggers tuning when starting the distributor process, based on CPU core sampling on the node. + assertEquals(0, resolveNumDistributorStripesConfig(Optional.empty())); } @Test diff --git a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java index 087be0f17c5..0c4709e4a2c 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java @@ -17,6 +17,7 @@ import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.test.MockMetric; import com.yahoo.language.Linguistics; +import com.yahoo.language.process.Encoder; import com.yahoo.language.simple.SimpleLinguistics; import java.io.File; @@ -140,6 +141,7 @@ public class HandlersConfigurerTestWrapper { protected void configure() { // Needed by e.g. SearchHandler bind(Linguistics.class).to(SimpleLinguistics.class).in(Scopes.SINGLETON); + bind(Encoder.class).to(Encoder.FailingEncoder.class).in(Scopes.SINGLETON); bind(ContainerThreadPool.class).to(SimpleContainerThreadpool.class); bind(Metric.class).to(MockMetric.class); } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index 9fe3728dc2c..853224a5b91 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -40,8 +40,10 @@ import com.yahoo.log.LogSetup; import com.yahoo.messagebus.network.rpc.SlobrokConfigSubscriber; import com.yahoo.net.HostName; import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.defaults.Defaults; import com.yahoo.yolean.Exceptions; +import java.io.IOException; import java.util.Collections; import java.util.HashSet; import java.util.IdentityHashMap; @@ -400,9 +402,17 @@ public final class ConfiguredApplication implements Application { shutdownDeadlineExecutor = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("Shutdown deadline timer")); shutdownDeadlineExecutor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); long delayMillis = 50 * 1000; - shutdownDeadlineExecutor.schedule(() -> com.yahoo.protect.Process.logAndDie( - "Timed out waiting for application shutdown. Please check that all your request handlers " + - "drain their request content channels.", true), delayMillis, TimeUnit.MILLISECONDS); + shutdownDeadlineExecutor.schedule(() -> { + String heapDumpName = Defaults.getDefaults().underVespaHome("var/crash/java_pid.") + ProcessHandle.current().pid() + ".hprof"; + try { + com.yahoo.protect.Process.dumpHeap(heapDumpName, true); + } catch (IOException e) { + log.log(Level.WARNING, "Failed writing heap dump:", e); + } + com.yahoo.protect.Process.logAndDie( + "Timed out waiting for application shutdown. Please check that all your request handlers " + + "drain their request content channels.", true); + }, delayMillis, TimeUnit.MILLISECONDS); } private static void addHandlerBindings(ContainerBuilder builder, diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index b577660c1b9..7016eff3185 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -1786,6 +1786,27 @@ ], "fields": [] }, + "com.yahoo.search.Query$Builder": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public com.yahoo.search.Query$Builder setRequest(java.lang.String)", + "public com.yahoo.search.Query$Builder setRequest(com.yahoo.container.jdisc.HttpRequest)", + "public com.yahoo.container.jdisc.HttpRequest getRequest()", + "public com.yahoo.search.Query$Builder setRequestMap(java.util.Map)", + "public java.util.Map getRequestMap()", + "public com.yahoo.search.Query$Builder setQueryProfile(com.yahoo.search.query.profile.compiled.CompiledQueryProfile)", + "public com.yahoo.search.query.profile.compiled.CompiledQueryProfile getQueryProfile()", + "public com.yahoo.search.Query$Builder setEncoder(com.yahoo.language.process.Encoder)", + "public com.yahoo.language.process.Encoder getEncoder()", + "public com.yahoo.search.Query build()" + ], + "fields": [] + }, "com.yahoo.search.Query$Type": { "superClass": "java.lang.Enum", "interfaces": [], @@ -4237,6 +4258,7 @@ "public" ], "methods": [ + "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, com.yahoo.container.handler.threadpool.ContainerThreadPool, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.container.core.ContainerHttpConfig, com.yahoo.language.process.Encoder, com.yahoo.search.searchchain.ExecutionFactory)", "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, com.yahoo.container.handler.threadpool.ContainerThreadPool, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.container.core.ContainerHttpConfig, com.yahoo.search.searchchain.ExecutionFactory)", "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.container.core.ContainerHttpConfig, com.yahoo.search.searchchain.ExecutionFactory)", "public void <init>(com.yahoo.statistics.Statistics, com.yahoo.jdisc.Metric, java.util.concurrent.Executor, com.yahoo.container.logging.AccessLog, com.yahoo.search.query.profile.config.QueryProfilesConfig, com.yahoo.container.core.ContainerHttpConfig, com.yahoo.search.searchchain.ExecutionFactory)", @@ -5863,6 +5885,7 @@ ], "methods": [ "public void <init>(com.yahoo.search.query.profile.compiled.CompiledQueryProfile)", + "public void <init>(com.yahoo.search.query.profile.compiled.CompiledQueryProfile, com.yahoo.language.process.Encoder)", "public com.yahoo.search.query.profile.compiled.CompiledQueryProfile getQueryProfile()", "public java.lang.Object get(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)", "public void set(com.yahoo.processing.request.CompoundName, java.lang.Object, java.util.Map)", @@ -6229,6 +6252,18 @@ ], "fields": [] }, + "com.yahoo.search.query.profile.types.ConversionContext": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.language.process.Encoder, java.util.Map)", + "public static com.yahoo.search.query.profile.types.ConversionContext empty()" + ], + "fields": [] + }, "com.yahoo.search.query.profile.types.FieldDescription": { "superClass": "java.lang.Object", "interfaces": [ @@ -6276,7 +6311,7 @@ "public abstract java.lang.String toString()", "public abstract java.lang.String toInstanceDescription()", "public abstract java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.QueryProfileRegistry)", - "public abstract java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry)", + "public abstract java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.types.ConversionContext)", "public com.yahoo.tensor.TensorType asTensorType()", "public static com.yahoo.search.query.profile.types.FieldType fromString(java.lang.String, com.yahoo.search.query.profile.types.QueryProfileTypeRegistry)", "public static boolean isLegalFieldValue(java.lang.Object)" @@ -6303,7 +6338,7 @@ "public java.lang.String stringValue()", "public java.lang.String toString()", "public java.lang.String toInstanceDescription()", - "public java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry)", + "public java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.types.ConversionContext)", "public java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.QueryProfileRegistry)", "public int hashCode()", "public boolean equals(java.lang.Object)" @@ -6323,7 +6358,7 @@ "public java.lang.String toString()", "public java.lang.String toInstanceDescription()", "public java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.QueryProfileRegistry)", - "public java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry)" + "public java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.types.ConversionContext)" ], "fields": [] }, @@ -6342,11 +6377,11 @@ "public java.lang.String stringValue()", "public java.lang.String toString()", "public java.lang.String toInstanceDescription()", - "public com.yahoo.search.query.profile.compiled.CompiledQueryProfile convertFrom(java.lang.Object, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry)", + "public com.yahoo.search.query.profile.compiled.CompiledQueryProfile convertFrom(java.lang.Object, com.yahoo.search.query.profile.types.ConversionContext)", "public com.yahoo.search.query.profile.QueryProfile convertFrom(java.lang.Object, com.yahoo.search.query.profile.QueryProfileRegistry)", "public int hashCode()", "public boolean equals(java.lang.Object)", - "public bridge synthetic java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry)", + "public bridge synthetic java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.types.ConversionContext)", "public bridge synthetic java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.QueryProfileRegistry)" ], "fields": [] @@ -6419,7 +6454,7 @@ "public java.lang.String toString()", "public java.lang.String toInstanceDescription()", "public java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.QueryProfileRegistry)", - "public java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry)", + "public java.lang.Object convertFrom(java.lang.Object, com.yahoo.search.query.profile.types.ConversionContext)", "public static com.yahoo.search.query.profile.types.TensorFieldType fromTypeString(java.lang.String)" ], "fields": [] @@ -6496,7 +6531,7 @@ "public" ], "methods": [ - "public void <init>(com.yahoo.search.Query, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry)", + "public void <init>(com.yahoo.search.Query, com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry, com.yahoo.language.process.Encoder)", "public void setParentQuery(com.yahoo.search.Query)", "public java.lang.Object get(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)", "public void set(com.yahoo.processing.request.CompoundName, java.lang.Object, java.util.Map)", diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 8c3a30a5a4d..06b71599103 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -7,6 +7,7 @@ import com.yahoo.collections.Tuple2; import com.yahoo.component.Version; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.fs4.MapEncoder; +import com.yahoo.language.process.Encoder; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.query.Highlight; import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation; @@ -333,20 +334,32 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { public Query(HttpRequest request, Map<String, String> requestMap, CompiledQueryProfile queryProfile) { super(new QueryPropertyAliases(propertyAliases)); this.httpRequest = request; - init(requestMap, queryProfile); + init(requestMap, queryProfile, Encoder.throwsOnUse); } - private void init(Map<String, String> requestMap, CompiledQueryProfile queryProfile) { + // TODO: Deprecate most constructors above here + + private Query(Builder builder) { + this(builder.getRequest(), builder.getRequestMap(), builder.getQueryProfile(), builder.getEncoder()); + } + + private Query(HttpRequest request, Map<String, String> requestMap, CompiledQueryProfile queryProfile, Encoder encoder) { + super(new QueryPropertyAliases(propertyAliases)); + this.httpRequest = request; + init(requestMap, queryProfile, encoder); + } + + private void init(Map<String, String> requestMap, CompiledQueryProfile queryProfile, Encoder encoder) { startTime = httpRequest.getJDiscRequest().creationTime(TimeUnit.MILLISECONDS); if (queryProfile != null) { // Move all request parameters to the query profile just to validate that the parameter settings are legal - Properties queryProfileProperties = new QueryProfileProperties(queryProfile); + Properties queryProfileProperties = new QueryProfileProperties(queryProfile, encoder); properties().chain(queryProfileProperties); // TODO: Just checking legality rather than actually setting would be faster setPropertiesFromRequestMap(requestMap, properties(), true); // Adds errors to the query for illegal set attempts // Create the full chain - properties().chain(new QueryProperties(this, queryProfile.getRegistry())). + properties().chain(new QueryProperties(this, queryProfile.getRegistry(), encoder)). chain(new ModelObjectMap()). chain(new RequestContextProperties(requestMap)). chain(queryProfileProperties). @@ -365,7 +378,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { } else { // bypass these complications if there is no query profile to get values from and validate against properties(). - chain(new QueryProperties(this, CompiledQueryProfileRegistry.empty)). + chain(new QueryProperties(this, CompiledQueryProfileRegistry.empty, encoder)). chain(new PropertyMap()). chain(new DefaultProperties()); setPropertiesFromRequestMap(requestMap, properties(), false); @@ -1112,4 +1125,59 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { getRanking().prepare(); } + public static class Builder { + + private HttpRequest request = null; + private Map<String, String> requestMap = null; + private CompiledQueryProfile queryProfile = null; + private Encoder encoder = Encoder.throwsOnUse; + + public Builder setRequest(String query) { + request = HttpRequest.createTestRequest(query, com.yahoo.jdisc.http.HttpRequest.Method.GET); + return this; + } + + public Builder setRequest(HttpRequest request) { + this.request = request; + return this; + } + + public HttpRequest getRequest() { + if (request == null) + return HttpRequest.createTestRequest("", com.yahoo.jdisc.http.HttpRequest.Method.GET); + return request; + } + + /** Sets the request mao to use explicitly. If not set, the request map will be getRequest().propertyMap() */ + public Builder setRequestMap(Map<String, String> requestMap) { + this.requestMap = requestMap; + return this; + } + + public Map<String, String> getRequestMap() { + if (requestMap == null) + return getRequest().propertyMap(); + return requestMap; + } + + public Builder setQueryProfile(CompiledQueryProfile queryProfile) { + this.queryProfile = queryProfile; + return this; + } + + /** Returns the query profile of this query, or null if none. */ + public CompiledQueryProfile getQueryProfile() { return queryProfile; } + + public Builder setEncoder(Encoder encoder) { + this.encoder = encoder; + return this; + } + + public Encoder getEncoder() { return encoder; } + + /** Creates a new query from this builder. No properties are required to before calling this. */ + public Query build() { return new Query(this); } + + } + } diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 9f67603f62b..d1e57a30206 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -23,6 +23,7 @@ import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Request; import com.yahoo.language.Linguistics; +import com.yahoo.language.process.Encoder; import com.yahoo.net.HostName; import com.yahoo.net.UriTools; import com.yahoo.prelude.query.parser.ParseException; @@ -105,6 +106,8 @@ public class SearchHandler extends LoggingRequestHandler { private final String selfHostname = HostName.getLocalhost(); + private final Encoder encoder; + private final ExecutionFactory executionFactory; private final AtomicLong numRequestsLeftToTrace; @@ -129,6 +132,22 @@ public class SearchHandler extends LoggingRequestHandler { public SearchHandler(Statistics statistics, Metric metric, ContainerThreadPool threadpool, + CompiledQueryProfileRegistry queryProfileRegistry, + ContainerHttpConfig config, + Encoder encoder, + ExecutionFactory executionFactory) { + this(statistics, metric, threadpool.executor(), queryProfileRegistry, encoder, executionFactory, + config.numQueriesToTraceOnDebugAfterConstruction(), + config.hostResponseHeaderKey().equals("") ? Optional.empty() : Optional.of(config.hostResponseHeaderKey())); + } + + /** + * @deprecated Use the @Inject annotated constructor instead. + */ + @Deprecated // Vespa 8 + public SearchHandler(Statistics statistics, + Metric metric, + ContainerThreadPool threadpool, AccessLog ignored, CompiledQueryProfileRegistry queryProfileRegistry, ContainerHttpConfig config, @@ -136,6 +155,10 @@ public class SearchHandler extends LoggingRequestHandler { this(statistics, metric, threadpool.executor(), ignored, queryProfileRegistry, config, executionFactory); } + /** + * @deprecated Use the @Inject annotated constructor instead. + */ + @Deprecated // Vespa 8 public SearchHandler(Statistics statistics, Metric metric, Executor executor, @@ -147,6 +170,7 @@ public class SearchHandler extends LoggingRequestHandler { metric, executor, queryProfileRegistry, + Encoder.throwsOnUse, executionFactory, containerHttpConfig.numQueriesToTraceOnDebugAfterConstruction(), containerHttpConfig.hostResponseHeaderKey().equals("") ? @@ -168,12 +192,17 @@ public class SearchHandler extends LoggingRequestHandler { metric, executor, QueryProfileConfigurer.createFromConfig(queryProfileConfig).compile(), + Encoder.throwsOnUse, executionFactory, containerHttpConfig.numQueriesToTraceOnDebugAfterConstruction(), containerHttpConfig.hostResponseHeaderKey().equals("") ? Optional.empty() : Optional.of( containerHttpConfig.hostResponseHeaderKey())); } + /** + * @deprecated Use the @Inject annotated constructor instead. + */ + @Deprecated // Vespa 8 public SearchHandler(Statistics statistics, Metric metric, Executor executor, @@ -181,19 +210,22 @@ public class SearchHandler extends LoggingRequestHandler { CompiledQueryProfileRegistry queryProfileRegistry, ExecutionFactory executionFactory, Optional<String> hostResponseHeaderKey) { - this(statistics, metric, executor, queryProfileRegistry, executionFactory, 0, hostResponseHeaderKey); + this(statistics, metric, executor, queryProfileRegistry, Encoder.throwsOnUse, + executionFactory, 0, hostResponseHeaderKey); } private SearchHandler(Statistics statistics, Metric metric, Executor executor, CompiledQueryProfileRegistry queryProfileRegistry, + Encoder encoder, ExecutionFactory executionFactory, long numQueriesToTraceOnDebugAfterStartup, Optional<String> hostResponseHeaderKey) { super(executor, metric, true); log.log(Level.FINE, () -> "SearchHandler.init " + System.identityHashCode(this)); this.queryProfileRegistry = queryProfileRegistry; + this.encoder = encoder; this.executionFactory = executionFactory; this.maxThreads = examineExecutor(executor); @@ -297,7 +329,11 @@ public class SearchHandler extends LoggingRequestHandler { String queryProfileName = requestMap.getOrDefault("queryProfile", null); CompiledQueryProfile queryProfile = queryProfileRegistry.findQueryProfile(queryProfileName); - Query query = new Query(request, requestMap, queryProfile); + Query query = new Query.Builder().setRequest(request) + .setRequestMap(requestMap) + .setQueryProfile(queryProfile) + .setEncoder(encoder) + .build(); boolean benchmarking = VespaHeaders.benchmarkOutput(request); boolean benchmarkCoverage = VespaHeaders.benchmarkCoverage(benchmarking, request.getJDiscRequest().headers()); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java index 34fe376150d..e555000272d 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java @@ -2,14 +2,15 @@ package com.yahoo.search.query.profile; import com.yahoo.collections.Pair; +import com.yahoo.language.process.Encoder; import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.processing.request.properties.PropertyMap; import com.yahoo.protect.Validator; -import com.yahoo.search.Query; import com.yahoo.search.query.Properties; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.query.profile.compiled.DimensionalValue; +import com.yahoo.search.query.profile.types.ConversionContext; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileFieldType; import com.yahoo.search.query.profile.types.QueryProfileType; @@ -29,6 +30,7 @@ import java.util.Map; public class QueryProfileProperties extends Properties { private final CompiledQueryProfile profile; + private final Encoder encoder; // Note: The priority order is: values has precedence over references @@ -42,10 +44,15 @@ public class QueryProfileProperties extends Properties { */ private List<Pair<CompoundName, CompiledQueryProfile>> references = null; - /** Creates an instance from a profile, throws an exception if the given profile is null */ public QueryProfileProperties(CompiledQueryProfile profile) { + this(profile, Encoder.throwsOnUse); + } + + /** Creates an instance from a profile, throws an exception if the given profile is null */ + public QueryProfileProperties(CompiledQueryProfile profile, Encoder encoder) { Validator.ensureNotNull("The profile wrapped by this cannot be null", profile); this.profile = profile; + this.encoder = encoder; } /** Returns the query profile backing this, or null if none */ @@ -114,7 +121,9 @@ public class QueryProfileProperties extends Properties { if (fieldDescription != null) { if (i == name.size() - 1) { // at the end of the path, check the assignment type - value = fieldDescription.getType().convertFrom(value, profile.getRegistry()); + value = fieldDescription.getType().convertFrom(value, new ConversionContext(profile.getRegistry(), + encoder, + context)); if (value == null) throw new IllegalInputException("'" + value + "' is not a " + fieldDescription.getType().toInstanceDescription()); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java new file mode 100644 index 00000000000..4aa95741b06 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/ConversionContext.java @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types; + +import com.yahoo.language.Language; +import com.yahoo.language.process.Encoder; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; + +import java.util.Map; + +/** + * @author bratseth + */ +public class ConversionContext { + + private final CompiledQueryProfileRegistry registry; + private final Encoder encoder; + private final Language language; + + public ConversionContext(CompiledQueryProfileRegistry registry, Encoder encoder, Map<String, String> context) { + this.registry = registry; + this.encoder = encoder; + this.language = context.containsKey("language") ? Language.fromLanguageTag(context.get("language")) + : Language.UNKNOWN; + } + + /** Returns the profile registry, or null if none */ + CompiledQueryProfileRegistry getRegistry() {return registry;} + + /** Returns the configured encoder, never null */ + Encoder getEncoder() { return encoder; } + + /** Returns the language, which is never null but may be UNKNOWN */ + Language getLanguage() { return language; } + + /** Returns an empty context */ + public static ConversionContext empty() { + return new ConversionContext(null, Encoder.throwsOnUse, Map.of()); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java index daab5f6a378..7f8836ef2c1 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java @@ -33,7 +33,7 @@ public class FieldDescription implements Comparable<FieldDescription> { } public FieldDescription(String name, String type) { - this(name,FieldType.fromString(type,null)); + this(name,FieldType.fromString(type, null)); } public FieldDescription(String name, FieldType type, boolean mandatory) { @@ -60,7 +60,7 @@ public class FieldDescription implements Comparable<FieldDescription> { * @param overridable whether this can be overridden when first set in a profile. Default: true */ public FieldDescription(String name, String typeString, String aliases, boolean mandatory, boolean overridable) { - this(name,FieldType.fromString(typeString,null),aliases,mandatory,overridable); + this(name,FieldType.fromString(typeString, null), aliases, mandatory, overridable); } public FieldDescription(String name, FieldType type, boolean mandatory, boolean overridable) { diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java index 3bfd33668e6..511b64c7b6e 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldType.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.types; +import com.yahoo.language.process.Encoder; import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; @@ -41,7 +42,7 @@ public abstract class FieldType { public abstract Object convertFrom(Object o, QueryProfileRegistry registry); /** Converts the given type to an instance of this type, if possible. Returns null if not possible. */ - public abstract Object convertFrom(Object o, CompiledQueryProfileRegistry registry); + public abstract Object convertFrom(Object o, ConversionContext context); /** * Returns this type as a tensor type: The true tensor type is this is a tensor field an an empty type - @@ -77,7 +78,7 @@ public abstract class FieldType { if ("query-profile".equals(typeString)) return genericQueryProfileType; if (typeString.startsWith("query-profile:")) - return QueryProfileFieldType.fromString(typeString.substring("query-profile:".length()),registry); + return QueryProfileFieldType.fromString(typeString.substring("query-profile:".length()), registry); throw new IllegalArgumentException("Unknown type '" + typeString + "'"); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java index 1e904e4f970..b1a9820c6fa 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/PrimitiveFieldType.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.types; +import com.yahoo.language.process.Encoder; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; @@ -37,7 +38,7 @@ public class PrimitiveFieldType extends FieldType { } @Override - public Object convertFrom(Object object, CompiledQueryProfileRegistry registry) { + public Object convertFrom(Object object, ConversionContext context) { return convertFrom(object, (QueryProfileRegistry)null); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryFieldType.java index 1797a2bd59f..09c1a4d0cc0 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryFieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryFieldType.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.types; +import com.yahoo.language.process.Encoder; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.yql.YqlQuery; @@ -32,7 +33,7 @@ public class QueryFieldType extends FieldType { } @Override - public Object convertFrom(Object o, CompiledQueryProfileRegistry registry) { + public Object convertFrom(Object o, ConversionContext context) { return convertFrom(o, (QueryProfileRegistry)null); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java index fda2d27e682..6958318bee4 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileFieldType.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.types; +import com.yahoo.language.process.Encoder; import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; @@ -57,11 +58,11 @@ public class QueryProfileFieldType extends FieldType { } @Override - public CompiledQueryProfile convertFrom(Object object, CompiledQueryProfileRegistry registry) { + public CompiledQueryProfile convertFrom(Object object, ConversionContext context) { String profileId = object.toString(); if (profileId.startsWith("ref:")) profileId = profileId.substring("ref:".length()); - CompiledQueryProfile profile = registry.getComponent(profileId); + CompiledQueryProfile profile = context.getRegistry().getComponent(profileId); if (profile == null) return null; if (type != null && ! type.equals(profile.getType())) return null; return profile; diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java index 9699a72cb31..34a9f8d41c3 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/TensorFieldType.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.profile.types; +import com.yahoo.language.Language; +import com.yahoo.language.process.Encoder; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.tensor.Tensor; @@ -38,14 +40,26 @@ public class TensorFieldType extends FieldType { @Override public Object convertFrom(Object o, QueryProfileRegistry registry) { + return convertFrom(o, ConversionContext.empty()); + } + + @Override + public Object convertFrom(Object o, ConversionContext context) { + return convertFrom(o, context.getEncoder(), context.getLanguage()); + } + + private Object convertFrom(Object o, Encoder encoder, Language language) { if (o instanceof Tensor) return o; + if (o instanceof String && ((String)o).startsWith("encode(")) return encode((String)o, encoder, language); if (o instanceof String) return Tensor.from(type, (String)o); return null; } - @Override - public Object convertFrom(Object o, CompiledQueryProfileRegistry registry) { - return convertFrom(o, (QueryProfileRegistry)null); + private Tensor encode(String s, Encoder encoder, Language language) { + if ( ! s.endsWith(")")) + throw new IllegalArgumentException("Expected any string enclosed in encode(), but the argument does not end by ')'"); + String text = s.substring("encode(".length(), s.length() - 1); + return encoder.encode(text, language, type); } public static TensorFieldType fromTypeString(String s) { diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index 4c65e8003e5..02648f84066 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.properties; +import com.yahoo.language.process.Encoder; import com.yahoo.processing.IllegalInputException; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; @@ -11,6 +12,7 @@ import com.yahoo.search.query.Properties; import com.yahoo.search.query.Ranking; import com.yahoo.search.query.Select; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.types.ConversionContext; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.query.ranking.Diversity; @@ -32,10 +34,12 @@ public class QueryProperties extends Properties { private Query query; private final CompiledQueryProfileRegistry profileRegistry; + private final Encoder encoder; - public QueryProperties(Query query, CompiledQueryProfileRegistry profileRegistry) { + public QueryProperties(Query query, CompiledQueryProfileRegistry profileRegistry, Encoder encoder) { this.query = query; this.profileRegistry = profileRegistry; + this.encoder = encoder; } public void setParentQuery(Query query) { @@ -256,9 +260,15 @@ public class QueryProperties extends Properties { else if (key.size() > 2) { String restKey = key.rest().rest().toString(); if (key.get(1).equals(Ranking.FEATURES)) - setRankingFeature(query, restKey, toSpecifiedType(restKey, value, profileRegistry.getTypeRegistry().getComponent("features"))); + setRankingFeature(query, restKey, toSpecifiedType(restKey, + value, + profileRegistry.getTypeRegistry().getComponent("features"), + context)); else if (key.get(1).equals(Ranking.PROPERTIES)) - ranking.getProperties().put(restKey, toSpecifiedType(restKey, value, profileRegistry.getTypeRegistry().getComponent("properties"))); + ranking.getProperties().put(restKey, toSpecifiedType(restKey, + value, + profileRegistry.getTypeRegistry().getComponent("properties"), + context)); else throwIllegalParameter(key.rest().toString(), Ranking.RANKING); } @@ -294,9 +304,15 @@ public class QueryProperties extends Properties { } } else if (key.first().equals("rankfeature") || key.first().equals("featureoverride") ) { // featureoverride is deprecated - setRankingFeature(query, key.rest().toString(), toSpecifiedType(key.rest().toString(), value, profileRegistry.getTypeRegistry().getComponent("features"))); + setRankingFeature(query, key.rest().toString(), toSpecifiedType(key.rest().toString(), + value, + profileRegistry.getTypeRegistry().getComponent("features"), + context)); } else if (key.first().equals("rankproperty")) { - query.getRanking().getProperties().put(key.rest().toString(), toSpecifiedType(key.rest().toString(), value, profileRegistry.getTypeRegistry().getComponent("properties"))); + query.getRanking().getProperties().put(key.rest().toString(), toSpecifiedType(key.rest().toString(), + value, + profileRegistry.getTypeRegistry().getComponent("properties"), + context)); } else if (key.size()==1) { if (key.equals(Query.HITS)) query.setHits(asInteger(value,10)); @@ -359,12 +375,12 @@ public class QueryProperties extends Properties { } } - private Object toSpecifiedType(String key, Object value, QueryProfileType type) { + private Object toSpecifiedType(String key, Object value, QueryProfileType type, Map<String,String> context) { if ( ! ( value instanceof String)) return value; // already typed if (type == null) return value; // no type info -> keep as string FieldDescription field = type.getField(key); if (field == null) return value; // ditto - return field.getType().convertFrom(value, profileRegistry); + return field.getType().convertFrom(value, new ConversionContext(profileRegistry, encoder, context)); } private void throwIllegalParameter(String key,String namespace) { diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java index 39ba607b741..45f53a1cdb9 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java @@ -3,7 +3,10 @@ package com.yahoo.search.query.profile.types.test; import com.yahoo.component.ComponentId; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.language.Language; +import com.yahoo.language.process.Encoder; import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; import com.yahoo.yolean.Exceptions; import com.yahoo.search.Query; import com.yahoo.processing.request.CompoundName; @@ -21,6 +24,8 @@ import org.junit.Test; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.List; import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.assertEquals; @@ -80,6 +85,7 @@ public class QueryProfileTypeTestCase { type.addField(new FieldDescription("ranking.features.query(myTensor1)", FieldType.fromString("tensor(a{},b{})", registry)), registry); type.addField(new FieldDescription("ranking.features.query(myTensor2)", FieldType.fromString("tensor(x[2],y[2])", registry)), registry); type.addField(new FieldDescription("ranking.features.query(myTensor3)", FieldType.fromString("tensor<float>(x{})",registry)), registry); + type.addField(new FieldDescription("ranking.features.query(myTensor4)", FieldType.fromString("tensor<float>(x[5])",registry)), registry); type.addField(new FieldDescription("myQuery", FieldType.fromString("query", registry)), registry); type.addField(new FieldDescription("myQueryProfile", FieldType.fromString("query-profile", registry),"qp"), registry); } @@ -400,15 +406,15 @@ public class QueryProfileTypeTestCase { } @Test - public void testTensorRankFeatureInRequest() throws UnsupportedEncodingException { + public void testTensorRankFeatureInRequest() { QueryProfile profile = new QueryProfile("test"); profile.setType(testtype); registry.register(profile); CompiledQueryProfileRegistry cRegistry = registry.compile(); String tensorString = "{{a:a1, b:b1}:1.0, {a:a2, b:b1}:2.0}}"; - Query query = new Query(HttpRequest.createTestRequest("?" + encode("ranking.features.query(myTensor1)") + - "=" + encode(tensorString), + Query query = new Query(HttpRequest.createTestRequest("?" + urlEncode("ranking.features.query(myTensor1)") + + "=" + urlEncode(tensorString), com.yahoo.jdisc.http.HttpRequest.Method.GET), cRegistry.getComponent("test")); assertEquals(0, query.errors().size()); @@ -418,15 +424,15 @@ public class QueryProfileTypeTestCase { // Expected to work exactly as testTensorRankFeatureInRequest @Test - public void testTensorRankFeatureInRequestWithInheritedQueryProfileType() throws UnsupportedEncodingException { + public void testTensorRankFeatureInRequestWithInheritedQueryProfileType() { QueryProfile profile = new QueryProfile("test"); profile.setType(emptyInheritingTesttype); registry.register(profile); CompiledQueryProfileRegistry cRegistry = registry.compile(); String tensorString = "{{a:a1, b:b1}:1.0, {a:a2, b:b1}:2.0}}"; - Query query = new Query(HttpRequest.createTestRequest("?" + encode("ranking.features.query(myTensor1)") + - "=" + encode(tensorString), + Query query = new Query(HttpRequest.createTestRequest("?" + urlEncode("ranking.features.query(myTensor1)") + + "=" + urlEncode(tensorString), com.yahoo.jdisc.http.HttpRequest.Method.GET), cRegistry.getComponent("test")); assertEquals(0, query.errors().size()); @@ -434,8 +440,41 @@ public class QueryProfileTypeTestCase { assertEquals(Tensor.from(tensorString), query.getRanking().getFeatures().getTensor("query(myTensor1)").get()); } - private String encode(String s) throws UnsupportedEncodingException { - return URLEncoder.encode(s, "utf8"); + @Test + public void testUnencodedTensorRankFeatureInRequest() { + QueryProfile profile = new QueryProfile("test"); + profile.setType(testtype); + registry.register(profile); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + String textToEncode = "text to encode as tensor"; + Tensor expectedTensor = Tensor.from("tensor<float>(x[5]):[3,7,4,0,0]]"); + Query query1 = new Query.Builder().setRequest(HttpRequest.createTestRequest("?" + urlEncode("ranking.features.query(myTensor4)") + + "=" + urlEncode("encode(" + textToEncode + ")"), + com.yahoo.jdisc.http.HttpRequest.Method.GET)) + .setQueryProfile(cRegistry.getComponent("test")) + .setEncoder(new MockEncoder(textToEncode, Language.UNKNOWN, expectedTensor)) + .build(); + assertEquals(0, query1.errors().size()); + assertEquals(expectedTensor, query1.properties().get("ranking.features.query(myTensor4)")); + assertEquals(expectedTensor, query1.getRanking().getFeatures().getTensor("query(myTensor4)").get()); + + // Explicit language + Query query2 = new Query.Builder().setRequest(HttpRequest.createTestRequest("?" + urlEncode("ranking.features.query(myTensor4)") + + "=" + urlEncode("encode(" + textToEncode + ")") + + "&language=en", + com.yahoo.jdisc.http.HttpRequest.Method.GET)) + .setQueryProfile(cRegistry.getComponent("test")) + .setEncoder(new MockEncoder(textToEncode, Language.ENGLISH, expectedTensor)) + .build(); + assertEquals(0, query2.errors().size()); + assertEquals(expectedTensor, query2.properties().get("ranking.features.query(myTensor4)")); + assertEquals(expectedTensor, query2.getRanking().getFeatures().getTensor("query(myTensor4)").get()); + + } + + private String urlEncode(String s) { + return URLEncoder.encode(s, StandardCharsets.UTF_8); } @Test @@ -684,4 +723,34 @@ public class QueryProfileTypeTestCase { } } + private static final class MockEncoder implements Encoder { + + private final String expectedText; + private final Language expectedLanguage; + private final Tensor tensorToReturn; + + public MockEncoder(String expectedText, + Language expectedLanguage, + Tensor tensorToReturn) { + this.expectedText = expectedText; + this.expectedLanguage = expectedLanguage; + this.tensorToReturn = tensorToReturn; + } + + @Override + public List<Integer> encode(String text, Language language) { + fail("Unexpected call"); + return null; + } + + @Override + public Tensor encode(String text, Language language, TensorType tensorType) { + assertEquals(expectedText, text); + assertEquals(expectedLanguage, language); + assertEquals(tensorToReturn.type(), tensorType); + return tensorToReturn; + } + + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzAccessControlService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzAccessControlService.java index 3391965dc67..617e87c55a9 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzAccessControlService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzAccessControlService.java @@ -12,6 +12,7 @@ import java.time.Instant; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; public class AthenzAccessControlService implements AccessControlService { @@ -37,7 +38,7 @@ public class AthenzAccessControlService implements AccessControlService { } Map<AthenzUser, String> users = zmsClient.listPendingRoleApprovals(dataPlaneAccessRole); if (users.containsKey(user)) { - zmsClient.approvePendingRoleMembership(dataPlaneAccessRole, user, expiry); + zmsClient.approvePendingRoleMembership(dataPlaneAccessRole, user, expiry, Optional.empty()); return true; } return false; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java index 77a49c6cbff..f02ba85c9bf 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java @@ -172,7 +172,7 @@ public class ZmsClientMock implements ZmsClient { } @Override - public void approvePendingRoleMembership(AthenzRole athenzRole, AthenzUser athenzUser, Instant expiry) { + public void approvePendingRoleMembership(AthenzRole athenzRole, AthenzUser athenzUser, Instant expiry, Optional<String> reason) { } @Override diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 325d13fb7fe..2e05bbf5d90 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -147,11 +147,17 @@ public class Flags { "Takes effect on next host-admin tick."); public static final UnboundBooleanFlag NEW_SPARE_DISKS = defineFeatureFlag( - "new-spare-disks", false, + "new-spare-disks", true, List.of("hakonhall"), "2021-09-08", "2021-11-08", "Use a new algorithm to calculate the spare disks of a host.", "Takes effect on first run of DiskTask, typically after host-admin restart/upgrade."); + public static final UnboundBooleanFlag LOCAL_SUSPEND = defineFeatureFlag( + "local-suspend", true, + List.of("hakonhall"), "2021-09-21", "2021-10-21", + "Whether the cfghost host admin should suspend against only the local cfg (true and legacy) or all.", + "Takes effect immediately."); + public static final UnboundBooleanFlag USE_UNKNOWN_SERVICE_STATUS = defineFeatureFlag( "use-unknown-service-status", true, List.of("hakonhall"), "2021-09-13", "2021-10-13", @@ -291,6 +297,14 @@ public class Flags { APPLICATION_ID ); + public static final UnboundStringFlag ENDPOINT_CERTIFICATE_ALGORITHM = defineStringFlag( + "endpoint-certificate-algorithm", "", + // Acceptable values are: "rsa_2048", "rsa_4096", "ecdsa_p256" + List.of("andreer"), "2021-09-21", "2022-01-01", + "Selects algorithm used for an applications endpoint certificate, or use provider default if blank", + "Takes effect when a new endpoint certificate is requested (first deployment of new application/instance)", + APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/linguistics/abi-spec.json b/linguistics/abi-spec.json index dc7450678c5..dbf4842ea1a 100644 --- a/linguistics/abi-spec.json +++ b/linguistics/abi-spec.json @@ -918,16 +918,5 @@ "public java.lang.String normalize(java.lang.String)" ], "fields": [] - }, - "com.yahoo.language.sentencepiece.Trie": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()" - ], - "fields": [] } }
\ No newline at end of file diff --git a/linguistics/src/main/java/com/yahoo/language/Linguistics.java b/linguistics/src/main/java/com/yahoo/language/Linguistics.java index 64ef8762be8..8af0fcd42cb 100644 --- a/linguistics/src/main/java/com/yahoo/language/Linguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/Linguistics.java @@ -88,4 +88,5 @@ public interface Linguistics { /** Check if another instance is equivalent to this one */ boolean equals(Linguistics other); + } diff --git a/linguistics/src/main/java/com/yahoo/language/detect/Detection.java b/linguistics/src/main/java/com/yahoo/language/detect/Detection.java index 4b816335154..127777db4d2 100644 --- a/linguistics/src/main/java/com/yahoo/language/detect/Detection.java +++ b/linguistics/src/main/java/com/yahoo/language/detect/Detection.java @@ -44,4 +44,5 @@ public class Detection { public boolean isLocal() { return local; } + } diff --git a/linguistics/src/main/java/com/yahoo/language/detect/DetectionException.java b/linguistics/src/main/java/com/yahoo/language/detect/DetectionException.java index a43dc0cb537..5fceabefae3 100644 --- a/linguistics/src/main/java/com/yahoo/language/detect/DetectionException.java +++ b/linguistics/src/main/java/com/yahoo/language/detect/DetectionException.java @@ -4,11 +4,12 @@ package com.yahoo.language.detect; /** * Exception that is thrown when detection fails. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public final class DetectionException extends RuntimeException { public DetectionException(String str) { super(str); } + } diff --git a/linguistics/src/main/java/com/yahoo/language/detect/Hint.java b/linguistics/src/main/java/com/yahoo/language/detect/Hint.java index 50291c922e8..b6bf4403cf3 100644 --- a/linguistics/src/main/java/com/yahoo/language/detect/Hint.java +++ b/linguistics/src/main/java/com/yahoo/language/detect/Hint.java @@ -2,9 +2,9 @@ package com.yahoo.language.detect; /** - * <p>A hint that can be given to a {@link Detector}.</p> + * A hint that can be given to a {@link Detector}. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class Hint { @@ -35,4 +35,5 @@ public class Hint { public static Hint newInstance(String market, String country) { return new Hint(market, country); } + } diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java index 64888dba183..0edd48f5ee3 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java @@ -49,4 +49,5 @@ public class OpenNlpLinguistics extends SimpleLinguistics { @Override public boolean equals(Linguistics other) { return (other instanceof OpenNlpLinguistics); } + } diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java index 73518876c3f..603905bead8 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java @@ -19,7 +19,6 @@ import opennlp.tools.stemmer.Stemmer; import opennlp.tools.stemmer.snowball.SnowballStemmer; import java.util.ArrayList; -import java.util.Collections; import java.util.List; /** @@ -52,7 +51,7 @@ public class OpenNlpTokenizer implements Tokenizer { @Override public Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents) { - if (input.isEmpty()) return Collections.emptyList(); + if (input.isEmpty()) return List.of(); Stemmer stemmer = stemmerFor(language, stemMode); if (stemmer == null) return simpleTokenizer.tokenize(input, language, stemMode, removeAccents); diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java index bf07c91ba44..9bf1281e015 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java @@ -32,10 +32,10 @@ import java.util.logging.Level; */ public class OptimaizeDetector implements Detector { - static private Object initGuard = new Object(); - static private TextObjectFactory textObjectFactory = null; - static private LanguageDetector languageDetector = null; - static private final Logger log = Logger.getLogger(OptimaizeDetector.class.getName()); + private static final Object initGuard = new Object(); + private static TextObjectFactory textObjectFactory = null; + private static LanguageDetector languageDetector = null; + private static final Logger log = Logger.getLogger(OptimaizeDetector.class.getName()); static private void initOptimaize() { synchronized (initGuard) { @@ -60,7 +60,7 @@ public class OptimaizeDetector implements Detector { } } - private SimpleDetector simpleDetector = new SimpleDetector(); + private final SimpleDetector simpleDetector = new SimpleDetector(); public OptimaizeDetector() { initOptimaize(); diff --git a/linguistics/src/main/java/com/yahoo/language/process/ProcessingException.java b/linguistics/src/main/java/com/yahoo/language/process/ProcessingException.java index 752992f5a26..99576240635 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/ProcessingException.java +++ b/linguistics/src/main/java/com/yahoo/language/process/ProcessingException.java @@ -15,4 +15,5 @@ public class ProcessingException extends RuntimeException { public ProcessingException(String message, Throwable cause) { super(message, cause); } + } diff --git a/linguistics/src/main/java/com/yahoo/language/process/StemList.java b/linguistics/src/main/java/com/yahoo/language/process/StemList.java index a38a2e51cb6..d5451e7660d 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/StemList.java +++ b/linguistics/src/main/java/com/yahoo/language/process/StemList.java @@ -3,6 +3,7 @@ package com.yahoo.language.process; import java.util.AbstractList; import java.util.ArrayList; +import java.util.List; /** * A list of strings which does not allow for duplicate elements. @@ -10,7 +11,8 @@ import java.util.ArrayList; * @author steinar */ public class StemList extends AbstractList<String> { - private final ArrayList<String> stems; + + private final List<String> stems; public StemList() { this(new String[0]); diff --git a/linguistics/src/main/java/com/yahoo/language/process/StemMode.java b/linguistics/src/main/java/com/yahoo/language/process/StemMode.java index 628f6910c9e..4adb5de62da 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/StemMode.java +++ b/linguistics/src/main/java/com/yahoo/language/process/StemMode.java @@ -10,16 +10,10 @@ package com.yahoo.language.process; */ public enum StemMode { - NONE(0), - DEFAULT(1), - ALL(2), - SHORTEST(4), - BEST(5); - - private final int value; - - StemMode(int value) { - this.value = value; - } + NONE, + DEFAULT, + ALL, + SHORTEST, + BEST; } diff --git a/linguistics/src/main/java/com/yahoo/language/process/Stemmer.java b/linguistics/src/main/java/com/yahoo/language/process/Stemmer.java index a2d0d0a84c9..1c6180c1f59 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/Stemmer.java +++ b/linguistics/src/main/java/com/yahoo/language/process/Stemmer.java @@ -18,7 +18,7 @@ public interface Stemmer { * @param input the string to stem. * @param mode the stemming mode * @param language the language to use for stemming - * @return list of possible stems. Empty if none. + * @return a list of possible stems. Empty if none. * @throws ProcessingException thrown if there is an exception stemming this input */ List<StemList> stem(String input, StemMode mode, Language language); diff --git a/linguistics/src/main/java/com/yahoo/language/process/StemmerImpl.java b/linguistics/src/main/java/com/yahoo/language/process/StemmerImpl.java index f401ddaba99..dd830570e88 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/StemmerImpl.java +++ b/linguistics/src/main/java/com/yahoo/language/process/StemmerImpl.java @@ -43,4 +43,5 @@ public class StemmerImpl implements Stemmer { } } } + } diff --git a/linguistics/src/main/java/com/yahoo/language/process/TokenScript.java b/linguistics/src/main/java/com/yahoo/language/process/TokenScript.java index efe4073d97e..ff87b9b128b 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/TokenScript.java +++ b/linguistics/src/main/java/com/yahoo/language/process/TokenScript.java @@ -5,7 +5,7 @@ package com.yahoo.language.process; * List of token scripts (e.g. latin, japanese, chinese, etc.) which may warrant different * linguistics treatment. * - * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias Mølster Lidal</a> + * @author Mathias Mølster Lidal */ public enum TokenScript { diff --git a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Trie.java b/linguistics/src/main/java/com/yahoo/language/sentencepiece/Trie.java index 9abed89e7a2..8e7c2db2ed3 100644 --- a/linguistics/src/main/java/com/yahoo/language/sentencepiece/Trie.java +++ b/linguistics/src/main/java/com/yahoo/language/sentencepiece/Trie.java @@ -9,7 +9,7 @@ import java.util.Map; * * @author bratseth */ -public class Trie { +class Trie { final Node root = new Node(); diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java index 3de0eb3e997..e15c6257414 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java @@ -70,15 +70,6 @@ public class SimpleDetector implements Detector { block == Character.UnicodeBlock.HANGUL_COMPATIBILITY_JAMO) { return Language.KOREAN; } - // katakana phonetic extensions. - if (0x31f0 <= c && c <= 0x31ff) { - // See http://www.unicode.org/charts/PDF/U31F0.pdf - // This is a special case because This range of character - // codes is classified as unasigned in - // Character.UnicodeBlock. But clearly it is assigned as - // per above. - return Language.JAPANESE; - } if (0x31f0 <= c && c <= 0x31ff || // these are standard character blocks for japanese characters. block == Character.UnicodeBlock.HIRAGANA || block == Character.UnicodeBlock.KATAKANA || diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java index 026bc8add25..b319c343510 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java @@ -72,4 +72,5 @@ public class SimpleLinguistics implements Linguistics { @Override public boolean equals(Linguistics other) { return (other instanceof SimpleLinguistics); } + } diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenType.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenType.java index d7eb8a72ed8..b5c11b13c67 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenType.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenType.java @@ -65,4 +65,5 @@ public class SimpleTokenType { } throw new UnsupportedOperationException(String.valueOf(Character.getType(codePoint))); } + } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java index 1bbda1a463c..2603b9025c2 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java @@ -63,7 +63,6 @@ public class ApplicationMetricsRetriever extends AbstractComponent implements Ru taskTimeout = timeout(clients.size()); stopped = false; consumerSet = new HashSet<>(); - consumerSet.add(defaultMetricsConsumerId); httpClient.start(); pollThread = new Thread(this, "metrics-poller"); pollThread.setDaemon(true); @@ -114,7 +113,7 @@ public class ApplicationMetricsRetriever extends AbstractComponent implements Ru super.deconstruct(); } - public Map<Node, List<MetricsPacket>> getMetrics() { + Map<Node, List<MetricsPacket>> getMetrics() { return getMetrics(defaultMetricsConsumerId); } @@ -141,7 +140,8 @@ public class ApplicationMetricsRetriever extends AbstractComponent implements Ru } long before = pollCount; pollThread.notifyAll(); - while (pollCount == before) { + while (pollCount <= before + 1) { + pollThread.notifyAll(); pollThread.wait(); } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java index d07a52f42bd..3d834106ebc 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java @@ -24,7 +24,9 @@ import java.util.logging.Logger; public class ConfigSentinelClient extends AbstractComponent { private final static Logger log = Logger.getLogger(ConfigSentinelClient.class.getName()); + private static final Spec SPEC = new Spec("localhost", 19097); private final Supervisor supervisor; + private Target connection = null; @Inject public ConfigSentinelClient() { @@ -33,6 +35,12 @@ public class ConfigSentinelClient extends AbstractComponent { @Override public void deconstruct() { + synchronized (this) { + if (connection != null) { + connection.close(); + connection = null; + } + } supervisor.transport().shutdown().join(); super.deconstruct(); } @@ -126,7 +134,7 @@ public class ConfigSentinelClient extends AbstractComponent { } for (int i = 1; i < parts.length; i++) { - String keyValue[] = parts[i].split("="); + String [] keyValue = parts[i].split("="); String key = keyValue[0]; String value = keyValue[1]; @@ -155,26 +163,24 @@ public class ConfigSentinelClient extends AbstractComponent { String sentinelLs() { String servicelist = ""; - int rpcPort = 19097; - Spec spec = new Spec("localhost", rpcPort); - Target connection = supervisor.connect(spec); - try { - if (connection.isValid()) { - Request req = new Request("sentinel.ls"); - connection.invokeSync(req, 5.0); - if (req.errorCode() == ErrorCode.NONE && - req.checkReturnTypes("s")) - { - servicelist = req.returnValues().get(0).asString(); - } else { - log.log(Level.WARNING, "Bad answer to RPC request: " + req.errorMessage()); - } + synchronized (this) { + if (connection == null || ! connection.isValid()) { + connection = supervisor.connect(SPEC); + } + } + if (connection.isValid()) { + Request req = new Request("sentinel.ls"); + connection.invokeSync(req, 5.0); + if (req.errorCode() == ErrorCode.NONE && + req.checkReturnTypes("s")) + { + servicelist = req.returnValues().get(0).asString(); } else { - log.log(Level.WARNING, "Could not connect to sentinel at: "+spec); + log.log(Level.WARNING, "Bad answer to RPC request: " + req.errorMessage()); } - return servicelist; - } finally { - connection.close(); + } else { + log.log(Level.WARNING, "Could not connect to sentinel at: " + SPEC); } + return servicelist; } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java index 09087c32914..59db14670aa 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java @@ -32,8 +32,8 @@ public class RemoteHealthMetricFetcher extends HttpMetricFetcher { * Connect to remote service over http and fetch metrics */ public HealthMetric getHealth(int fetchCount) { - try { - return createHealthMetrics(getJson(), fetchCount); + try (InputStream stream = getJson()) { + return createHealthMetrics(stream, fetchCount); } catch (IOException | InterruptedException | ExecutionException e) { logMessageNoResponse(errMsgNoResponse(e), fetchCount); byte [] empty = {'{','}'}; diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java index 3ff0daf37a7..3ee1e05c263 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java @@ -22,8 +22,8 @@ public class RemoteMetricsFetcher extends HttpMetricFetcher { * Connect to remote service over http and fetch metrics */ public void getMetrics(MetricsParser.Consumer consumer, int fetchCount) { - try { - createMetrics(getJson(), consumer, fetchCount); + try (InputStream stream = getJson()) { + createMetrics(stream, consumer, fetchCount); } catch (IOException | InterruptedException | ExecutionException e) { } } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java index 5f1c045781c..d568e83c9ad 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java @@ -85,6 +85,7 @@ public class ApplicationMetricsHandlerTest { ApplicationMetricsHandler handler = new ApplicationMetricsHandler(Executors.newSingleThreadExecutor(), applicationMetricsRetriever, getMetricsConsumers()); + applicationMetricsRetriever.getMetrics(defaultMetricsConsumerId); applicationMetricsRetriever.getMetrics(ConsumerId.toConsumerId(CUSTOM_CONSUMER)); applicationMetricsRetriever.startPollAndWait(); testDriver = new RequestHandlerTestDriver(handler); diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetrieverTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetrieverTest.java index 09cc355d292..c98b962f671 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetrieverTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetrieverTest.java @@ -53,6 +53,7 @@ public class ApplicationMetricsRetrieverTest { .willReturn(aResponse().withBody(RESPONSE))); ApplicationMetricsRetriever retriever = new ApplicationMetricsRetriever(config); + retriever.getMetrics(); retriever.startPollAndWait(); var metricsByNode = retriever.getMetrics(); assertEquals(1, metricsByNode.size()); @@ -71,6 +72,7 @@ public class ApplicationMetricsRetrieverTest { .willReturn(aResponse().withBody(RESPONSE))); ApplicationMetricsRetriever retriever = new ApplicationMetricsRetriever(config); + retriever.getMetrics(); retriever.startPollAndWait(); var metricsByNode = retriever.getMetrics(); assertEquals(2, metricsByNode.size()); @@ -100,6 +102,7 @@ public class ApplicationMetricsRetrieverTest { .willReturn(aResponse().withBody(RESPONSE))); ApplicationMetricsRetriever retriever = new ApplicationMetricsRetriever(config); + retriever.getMetrics(); retriever.startPollAndWait(); var metricsByNode = retriever.getMetrics(); assertEquals(2, metricsByNode.size()); @@ -134,6 +137,7 @@ public class ApplicationMetricsRetrieverTest { .withFixedDelay(10))); ApplicationMetricsRetriever retriever = new ApplicationMetricsRetriever(config); + retriever.getMetrics(); retriever.setTaskTimeout(Duration.ofMillis(1)); retriever.startPollAndWait(); assertTrue(retriever.getMetrics().get(node).isEmpty()); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java index 90768facf34..d8d58aee8c2 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.configserver; +import java.net.URI; import java.time.Duration; import java.util.Optional; @@ -10,41 +11,65 @@ import java.util.Optional; * @author freva */ public interface ConfigServerApi extends AutoCloseable { - class Params { - private Optional<Duration> connectionTimeout; + + /** + * The result of sending a request to a config server results in a jackson response or exception. If a response + * is returned, an instance of this interface is conferred to discard the result and try the next config server, + * unless it was the last attempt. + * + * @param <T> the type of the returned jackson response + */ + interface RetryPolicy<T> { + boolean tryNextConfigServer(URI configServerEndpoint, T response); + } + + class Params<T> { + private Optional<Duration> connectionTimeout = Optional.empty(); + + private RetryPolicy<T> retryPolicy = (configServerEndpoint, response) -> false; + + public Params() {} /** Set the socket connect and read timeouts. */ - public Params setConnectionTimeout(Duration connectionTimeout) { + public Params<T> setConnectionTimeout(Duration connectionTimeout) { this.connectionTimeout = Optional.of(connectionTimeout); return this; } public Optional<Duration> getConnectionTimeout() { return connectionTimeout; } + + /** Set the retry policy to use against the config servers. */ + public Params<T> setRetryPolicy(RetryPolicy<T> retryPolicy) { + this.retryPolicy = retryPolicy; + return this; + } + + public RetryPolicy<T> getRetryPolicy() { return retryPolicy; } } - <T> T get(String path, Class<T> wantedReturnType, Params params); + <T> T get(String path, Class<T> wantedReturnType, Params<T> params); default <T> T get(String path, Class<T> wantedReturnType) { - return get(path, wantedReturnType, null); + return get(path, wantedReturnType, new Params<>()); } - <T> T post(String path, Object bodyJsonPojo, Class<T> wantedReturnType, Params params); + <T> T post(String path, Object bodyJsonPojo, Class<T> wantedReturnType, Params<T> params); default <T> T post(String path, Object bodyJsonPojo, Class<T> wantedReturnType) { - return post(path, bodyJsonPojo, wantedReturnType, null); + return post(path, bodyJsonPojo, wantedReturnType, new Params<>()); } - <T> T put(String path, Optional<Object> bodyJsonPojo, Class<T> wantedReturnType, Params params); + <T> T put(String path, Optional<Object> bodyJsonPojo, Class<T> wantedReturnType, Params<T> params); default <T> T put(String path, Optional<Object> bodyJsonPojo, Class<T> wantedReturnType) { - return put(path, bodyJsonPojo, wantedReturnType, null); + return put(path, bodyJsonPojo, wantedReturnType, new Params<>()); } - <T> T patch(String path, Object bodyJsonPojo, Class<T> wantedReturnType, Params params); + <T> T patch(String path, Object bodyJsonPojo, Class<T> wantedReturnType, Params<T> params); default <T> T patch(String path, Object bodyJsonPojo, Class<T> wantedReturnType) { - return patch(path, bodyJsonPojo, wantedReturnType, null); + return patch(path, bodyJsonPojo, wantedReturnType, new Params<>()); } - <T> T delete(String path, Class<T> wantedReturnType, Params params); + <T> T delete(String path, Class<T> wantedReturnType, Params<T> params); default <T> T delete(String path, Class<T> wantedReturnType) { - return delete(path, wantedReturnType, null); + return delete(path, wantedReturnType, new Params<>()); } /** Close the underlying HTTP client and any threads this class might have started. */ diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java index 67dcb6744ce..c41528c64ec 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java @@ -106,8 +106,10 @@ public class ConfigServerApiImpl implements ConfigServerApi { HttpUriRequest createRequest(URI configServerUri) throws JsonProcessingException, UnsupportedEncodingException; } - private <T> T tryAllConfigServers(CreateRequest requestFactory, Class<T> wantedReturnType) { + private <T> T tryAllConfigServers(CreateRequest requestFactory, Class<T> wantedReturnType, Params<T> params) { + T lastResult = null; Exception lastException = null; + for (URI configServer : configServers) { var request = Exceptions.uncheck(() -> requestFactory.createRequest(configServer)); try (CloseableHttpResponse response = client.execute(request)) { @@ -115,15 +117,26 @@ public class ConfigServerApiImpl implements ConfigServerApi { HttpException.handleStatusCode(response.getStatusLine().getStatusCode(), request.getMethod() + " " + request.getURI() + " failed with response '" + responseBody + "'"); + + T result; try { - return mapper.readValue(responseBody, wantedReturnType); + result = mapper.readValue(responseBody, wantedReturnType); } catch (IOException e) { throw new UncheckedIOException("Failed parse response from config server", e); } + + if (params.getRetryPolicy().tryNextConfigServer(configServer, result)) { + lastResult = result; + lastException = null; + } else { + return result; + } } catch (HttpException e) { if (!e.isRetryable()) throw e; + lastResult = null; lastException = e; } catch (Exception e) { + lastResult = null; lastException = e; if (configServers.size() == 1) break; @@ -136,6 +149,11 @@ public class ConfigServerApiImpl implements ConfigServerApi { } } + if (lastResult != null) { + logger.warning("Giving up after trying all config servers: returning result: " + lastResult); + return lastResult; + } + String prefix = configServers.size() == 1 ? "Request against " + configServers.get(0) + " failed: " : "All requests against the config servers (" + configServers + ") failed, last as follows: "; @@ -143,8 +161,8 @@ public class ConfigServerApiImpl implements ConfigServerApi { } @Override - public <T> T put(String path, Optional<Object> bodyJsonPojo, Class<T> wantedReturnType, Params paramsOrNull) { - Optional<RequestConfig> requestConfigOverride = getRequestConfigOverride(paramsOrNull); + public <T> T put(String path, Optional<Object> bodyJsonPojo, Class<T> wantedReturnType, Params<T> params) { + Optional<RequestConfig> requestConfigOverride = getRequestConfigOverride(params); return tryAllConfigServers(configServer -> { HttpPut put = new HttpPut(configServer.resolve(path)); requestConfigOverride.ifPresent(put::setConfig); @@ -153,51 +171,51 @@ public class ConfigServerApiImpl implements ConfigServerApi { put.setEntity(new StringEntity(mapper.writeValueAsString(bodyJsonPojo.get()))); } return put; - }, wantedReturnType); + }, wantedReturnType, params); } @Override - public <T> T patch(String path, Object bodyJsonPojo, Class<T> wantedReturnType, Params paramsOrNull) { - Optional<RequestConfig> requestConfigOverride = getRequestConfigOverride(paramsOrNull); + public <T> T patch(String path, Object bodyJsonPojo, Class<T> wantedReturnType, Params<T> params) { + Optional<RequestConfig> requestConfigOverride = getRequestConfigOverride(params); return tryAllConfigServers(configServer -> { HttpPatch patch = new HttpPatch(configServer.resolve(path)); requestConfigOverride.ifPresent(patch::setConfig); setContentTypeToApplicationJson(patch); patch.setEntity(new StringEntity(mapper.writeValueAsString(bodyJsonPojo))); return patch; - }, wantedReturnType); + }, wantedReturnType, params); } @Override - public <T> T delete(String path, Class<T> wantedReturnType, Params paramsOrNull) { - Optional<RequestConfig> requestConfigOverride = getRequestConfigOverride(paramsOrNull); + public <T> T delete(String path, Class<T> wantedReturnType, Params<T> params) { + Optional<RequestConfig> requestConfigOverride = getRequestConfigOverride(params); return tryAllConfigServers(configServer -> { HttpDelete delete = new HttpDelete(configServer.resolve(path)); requestConfigOverride.ifPresent(delete::setConfig); return delete; - }, wantedReturnType); + }, wantedReturnType, params); } @Override - public <T> T get(String path, Class<T> wantedReturnType, Params paramsOrNull) { - Optional<RequestConfig> requestConfig = getRequestConfigOverride(paramsOrNull); + public <T> T get(String path, Class<T> wantedReturnType, Params<T> params) { + Optional<RequestConfig> requestConfig = getRequestConfigOverride(params); return tryAllConfigServers(configServer -> { HttpGet get = new HttpGet(configServer.resolve(path)); requestConfig.ifPresent(get::setConfig); return get; - }, wantedReturnType); + }, wantedReturnType, params); } @Override - public <T> T post(String path, Object bodyJsonPojo, Class<T> wantedReturnType, Params paramsOrNull) { - Optional<RequestConfig> requestConfigOverride = getRequestConfigOverride(paramsOrNull); + public <T> T post(String path, Object bodyJsonPojo, Class<T> wantedReturnType, Params<T> params) { + Optional<RequestConfig> requestConfigOverride = getRequestConfigOverride(params); return tryAllConfigServers(configServer -> { HttpPost post = new HttpPost(configServer.resolve(path)); requestConfigOverride.ifPresent(post::setConfig); setContentTypeToApplicationJson(post); post.setEntity(new StringEntity(mapper.writeValueAsString(bodyJsonPojo))); return post; - }, wantedReturnType); + }, wantedReturnType, params); } @Override @@ -235,12 +253,12 @@ public class ConfigServerApiImpl implements ConfigServerApi { .build(); } - private static Optional<RequestConfig> getRequestConfigOverride(Params paramsOrNull) { - if (paramsOrNull == null) return Optional.empty(); + private static <T> Optional<RequestConfig> getRequestConfigOverride(Params<T> params) { + if (params.getConnectionTimeout().isEmpty()) return Optional.empty(); RequestConfig.Builder builder = RequestConfig.copy(DEFAULT_REQUEST_CONFIG); - paramsOrNull.getConnectionTimeout().ifPresent(connectionTimeout -> { + params.getConnectionTimeout().ifPresent(connectionTimeout -> { builder.setConnectTimeout((int) connectionTimeout.toMillis()); builder.setSocketTimeout((int) connectionTimeout.toMillis()); }); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java index a3cc7042c47..8b74dd35f96 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java @@ -6,11 +6,14 @@ import com.yahoo.vespa.hosted.node.admin.configserver.ConnectionException; import com.yahoo.vespa.hosted.node.admin.configserver.HttpException; import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult; +import com.yahoo.vespa.orchestrator.restapi.wire.HostStateChangeDenialReason; import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse; +import java.net.URI; import java.time.Duration; import java.util.List; import java.util.Optional; +import java.util.logging.Logger; /** * @author stiankri @@ -18,6 +21,8 @@ import java.util.Optional; * @author dybis */ public class OrchestratorImpl implements Orchestrator { + private static final Logger logger = Logger.getLogger(OrchestratorImpl.class.getName()); + // The server-side Orchestrator has an internal timeout of 10s. // // Note: A 409 has been observed to be returned after 33s in a case possibly involving @@ -44,7 +49,10 @@ public class OrchestratorImpl implements Orchestrator { public void suspend(final String hostName) { UpdateHostResponse response; try { - var params = new ConfigServerApi.Params().setConnectionTimeout(CONNECTION_TIMEOUT); + var params = new ConfigServerApi + .Params<UpdateHostResponse>() + .setConnectionTimeout(CONNECTION_TIMEOUT) + .setRetryPolicy(createRetryPolicyForSuspend()); response = configServerApi.put(getSuspendPath(hostName), Optional.empty(), UpdateHostResponse.class, params); } catch (HttpException.NotFoundException n) { throw new OrchestratorNotFoundException("Failed to suspend " + hostName + ", host not found"); @@ -61,11 +69,35 @@ public class OrchestratorImpl implements Orchestrator { }); } + private static ConfigServerApi.RetryPolicy<UpdateHostResponse> createRetryPolicyForSuspend() { + return new ConfigServerApi.RetryPolicy<UpdateHostResponse>() { + @Override + public boolean tryNextConfigServer(URI configServerEndpoint, UpdateHostResponse response) { + HostStateChangeDenialReason reason = response.reason(); + if (reason == null) { + return false; + } + + // The config server has likely just bootstrapped, so try the next. + if ("unknown-service-status".equals(reason.constraintName())) { + // Warn for now and until this feature has proven to work well + logger.warning("Config server at [" + configServerEndpoint + + "] failed with transient error (will try next): " + + reason.message()); + + return true; + } + + return false; + } + }; + } + @Override public void suspend(String parentHostName, List<String> hostNames) { final BatchOperationResult batchOperationResult; try { - var params = new ConfigServerApi.Params().setConnectionTimeout(CONNECTION_TIMEOUT); + var params = new ConfigServerApi.Params<BatchOperationResult>().setConnectionTimeout(CONNECTION_TIMEOUT); String hostnames = String.join("&hostname=", hostNames); String url = String.format("%s/%s?hostname=%s", ORCHESTRATOR_PATH_PREFIX_HOST_SUSPENSION_API, parentHostName, hostnames); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java index a11fdc903e7..bccf34e87ab 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImplTest.java @@ -116,7 +116,7 @@ public class ConfigServerApiImplTest { public void testBasicSuccessWithCustomTimeouts() { mockReturnCode = TIMEOUT_RETURN_CODE; - var params = new ConfigServerApi.Params(); + var params = new ConfigServerApi.Params<TestPojo>(); params.setConnectionTimeout(Duration.ofSeconds(3)); try { diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp index 18b9962003c..b19c9163254 100644 --- a/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp @@ -12,6 +12,7 @@ LOG_SETUP("attribute_usage_filter_test"); using proton::AttributeUsageFilter; using proton::AttributeUsageStats; using proton::IAttributeUsageListener; +using search::AddressSpaceComponents; using search::AddressSpaceUsage; using vespalib::AddressSpace; @@ -27,17 +28,15 @@ class MyAttributeStats : public AttributeUsageStats { public: void triggerEnumStoreLimit() { - merge({ enumStoreOverLoad, - search::AddressSpaceComponents::default_multi_value_usage() }, - "enumeratedName", - "ready"); + AddressSpaceUsage usage; + usage.set(AddressSpaceComponents::enum_store, enumStoreOverLoad); + merge(usage, "enumeratedName", "ready"); } void triggerMultiValueLimit() { - merge({ search::AddressSpaceComponents::default_enum_store_usage(), - multiValueOverLoad }, - "multiValueName", - "ready"); + AddressSpaceUsage usage; + usage.set(AddressSpaceComponents::multi_value, multiValueOverLoad); + merge(usage, "multiValueName", "ready"); } }; @@ -130,7 +129,8 @@ TEST_F("Check that multivalue limit can be reached", Fixture) TEST_F("listener is updated when attribute stats change", Fixture) { AttributeUsageStats stats; - AddressSpaceUsage usage(AddressSpace(12, 10, 15), AddressSpace(22, 20, 25)); + AddressSpaceUsage usage; + usage.set("my_comp", AddressSpace(12, 10, 15)); stats.merge(usage, "my_attr", "my_subdb"); f.setAttributeStats(stats); EXPECT_EQUAL(stats, f.listener->stats); diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index daaab2b823a..43743c23629 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -435,18 +435,6 @@ initialize.threads int default = 0 ## before put and update operations in feed is blocked. writefilter.attribute.address_space_limit double default = 0.9 -## Portion of enumstore address space that can be used before put and update -## portion of feed is blocked. -## Deprecated -> Use address_space_limit -## TODO: remove this when enum store is removed from AttributeUsageStats -writefilter.attribute.enumstorelimit double default = 0.9 - -## Portion of attribute multivalue mapping address space that can be used -## before put and update portion of feed is blocked. -## Deprecated -> Use address_space_limit -## TODO: remove this when multi value is removed from AttributeUsageStats -writefilter.attribute.multivaluelimit double default = 0.9 - ## Portion of physical memory that can be resident memory in anonymous mapping ## by the proton process before put and update portion of feed is blocked. writefilter.memorylimit double default = 0.8 diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp index f0ab56562a6..d89e273df27 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp @@ -9,9 +9,7 @@ using search::AddressSpaceComponents; namespace proton { AttributeUsageStats::AttributeUsageStats() - : _enumStoreUsage(AddressSpaceComponents::default_enum_store_usage()), - _multiValueUsage(AddressSpaceComponents::default_multi_value_usage()), - _max_usage(vespalib::AddressSpace()) + : _max_usage(vespalib::AddressSpace()) { } @@ -22,8 +20,6 @@ AttributeUsageStats::merge(const search::AddressSpaceUsage &usage, const vespalib::string &attributeName, const vespalib::string &subDbName) { - _enumStoreUsage.merge(usage.enum_store_usage(), attributeName, AddressSpaceComponents::enum_store, subDbName); - _multiValueUsage.merge(usage.multi_value_usage(), attributeName, AddressSpaceComponents::multi_value, subDbName); for (const auto& entry : usage.get_all()) { _max_usage.merge(entry.second, attributeName, entry.first, subDbName); } @@ -32,9 +28,7 @@ AttributeUsageStats::merge(const search::AddressSpaceUsage &usage, std::ostream& operator<<(std::ostream& out, const AttributeUsageStats& rhs) { - out << "{enum_store=" << rhs.enumStoreUsage() << - ", multi_value=" << rhs.multiValueUsage() << - ", max_address_space_usage=" << rhs.max_address_space_usage() << "}"; + out << "{max_address_space_usage=" << rhs.max_address_space_usage() << "}"; return out; } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.h index 762cc324f89..1411c626bfb 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.h @@ -8,14 +8,11 @@ namespace proton { /** - * Class representing aggregated attribute usage, with info about - * the most bloated attributes with regards to enum store and - * multivalue mapping. + * Class representing aggregated max address space usage + * among components in attributes vectors in all sub databases. */ class AttributeUsageStats { - AddressSpaceUsageStats _enumStoreUsage; - AddressSpaceUsageStats _multiValueUsage; AddressSpaceUsageStats _max_usage; public: @@ -25,14 +22,10 @@ public: const vespalib::string &attributeName, const vespalib::string &subDbName); - const AddressSpaceUsageStats& enumStoreUsage() const { return _enumStoreUsage; } - const AddressSpaceUsageStats& multiValueUsage() const { return _multiValueUsage; } const AddressSpaceUsageStats& max_address_space_usage() const { return _max_usage; } bool operator==(const AttributeUsageStats& rhs) const { - return (_enumStoreUsage == rhs._enumStoreUsage) && - (_multiValueUsage == rhs._multiValueUsage) && - (_max_usage == rhs._max_usage); + return (_max_usage == rhs._max_usage); } }; diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp index 7bd07dba505..d44f9ff6d2e 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp @@ -92,10 +92,6 @@ DocumentDBTaggedMetrics::AttributeMetrics::~AttributeMetrics() = default; DocumentDBTaggedMetrics::AttributeMetrics::ResourceUsageMetrics::ResourceUsageMetrics(MetricSet *parent) : MetricSet("resource_usage", {}, "Metrics for various attribute vector resources usage", parent), - enumStore("enum_store", {}, "The highest relative amount of enum store address space used among " - "all enumerated attribute vectors in this document db (value in the range [0, 1])", this), - multiValue("multi_value", {}, "The highest relative amount of multi-value address space used among " - "all multi-value attribute vectors in this document db (value in the range [0, 1])", this), address_space("address_space", {}, "The max relative address space used among " "components in all attribute vectors in this document db (value in the range [0, 1])", this), feedingBlocked("feeding_blocked", {}, "Whether feeding is blocked due to attribute resource limits being reached (value is either 0 or 1)", this) diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h index 8d225115c37..04e16cd5cb7 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h @@ -86,8 +86,6 @@ struct DocumentDBTaggedMetrics : metrics::MetricSet { struct ResourceUsageMetrics : metrics::MetricSet { - metrics::DoubleValueMetric enumStore; - metrics::DoubleValueMetric multiValue; metrics::DoubleValueMetric address_space; metrics::LongValueMetric feedingBlocked; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp index 956e9ea198e..753bd1cd148 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp @@ -304,11 +304,7 @@ DocumentDBMetricsUpdater::updateAttributeResourceUsageMetrics(DocumentDBTaggedMe { AttributeUsageStats stats = _writeFilter.getAttributeUsageStats(); bool feedBlocked = !_writeFilter.acceptWriteOperation(); - double enumStoreUsed = stats.enumStoreUsage().getUsage().usage(); - double multiValueUsed = stats.multiValueUsage().getUsage().usage(); double address_space_used = stats.max_address_space_usage().getUsage().usage(); - metrics.resourceUsage.enumStore.set(enumStoreUsed); - metrics.resourceUsage.multiValue.set(multiValueUsed); metrics.resourceUsage.address_space.set(address_space_used); metrics.resourceUsage.feedingBlocked.set(feedBlocked ? 1 : 0); } diff --git a/searchlib/src/vespa/searchlib/attribute/address_space_usage.cpp b/searchlib/src/vespa/searchlib/attribute/address_space_usage.cpp index da2e376719c..6783ea84354 100644 --- a/searchlib/src/vespa/searchlib/attribute/address_space_usage.cpp +++ b/searchlib/src/vespa/searchlib/attribute/address_space_usage.cpp @@ -12,15 +12,6 @@ AddressSpaceUsage::AddressSpaceUsage() { } -AddressSpaceUsage::AddressSpaceUsage(const AddressSpace& enum_store_usage, - const AddressSpace& multi_value_usage) - : _map() -{ - // TODO: Remove this constructor and instead add usage for each relevant component explicit. - set(AddressSpaceComponents::enum_store, enum_store_usage); - set(AddressSpaceComponents::multi_value, multi_value_usage); -} - void AddressSpaceUsage::set(const vespalib::string& component, const vespalib::AddressSpace& usage) { diff --git a/searchlib/src/vespa/searchlib/attribute/address_space_usage.h b/searchlib/src/vespa/searchlib/attribute/address_space_usage.h index 9a92bb5d858..3fe24e39a14 100644 --- a/searchlib/src/vespa/searchlib/attribute/address_space_usage.h +++ b/searchlib/src/vespa/searchlib/attribute/address_space_usage.h @@ -20,8 +20,6 @@ private: public: AddressSpaceUsage(); - AddressSpaceUsage(const vespalib::AddressSpace& enum_store_usage, - const vespalib::AddressSpace& multi_value_usage); void set(const vespalib::string& component, const vespalib::AddressSpace& usage); vespalib::AddressSpace get(const vespalib::string& component) const; const AddressSpaceMap& get_all() const { return _map; } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java index 54f2b2fd9e3..297852e9584 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java @@ -259,14 +259,19 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { } @Override - public void approvePendingRoleMembership(AthenzRole athenzRole, AthenzUser athenzUser, Instant expiry) { + public void approvePendingRoleMembership(AthenzRole athenzRole, AthenzUser athenzUser, Instant expiry, Optional<String> reason) { URI uri = zmsUrl.resolve(String.format("domain/%s/role/%s/member/%s/decision", athenzRole.domain().getName(), athenzRole.roleName(), athenzUser.getFullName())); MembershipEntity membership = new MembershipEntity.RoleMembershipEntity(athenzUser.getFullName(), true, athenzRole.roleName(), Long.toString(expiry.getEpochSecond())); - HttpUriRequest request = RequestBuilder.put() + + var requestBuilder = RequestBuilder.put() .setUri(uri) - .setEntity(toJsonStringEntity(membership)) - .build(); - execute(request, response -> readEntity(response, Void.class)); + .setEntity(toJsonStringEntity(membership)); + + if (reason.filter(s -> !s.isBlank()).isPresent()) { + requestBuilder.addHeader("Y-Audit-Ref", reason.get()); + } + + execute(requestBuilder.build(), response -> readEntity(response, Void.class)); } @Override diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java index 2fd1cea0e50..7dd0585bfd4 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java @@ -54,7 +54,7 @@ public interface ZmsClient extends AutoCloseable { Map<AthenzUser, String> listPendingRoleApprovals(AthenzRole athenzRole); - void approvePendingRoleMembership(AthenzRole athenzRole, AthenzUser athenzUser, Instant expiry); + void approvePendingRoleMembership(AthenzRole athenzRole, AthenzUser athenzUser, Instant expiry, Optional<String> reason); List<AthenzIdentity> listMembers(AthenzRole athenzRole); diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 199291ac276..e5d5b8ba5b6 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -658,6 +658,7 @@ "public static void logAndDie(java.lang.String, boolean)", "public static void logAndDie(java.lang.String, java.lang.Throwable)", "public static void logAndDie(java.lang.String, java.lang.Throwable, boolean)", + "public static void dumpHeap(java.lang.String, boolean)", "public static void dumpThreads()" ], "fields": [] diff --git a/vespajlib/src/main/java/com/yahoo/protect/Process.java b/vespajlib/src/main/java/com/yahoo/protect/Process.java index 4d2fafd4665..f3674f665b2 100644 --- a/vespajlib/src/main/java/com/yahoo/protect/Process.java +++ b/vespajlib/src/main/java/com/yahoo/protect/Process.java @@ -1,7 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.protect; +import com.sun.management.HotSpotDiagnosticMXBean; +import javax.management.MBeanServer; +import java.io.IOException; +import java.lang.management.ManagementFactory; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; @@ -70,6 +74,16 @@ public final class Process { } } + public static void dumpHeap(String filePath, boolean live) throws IOException { + log.log(Level.INFO, "Will dump the heap to '" + filePath + "', with the live = " + live); + getHotspotMXBean().dumpHeap(filePath, live); + } + + private static HotSpotDiagnosticMXBean getHotspotMXBean() throws IOException { + MBeanServer server = ManagementFactory.getPlatformMBeanServer(); + return ManagementFactory.newPlatformMXBeanProxy( + server, "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class); + } public static void dumpThreads() { boolean alreadyDumpingThreads = busyDumpingThreads.getAndSet(true); |