diff options
100 files changed, 1539 insertions, 2549 deletions
diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt index 9ab262cffb9..052745f6f2f 100644 --- a/client/CMakeLists.txt +++ b/client/CMakeLists.txt @@ -14,3 +14,6 @@ add_custom_target(vespalog_logfmt ALL DEPENDS ${GODIR}/bin/vespa-logfmt) install(PROGRAMS ${GODIR}/bin/script-utils DESTINATION libexec/vespa) install_symlink(libexec/vespa/script-utils bin/vespa-logfmt) install_symlink(libexec/vespa/script-utils bin/vespa-deploy) +install_symlink(libexec/vespa/script-utils bin/vespa-get-cluster-state) +install_symlink(libexec/vespa/script-utils bin/vespa-get-node-state) +install_symlink(libexec/vespa/script-utils bin/vespa-set-node-state) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 3c4eb1046bc..838fd9b6b26 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -133,6 +133,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"vekterli"}) default boolean useTwoPhaseDocumentGc() { return false; } @ModelFeatureFlag(owners = {"hmusum"}) default int clusterControllerStateGatherCount() { return 2; } @ModelFeatureFlag(owners = {"tokle"}) default boolean useRestrictedDataPlaneBindings() { return false; } + @ModelFeatureFlag(owners = {"baldersheim", "vekterli"}) default boolean computeCoverageFromTargetActiveDocs() { return false; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/pom.xml b/config-model/pom.xml index d1af9eab915..a11e72fa1be 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -329,6 +329,9 @@ <artifactId>maven-surefire-plugin</artifactId> <configuration> <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> + <environmentVariables> + <VESPA_HOME>/opt/vespa</VESPA_HOME> + </environmentVariables> </configuration> </plugin> <plugin> diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 033fc4cd8de..b7b54e749f9 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -90,6 +90,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private int mbus_network_threads = 1; private Architecture adminClusterNodeResourcesArchitecture = Architecture.getDefault(); private boolean useRestrictedDataPlaneBindings = false; + private boolean computeCoverageFromTargetActiveDocs = false; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -155,6 +156,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean useTwoPhaseDocumentGc() { return useTwoPhaseDocumentGc; } @Override public String phraseOptimization() { return phraseOptimization; } @Override public boolean useRestrictedDataPlaneBindings() { return useRestrictedDataPlaneBindings; } + @Override public boolean computeCoverageFromTargetActiveDocs() { return computeCoverageFromTargetActiveDocs; } public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim; @@ -432,6 +434,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setComputeCoverageFromTargetActiveDocs(boolean computeCoverageFromTargetActiveDocs) { + this.computeCoverageFromTargetActiveDocs = computeCoverageFromTargetActiveDocs; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java index b8aa89fc82a..b9355693da8 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java @@ -5,6 +5,7 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.DataType; import com.yahoo.prelude.fastsearch.DocsumDefinitionSet; import com.yahoo.schema.Schema; +import com.yahoo.schema.processing.DynamicSummaryTransformUtils; import com.yahoo.vespa.config.search.SummaryConfig; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; @@ -158,14 +159,10 @@ public class SummaryClass extends Derived { summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER) { return summaryField.getSingleSource(); + } else if (summaryField.getTransform().isDynamic()) { + return DynamicSummaryTransformUtils.getSource(summaryField); } else { - // Note: Currently source mapping is handled in the indexing statement, - // by creating a summary field for each of the values - // This works, but is suboptimal. We could consolidate to a minimal set and - // use the right value from the minimal set as the third parameter here, - // and add "override" commands to multiple static values - boolean useFieldNameAsArgument = summaryField.getTransform().isDynamic(); - return useFieldNameAsArgument ? summaryField.getName() : ""; + return ""; } } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java index 3afc25131c0..77a17878840 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java @@ -10,6 +10,7 @@ import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.schema.document.SDDocumentType; import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; /** @@ -33,15 +34,14 @@ public class AddExtraFieldsToDocument extends Processor { } for (var docsum : schema.getSummaries().values()) { for (var summaryField : docsum.getSummaryFields().values()) { - switch (summaryField.getTransform()) { - case NONE: - case BOLDED: - case DYNAMICBOLDED: - case DYNAMICTEASER: - case DOCUMENT_ID: // TODO: Adding the 'documentid' field should no longer be needed when the docsum framework in the backend has been simplified and the transform is always used. + var transform = summaryField.getTransform(); + if (transform.isDynamic() && DynamicSummaryTransformUtils.summaryFieldIsRequiredInDocumentType(summaryField) || + transform == SummaryTransform.NONE || + transform == SummaryTransform.DOCUMENT_ID) + { + // TODO: Adding the 'documentid' field should no longer be needed when the docsum framework in the backend has been simplified and the transform is always used. addSummaryField(schema, document, summaryField, validate); - break; - default: + } else { // skip: generated from attribute or similar, // so does not need to be included as an extra // field in the document type diff --git a/config-model/src/main/java/com/yahoo/schema/processing/Bolding.java b/config-model/src/main/java/com/yahoo/schema/processing/Bolding.java index 53a3d462d54..73ad4225c88 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/Bolding.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/Bolding.java @@ -3,7 +3,6 @@ package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.schema.RankProfileRegistry; -import com.yahoo.document.DataType; import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.schema.Schema; import com.yahoo.vespa.documentmodel.SummaryField; @@ -26,16 +25,12 @@ public class Bolding extends Processor { if ( ! validate) return; for (ImmutableSDField field : schema.allConcreteFields()) { for (SummaryField summary : field.getSummaryFields().values()) { - if (summary.getTransform().isBolded() && - !((summary.getDataType() == DataType.STRING) || (summary.getDataType() == DataType.URI))) - { + if (summary.getTransform().isBolded() && !DynamicSummaryTransformUtils.hasSupportedType(summary)) { throw new IllegalArgumentException("'bolding: on' for non-text field " + "'" + field.getName() + "'" + " (" + summary.getDataType() + ")" + " is not allowed"); - } else if (summary.getTransform().isDynamic() && - !((summary.getDataType() == DataType.STRING) || (summary.getDataType() == DataType.URI))) - { + } else if (summary.getTransform().isDynamic() && !DynamicSummaryTransformUtils.hasSupportedType(summary)) { throw new IllegalArgumentException("'summary: dynamic' for non-text field " + "'" + field.getName() + "'" + " (" + summary.getDataType() + ")" + diff --git a/config-model/src/main/java/com/yahoo/schema/processing/DynamicSummaryTransformUtils.java b/config-model/src/main/java/com/yahoo/schema/processing/DynamicSummaryTransformUtils.java new file mode 100644 index 00000000000..20597eca64f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/processing/DynamicSummaryTransformUtils.java @@ -0,0 +1,59 @@ +package com.yahoo.schema.processing; + +import com.yahoo.document.DataType; +import com.yahoo.vespa.documentmodel.SummaryField; + +/** + * This class contains utils used when handling summary fields with dynamic transforms during processing and deriving. + * + * Originally (before Vespa 8.52), dynamic transforms where only supported for string fields. + * Due to legacy functionality in the backend docsum framework, + * such summary fields are in some cases added as extra document fields and populated in indexing scripts. + * This is something we want to avoid in the future, but it might not be entirely possible before Vespa 9. + * + * With the introduction of dynamic transform for array of string fields, + * we move in the right direction and avoid adding extra document fields with indexing script population for this type. + * Instead, we configure the dynamic transform in the backend to use the original source field directly. + * + * See SummaryTransform.isDynamic() for which transforms this applies to. + */ +public class DynamicSummaryTransformUtils { + + public static boolean hasSupportedType(SummaryField field) { + return isSupportedType(field.getDataType()); + } + + public static boolean isSupportedType(DataType type) { + return isOriginalSupportedType(type) || isNewSupportedType(type); + } + + private static boolean isOriginalSupportedType(DataType type) { + return (type == DataType.STRING) || + (type == DataType.URI); + } + + private static boolean isNewSupportedType(DataType type) { + return (type.equals(DataType.getArray(DataType.STRING))); + } + + /** + * Whether a summary field must be populated by the source field with the given type in an indexing script. + */ + public static boolean summaryFieldIsPopulatedBySourceField(DataType sourceFieldType) { + return isOriginalSupportedType(sourceFieldType); + } + + /** + * Whether a summary field is required as an extra field in the document type. + */ + public static boolean summaryFieldIsRequiredInDocumentType(SummaryField summaryField) { + return summaryFieldIsPopulatedBySourceField(summaryField.getDataType()); + } + + public static String getSource(SummaryField summaryField) { + // Summary fields with the original supported type is always present in the document type, + // and we must use that field as source at run-time. + return isOriginalSupportedType(summaryField.getDataType()) ? + summaryField.getName() : summaryField.getSingleSource(); + } +} diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java index ea65a223686..2c24d3e53e1 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java @@ -68,11 +68,13 @@ public class IndexingOutputs extends Processor { } if (summaryTransform.isDynamic()) { DataType fieldType = field.getDataType(); - if (fieldType != DataType.URI && fieldType != DataType.STRING) { - warn(schema, field, "Dynamic summaries are only supported for fields of type " + - "string, ignoring summary field '" + summaryField.getName() + - "' for sd field '" + field.getName() + "' of type " + - fieldType.getName() + "."); + if (!DynamicSummaryTransformUtils.summaryFieldIsPopulatedBySourceField(fieldType)) { + if (!DynamicSummaryTransformUtils.isSupportedType(fieldType)) { + warn(schema, field, "Dynamic summaries are only supported for fields of type " + + "string and array<string>, ignoring summary field '" + summaryField.getName() + + "' for sd field '" + field.getName() + "' of type " + + fieldType.getName() + "."); + } return; } dynamicSummary.add(summaryName); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/SummaryDynamicStructsArrays.java b/config-model/src/main/java/com/yahoo/schema/processing/SummaryDynamicStructsArrays.java index ed1f47611eb..a899f5e82ab 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/SummaryDynamicStructsArrays.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/SummaryDynamicStructsArrays.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; /** * Fail if: - * An SD field explicitly says summary:dynamic , but the field is wset, array or struct. + * An SD field explicitly says summary:dynamic , but the field is non-string array, wset, or struct. * If there is an explicitly defined summary class, saying dynamic in one of its summary * fields is always legal. * @@ -31,7 +31,7 @@ public class SummaryDynamicStructsArrays extends Processor { for (SDField field : schema.allConcreteFields()) { DataType type = field.getDataType(); - if (type instanceof ArrayDataType || type instanceof WeightedSetDataType || type instanceof StructDataType) { + if (isNonStringArray(type) || type instanceof WeightedSetDataType || type instanceof StructDataType) { for (SummaryField sField : field.getSummaryFields().values()) { if (sField.getTransform().equals(SummaryTransform.DYNAMICTEASER)) { throw new IllegalArgumentException("For field '"+field.getName()+"': dynamic summary is illegal " + @@ -44,4 +44,8 @@ public class SummaryDynamicStructsArrays extends Processor { } } + private boolean isNonStringArray(DataType type) { + return (type instanceof ArrayDataType) && (!type.equals(DataType.getArray(DataType.STRING))); + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java index 718f1646126..a57a8fa9e70 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/UriBindingsValidator.java @@ -41,7 +41,7 @@ class UriBindingsValidator extends Validator { private static void validateUserBinding(BindingPattern binding, VespaModel model, DeployState deployState) { validateScheme(binding, deployState); if (isHostedApplication(model, deployState)) { - validateHostedApplicationUserBinding(binding); + validateHostedApplicationUserBinding(binding, deployState); } } @@ -53,13 +53,14 @@ class UriBindingsValidator extends Validator { } } - private static void validateHostedApplicationUserBinding(BindingPattern binding) { + private static void validateHostedApplicationUserBinding(BindingPattern binding, DeployState deployState) { // only perform these validation for used-generated bindings // bindings produced by the hosted config model amender will violate some of the rules below if (binding instanceof SystemBindingPattern) return; - if (!binding.matchesAnyPort()) { - throw new IllegalArgumentException(createErrorMessage(binding, "binding with port is not allowed")); + // Allow binding to port if we are restricting data plane bindings + if (!binding.matchesAnyPort() && !deployState.featureFlags().useRestrictedDataPlaneBindings()) { + throw new IllegalArgumentException(createErrorMessage(binding, "binding with port is not allowed")); } if (!binding.host().equals(BindingPattern.WILDCARD_PATTERN)) { throw new IllegalArgumentException(createErrorMessage(binding, "only binding with wildcard ('*') for hostname is allowed")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java index 7bfe971981e..dde38544924 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java @@ -14,6 +14,7 @@ import com.yahoo.vespa.model.container.component.UserBindingPattern; import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder; import org.w3c.dom.Element; +import java.util.OptionalInt; import java.util.Set; import static com.yahoo.vespa.model.container.ApplicationContainerCluster.METRICS_V2_HANDLER_BINDING_1; @@ -36,23 +37,38 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder< VIP_HANDLER_BINDING); private final ApplicationContainerCluster cluster; + private OptionalInt portBindingOverride; public DomHandlerBuilder(ApplicationContainerCluster cluster) { + this(cluster, OptionalInt.empty()); + } + public DomHandlerBuilder(ApplicationContainerCluster cluster, OptionalInt portBindingOverride) { this.cluster = cluster; + this.portBindingOverride = portBindingOverride; } @Override protected Handler doBuild(DeployState deployState, AbstractConfigProducer<?> parent, Element handlerElement) { Handler handler = createHandler(handlerElement); + OptionalInt port = portBindingOverride.isPresent() && deployState.isHosted() && deployState.featureFlags().useRestrictedDataPlaneBindings() + ? portBindingOverride + : OptionalInt.empty(); for (Element binding : XML.getChildren(handlerElement, "binding")) - addServerBinding(handler, UserBindingPattern.fromPattern(XML.getValue(binding)), deployState.getDeployLogger()); + addServerBinding(handler, userBindingPattern(XML.getValue(binding), port), deployState.getDeployLogger()); DomComponentBuilder.addChildren(deployState, parent, handlerElement, handler); return handler; } + private static UserBindingPattern userBindingPattern(String path, OptionalInt port) { + UserBindingPattern bindingPattern = UserBindingPattern.fromPattern(path); + return port.isPresent() + ? bindingPattern.withPort(port.getAsInt()) + : bindingPattern; + } + Handler createHandler(Element handlerElement) { BundleInstantiationSpecification bundleSpec = BundleInstantiationSpecificationBuilder.build(handlerElement); return new Handler(new ComponentModel(bundleSpec)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java index f23f27c0d8e..8163c268d09 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java @@ -6,6 +6,7 @@ import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerThreadpool; import com.yahoo.vespa.model.container.PlatformBundles; +import com.yahoo.vespa.model.container.component.BindingPattern; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.component.UserBindingPattern; @@ -13,6 +14,7 @@ import com.yahoo.vespa.model.container.component.UserBindingPattern; import java.nio.file.Path; import java.util.Collection; import java.util.Collections; +import java.util.OptionalInt; /** * @author Einar M R Rosenvinge @@ -26,10 +28,10 @@ public class ContainerDocumentApi { private final boolean ignoreUndefinedFields; - public ContainerDocumentApi(ContainerCluster<?> cluster, HandlerOptions handlerOptions, boolean ignoreUndefinedFields) { + public ContainerDocumentApi(ContainerCluster<?> cluster, HandlerOptions handlerOptions, boolean ignoreUndefinedFields, OptionalInt portOverride) { this.ignoreUndefinedFields = ignoreUndefinedFields; - addRestApiHandler(cluster, handlerOptions); - addFeedHandler(cluster, handlerOptions); + addRestApiHandler(cluster, handlerOptions, portOverride); + addFeedHandler(cluster, handlerOptions, portOverride); addVespaClientContainerBundle(cluster); } @@ -37,18 +39,18 @@ public class ContainerDocumentApi { c.addPlatformBundle(VESPACLIENT_CONTAINER_BUNDLE); } - private static void addFeedHandler(ContainerCluster<?> cluster, HandlerOptions handlerOptions) { + private static void addFeedHandler(ContainerCluster<?> cluster, HandlerOptions handlerOptions, OptionalInt portOverride) { String bindingSuffix = ContainerCluster.RESERVED_URI_PREFIX + "/feedapi"; var executor = new Threadpool("feedapi-handler", handlerOptions.feedApiThreadpoolOptions); var handler = newVespaClientHandler("com.yahoo.vespa.http.server.FeedHandler", - bindingSuffix, handlerOptions, executor); + bindingSuffix, handlerOptions, executor, portOverride); cluster.addComponent(handler); } - private static void addRestApiHandler(ContainerCluster<?> cluster, HandlerOptions handlerOptions) { + private static void addRestApiHandler(ContainerCluster<?> cluster, HandlerOptions handlerOptions, OptionalInt portOverride) { var handler = newVespaClientHandler("com.yahoo.document.restapi.resource.DocumentV1ApiHandler", - DOCUMENT_V1_PREFIX + "/*", handlerOptions, null); + DOCUMENT_V1_PREFIX + "/*", handlerOptions, null, portOverride); cluster.addComponent(handler); // We need to include a dummy implementation of the previous restapi handler (using the same class name). @@ -62,23 +64,37 @@ public class ContainerDocumentApi { private static Handler newVespaClientHandler(String componentId, String bindingSuffix, HandlerOptions handlerOptions, - Threadpool executor) { + Threadpool executor, + OptionalInt portOverride) { Handler handler = createHandler(componentId, executor); if (handlerOptions.bindings.isEmpty()) { handler.addServerBindings( - SystemBindingPattern.fromHttpPath(bindingSuffix), - SystemBindingPattern.fromHttpPath(bindingSuffix + '/')); + bindingPattern(bindingSuffix, portOverride), + bindingPattern(bindingSuffix + '/', portOverride)); } else { for (String rootBinding : handlerOptions.bindings) { String pathWithoutLeadingSlash = bindingSuffix.substring(1); handler.addServerBindings( - UserBindingPattern.fromPattern(rootBinding + pathWithoutLeadingSlash), - UserBindingPattern.fromPattern(rootBinding + pathWithoutLeadingSlash + '/')); + userBindingPattern(rootBinding + pathWithoutLeadingSlash, portOverride), + userBindingPattern(rootBinding + pathWithoutLeadingSlash + '/', portOverride)); } } return handler; } + private static BindingPattern bindingPattern(String path, OptionalInt port) { + return port.isPresent() + ? SystemBindingPattern.fromHttpPortAndPath(Integer.toString(port.getAsInt()), path) + : SystemBindingPattern.fromHttpPath(path); + } + + private static UserBindingPattern userBindingPattern(String path, OptionalInt port) { + UserBindingPattern bindingPattern = UserBindingPattern.fromPattern(path); + return port.isPresent() + ? bindingPattern.withPort(port.getAsInt()) + : bindingPattern; + } + private static Handler createHandler(String className, Threadpool executor) { return new Handler(new ComponentModel(className, null, "vespaclient-container-plugin"), executor); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/UserBindingPattern.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/UserBindingPattern.java index 5da11c06fb1..182eca835c1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/UserBindingPattern.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/UserBindingPattern.java @@ -13,6 +13,7 @@ public class UserBindingPattern extends BindingPattern { public static UserBindingPattern fromHttpPath(String path) { return new UserBindingPattern("http", "*", null, path); } public static UserBindingPattern fromPattern(String binding) { return new UserBindingPattern(binding); } + public UserBindingPattern withPort(int port) { return new UserBindingPattern(scheme(), host(), Integer.toString(port), path()); } @Override public String toString() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index f55fb547bb0..cb52f701da4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -32,10 +32,10 @@ public class HostedSslConnectorFactory extends ConnectorFactory { */ public static HostedSslConnectorFactory withProvidedCertificate( String serverName, EndpointCertificateSecrets endpointCertificateSecrets, boolean enforceHandshakeClientAuth, - Collection<String> tlsCiphersOverride, boolean enableProxyProtocolMixedMode) { + Collection<String> tlsCiphersOverride, boolean enableProxyProtocolMixedMode, int port) { ConfiguredDirectSslProvider sslProvider = createConfiguredDirectSslProvider( serverName, endpointCertificateSecrets, DEFAULT_HOSTED_TRUSTSTORE, /*tlsCaCertificates*/null, enforceHandshakeClientAuth); - return new HostedSslConnectorFactory(sslProvider, false, enforceHandshakeClientAuth, tlsCiphersOverride, enableProxyProtocolMixedMode); + return new HostedSslConnectorFactory(sslProvider, false, enforceHandshakeClientAuth, tlsCiphersOverride, enableProxyProtocolMixedMode, port); } /** @@ -43,24 +43,24 @@ public class HostedSslConnectorFactory extends ConnectorFactory { */ public static HostedSslConnectorFactory withProvidedCertificateAndTruststore( String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates, - Collection<String> tlsCiphersOverride, boolean enableProxyProtocolMixedMode) { + Collection<String> tlsCiphersOverride, boolean enableProxyProtocolMixedMode, int port) { ConfiguredDirectSslProvider sslProvider = createConfiguredDirectSslProvider( serverName, endpointCertificateSecrets, /*tlsCaCertificatesPath*/null, tlsCaCertificates, false); - return new HostedSslConnectorFactory(sslProvider, true, false, tlsCiphersOverride, enableProxyProtocolMixedMode); + return new HostedSslConnectorFactory(sslProvider, true, false, tlsCiphersOverride, enableProxyProtocolMixedMode, port); } /** * Create connector factory that uses the default certificate and truststore provided by Vespa (through Vespa-global TLS configuration). */ public static HostedSslConnectorFactory withDefaultCertificateAndTruststore(String serverName, Collection<String> tlsCiphersOverride, - boolean enableProxyProtocolMixedMode) { - return new HostedSslConnectorFactory(new DefaultSslProvider(serverName), true, false, tlsCiphersOverride, enableProxyProtocolMixedMode); + boolean enableProxyProtocolMixedMode, int port) { + return new HostedSslConnectorFactory(new DefaultSslProvider(serverName), true, false, tlsCiphersOverride, enableProxyProtocolMixedMode, port); } private HostedSslConnectorFactory(SslProvider sslProvider, boolean enforceClientAuth, boolean enforceHandshakeClientAuth, Collection<String> tlsCiphersOverride, - boolean enableProxyProtocolMixedMode) { - super(new Builder("tls4443", 4443).sslProvider(sslProvider)); + boolean enableProxyProtocolMixedMode, int port) { + super(new Builder("tls"+port, port).sslProvider(sslProvider)); this.enforceClientAuth = enforceClientAuth; this.enforceHandshakeClientAuth = enforceHandshakeClientAuth; this.tlsCiphersOverride = tlsCiphersOverride; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 0bf586a089f..bb45c509b5b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -97,6 +97,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalInt; import java.util.Set; import java.util.function.Consumer; import java.util.logging.Level; @@ -114,6 +115,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // Default path to vip status file for container in Hosted Vespa. static final String HOSTED_VESPA_STATUS_FILE = Defaults.getDefaults().underVespaHome("var/vespa/load-balancer/status.html"); + // Data plane port for hosted Vespa + static final int HOSTED_VESPA_DATAPLANE_PORT = 4443; + //Path to vip status file for container in Hosted Vespa. Only used if set, else use HOSTED_VESPA_STATUS_FILE private static final String HOSTED_VESPA_STATUS_FILE_SETTING = "VESPA_LB_STATUS_FILE"; @@ -190,7 +194,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addProcessing(deployState, spec, cluster); addSearch(deployState, spec, cluster); addDocproc(deployState, spec, cluster); - addDocumentApi(spec, cluster); // NOTE: Must be done after addSearch + addDocumentApi(deployState, spec, cluster); // NOTE: Must be done after addSearch cluster.addDefaultHandlersExceptStatus(); addStatusHandlers(cluster, context.getDeployState().isHosted()); @@ -450,11 +454,11 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { connectorFactory = authorizeClient ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore( - serverName, endpointCertificateSecrets, getTlsClientAuthorities(deployState), tlsCiphersOverride, proxyProtocolMixedMode) + serverName, endpointCertificateSecrets, getTlsClientAuthorities(deployState), tlsCiphersOverride, proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT) : HostedSslConnectorFactory.withProvidedCertificate( - serverName, endpointCertificateSecrets, enforceHandshakeClientAuth, tlsCiphersOverride, proxyProtocolMixedMode); + serverName, endpointCertificateSecrets, enforceHandshakeClientAuth, tlsCiphersOverride, proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT); } else { - connectorFactory = HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName, tlsCiphersOverride, proxyProtocolMixedMode); + connectorFactory = HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName, tlsCiphersOverride, proxyProtocolMixedMode, HOSTED_VESPA_DATAPLANE_PORT); } cluster.getHttp().getAccessControl().ifPresent(accessControl -> accessControl.configureHostedConnector(connectorFactory)); server.addConnector(connectorFactory); @@ -516,8 +520,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return http; } - private void addDocumentApi(Element spec, ApplicationContainerCluster cluster) { - ContainerDocumentApi containerDocumentApi = buildDocumentApi(cluster, spec); + private void addDocumentApi(DeployState deployState, Element spec, ApplicationContainerCluster cluster) { + ContainerDocumentApi containerDocumentApi = buildDocumentApi(deployState, cluster, spec); if (containerDocumentApi == null) return; cluster.setDocumentApi(containerDocumentApi); @@ -540,7 +544,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addIncludes(searchElement); cluster.setSearch(buildSearch(deployState, cluster, searchElement)); - addSearchHandler(cluster, searchElement); + addSearchHandler(deployState, cluster, searchElement); validateAndAddConfiguredComponents(deployState, cluster, searchElement, "renderer", ContainerModelBuilder::validateRendererElement); } @@ -591,7 +595,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.addSearchAndDocprocBundles(); addIncludes(processingElement); cluster.setProcessingChains(new DomProcessingBuilder(null).build(deployState, cluster, processingElement), - serverBindings(processingElement, ProcessingChains.defaultBindings).toArray(BindingPattern[]::new)); + serverBindings(deployState, processingElement, ProcessingChains.defaultBindings).toArray(BindingPattern[]::new)); validateAndAddConfiguredComponents(deployState, cluster, processingElement, "renderer", ContainerModelBuilder::validateRendererElement); } @@ -616,7 +620,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addUserHandlers(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { for (Element component: XML.getChildren(spec, "handler")) { cluster.addComponent( - new DomHandlerBuilder(cluster).build(deployState, cluster, component)); + new DomHandlerBuilder(cluster, OptionalInt.of(HOSTED_VESPA_DATAPLANE_PORT)).build(deployState, cluster, component)); } } @@ -875,9 +879,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { container.setPreLoad(nodesElement.getAttribute(VespaDomBuilder.PRELOAD_ATTRIB_NAME)); } - private void addSearchHandler(ApplicationContainerCluster cluster, Element searchElement) { + private void addSearchHandler(DeployState deployState, ApplicationContainerCluster cluster, Element searchElement) { + BindingPattern bindingPattern = SearchHandler.DEFAULT_BINDING; + if (deployState.isHosted() && deployState.featureFlags().useRestrictedDataPlaneBindings()) { + bindingPattern = SearchHandler.bindingPattern(Optional.of(Integer.toString(HOSTED_VESPA_DATAPLANE_PORT))); + } SearchHandler searchHandler = new SearchHandler(cluster, - serverBindings(searchElement, SearchHandler.DEFAULT_BINDING), + serverBindings(deployState, searchElement, bindingPattern), ContainerThreadpool.UserOptions.fromXml(searchElement).orElse(null)); cluster.addComponent(searchHandler); @@ -885,34 +893,43 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { searchHandler.addComponent(Component.fromClassAndBundle(SearchHandler.EXECUTION_FACTORY_CLASS, PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE)); } - private List<BindingPattern> serverBindings(Element searchElement, BindingPattern... defaultBindings) { + private List<BindingPattern> serverBindings(DeployState deployState, Element searchElement, BindingPattern... defaultBindings) { List<Element> bindings = XML.getChildren(searchElement, "binding"); if (bindings.isEmpty()) return List.of(defaultBindings); - return toBindingList(bindings); + return toBindingList(deployState, bindings); } - private List<BindingPattern> toBindingList(List<Element> bindingElements) { + private List<BindingPattern> toBindingList(DeployState deployState, List<Element> bindingElements) { List<BindingPattern> result = new ArrayList<>(); - + OptionalInt port = deployState.isHosted() && deployState.featureFlags().useRestrictedDataPlaneBindings() ? OptionalInt.of(HOSTED_VESPA_DATAPLANE_PORT) : OptionalInt.empty(); for (Element element: bindingElements) { String text = element.getTextContent().trim(); if (!text.isEmpty()) - result.add(UserBindingPattern.fromPattern(text)); + result.add(userBindingPattern(text, port)); } return result; } + private static UserBindingPattern userBindingPattern(String path, OptionalInt port) { + UserBindingPattern bindingPattern = UserBindingPattern.fromPattern(path); + return port.isPresent() + ? bindingPattern.withPort(port.getAsInt()) + : bindingPattern; + } - private ContainerDocumentApi buildDocumentApi(ApplicationContainerCluster cluster, Element spec) { + private ContainerDocumentApi buildDocumentApi(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { Element documentApiElement = XML.getChild(spec, "document-api"); if (documentApiElement == null) return null; ContainerDocumentApi.HandlerOptions documentApiOptions = DocumentApiOptionsBuilder.build(documentApiElement); Element ignoreUndefinedFields = XML.getChild(documentApiElement, "ignore-undefined-fields"); + OptionalInt portBindingOverride = deployState.featureFlags().useRestrictedDataPlaneBindings() && deployState.isHosted() + ? OptionalInt.of(HOSTED_VESPA_DATAPLANE_PORT) + : OptionalInt.empty(); return new ContainerDocumentApi(cluster, documentApiOptions, - "true".equals(XML.getValue(ignoreUndefinedFields))); + "true".equals(XML.getValue(ignoreUndefinedFields)), portBindingOverride); } private ContainerDocproc buildDocproc(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java index 54cd061d2c5..596375ea93a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/SearchHandler.java @@ -11,6 +11,8 @@ import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; import java.util.List; +import java.util.Optional; +import java.util.OptionalInt; import static com.yahoo.container.bundle.BundleInstantiationSpecification.fromSearchAndDocproc; @@ -25,7 +27,7 @@ class SearchHandler extends ProcessingHandler<SearchChains> { static final String EXECUTION_FACTORY_CLASS = com.yahoo.search.searchchain.ExecutionFactory.class.getName(); static final BundleInstantiationSpecification HANDLER_SPEC = fromSearchAndDocproc(HANDLER_CLASS); - static final BindingPattern DEFAULT_BINDING = SystemBindingPattern.fromHttpPath("/search/*"); + static final BindingPattern DEFAULT_BINDING = bindingPattern(Optional.empty()); SearchHandler(ApplicationContainerCluster cluster, List<BindingPattern> bindings, @@ -34,6 +36,13 @@ class SearchHandler extends ProcessingHandler<SearchChains> { bindings.forEach(this::addServerBindings); } + static BindingPattern bindingPattern(Optional<String> port) { + String path = "/search/*"; + return port + .filter(s -> !s.isBlank()) + .map(s -> SystemBindingPattern.fromHttpPortAndPath(s, path)) + .orElseGet(() -> SystemBindingPattern.fromHttpPath(path)); + } private static class Threadpool extends ContainerThreadpool { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index a74cb040631..daade2d74d9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -322,8 +322,12 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } public void handleRedundancy(Redundancy redundancy) { - if (hasIndexedCluster()) + if (hasIndexedCluster()) { + // Important: these must all be the normalized "within a single leaf group" values, + // _not_ the cluster-wide, cross-group values. indexedCluster.setSearchableCopies(redundancy.readyCopies()); + indexedCluster.setRedundancy(redundancy.finalRedundancy()); + } this.redundancy = redundancy; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java index d9eddc1a548..643a305f369 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java @@ -54,6 +54,8 @@ public class DispatchGroup { public int getSearchableCopies() { return sc.getSearchableCopies(); } + public int getRedundancy() { return sc.getRedundancy(); } + static class Iterator implements java.util.Iterator<SearchInterface> { private java.util.Iterator<Map<Integer, SearchInterface>> it1; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 56fb915797b..94eb13dd967 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -54,11 +54,13 @@ public class IndexedSearchCluster extends SearchCluster private final MultipleDocumentDatabasesConfigProducer documentDbsConfigProducer; private int searchableCopies = 1; + private int redundancy = 1; private final DispatchGroup rootDispatch; private DispatchSpec dispatchSpec; private final List<SearchNode> searchNodes = new ArrayList<>(); private final DispatchTuning.DispatchPolicy defaultDispatchPolicy; + private final boolean computeCoverageFromTargetActiveDocs; /** * Returns the document selector that is able to resolve what documents are to be routed to this search cluster. @@ -75,6 +77,7 @@ public class IndexedSearchCluster extends SearchCluster documentDbsConfigProducer = new MultipleDocumentDatabasesConfigProducer(this, documentDbs); rootDispatch = new DispatchGroup(this); defaultDispatchPolicy = DispatchTuning.Builder.toDispatchPolicy(featureFlags.queryDispatchPolicy()); + computeCoverageFromTargetActiveDocs = featureFlags.computeCoverageFromTargetActiveDocs(); } @Override @@ -263,6 +266,14 @@ public class IndexedSearchCluster extends SearchCluster this.searchableCopies = searchableCopies; } + public int getRedundancy() { + return redundancy; + } + + public void setRedundancy(int redundancy) { + this.redundancy = redundancy; + } + public void setDispatchSpec(DispatchSpec dispatchSpec) { if (dispatchSpec.getNumDispatchGroups() != null) { this.dispatchSpec = new DispatchSpec.Builder().setGroups @@ -310,6 +321,8 @@ public class IndexedSearchCluster extends SearchCluster builder.maxHitsPerNode(tuning.dispatch.getMaxHitsPerPartition()); builder.searchableCopies(rootDispatch.getSearchableCopies()); + builder.redundancy(rootDispatch.getRedundancy()); + builder.computeCoverageFromTargetActiveDocs(computeCoverageFromTargetActiveDocs); if (searchCoverage != null) { if (searchCoverage.getMinimum() != null) builder.minSearchCoverage(searchCoverage.getMinimum() * 100.0); diff --git a/config-model/src/main/perl/vespa-deploy b/config-model/src/main/perl/vespa-deploy deleted file mode 100755 index dde074b5e8c..00000000000 --- a/config-model/src/main/perl/vespa-deploy +++ /dev/null @@ -1,654 +0,0 @@ -#!/usr/bin/env perl -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# This script is for uploading, preparing, activating and fetching -# application packages to a cloud config server - -# BEGIN perl environment bootstrap section -# Do not edit between here and END as this section should stay identical in all scripts - -use File::Basename; -use File::Path; - -sub findpath { - my $myfullname = ${0}; - my($myname, $mypath) = fileparse($myfullname); - - return $mypath if ( $mypath && -d $mypath ); - $mypath=`pwd`; - - my $pwdfullname = $mypath . "/" . $myname; - return $mypath if ( -f $pwdfullname ); - return 0; -} - -# Returns the argument path if it seems to point to VESPA_HOME, 0 otherwise -sub is_vespa_home { - my($VESPA_HOME) = shift; - my $COMMON_ENV="libexec/vespa/common-env.sh"; - if ( $VESPA_HOME && -d $VESPA_HOME ) { - my $common_env = $VESPA_HOME . "/" . $COMMON_ENV; - return $VESPA_HOME if -f $common_env; - } - return 0; -} - -# Returns the home of Vespa, or dies if it cannot -sub findhome { - # Try the VESPA_HOME env variable - return $ENV{'VESPA_HOME'} if is_vespa_home($ENV{'VESPA_HOME'}); - if ( $ENV{'VESPA_HOME'} ) { # was set, but not correctly - die "FATAL: bad VESPA_HOME value '" . $ENV{'VESPA_HOME'} . "'\n"; - } - - # Try the ROOT env variable - my $ROOT = $ENV{'ROOT'}; - return $ROOT if is_vespa_home($ROOT); - - # Try the script location or current dir - my $mypath = findpath(); - if ($mypath) { - while ( $mypath =~ s|/[^/]*$|| ) { - return $mypath if is_vespa_home($mypath); - } - } - die "FATAL: Missing VESPA_HOME environment variable\n"; -} - -sub findhost { - my $tmp = $ENV{'VESPA_HOSTNAME'}; - my $bin = $ENV{'VESPA_HOME'} . "/bin"; - if (!defined $tmp) { - $tmp = `${bin}/vespa-detect-hostname || hostname -f || hostname || echo "localhost"`; - chomp $tmp; - } - my $validate = "${bin}/vespa-validate-hostname"; - if (-f "${validate}") { - system("${validate} $tmp"); - ( $? == 0 ) or die "Could not validate hostname\n"; - } - return $tmp; -} - -BEGIN { - my $tmp = findhome(); - $ENV{'VESPA_HOME'} = $tmp; - $tmp = findhost(); - $ENV{'VESPA_HOSTNAME'} = $tmp; -} -my $VESPA_HOME = $ENV{'VESPA_HOME'}; - -use lib $ENV{'VESPA_HOME'} . "/lib/perl5/site_perl"; - -# END perl environment bootstrap section - -use Yahoo::Vespa::Defaults; -readConfFile(); - -use strict; -use warnings; -use feature qw(switch say); -use vars qw/ $opt_c $opt_h $opt_n $opt_v $opt_f $opt_t $opt_a $opt_e $opt_E $opt_r $opt_i $opt_p $opt_H $opt_R $opt_F $opt_V /; -use Env qw($HOME); -use JSON; -use Getopt::Std; -use File::Path qw(make_path); -use Scalar::Util qw(looks_like_number); - -my $cloudconfig_dir = "$HOME/.cloudconfig"; -my $session_id_file; -my $configsource_url_used_file = "$cloudconfig_dir/deploy-configsource-url-used"; - - - -my $pathPrefix; -my $tenant = "default"; -my $application = "default"; -my $environment = "prod"; -my $region = "default"; -my $instance = "default"; -my $version = "v2"; -my $configserver = ""; -my $port = "19071"; -getopts('c:fhnt:ve:E:r:a:i:p:HR:F:V:'); - -if ($opt_h) { - usage(); - exit 0; -} - -if ($opt_c) { - $configserver = $opt_c; -} - -if ($opt_e) { - $tenant = $opt_e; -} - -if ($opt_r) { - $region = $opt_r; -} - -if ($opt_E) { - $environment = $opt_E; -} - -if ($opt_a) { - $application = $opt_a; -} - -if ($opt_i) { - $instance = $opt_i; -} - -if ($opt_p) { - $port = $opt_p; -} - -$pathPrefix = "/application/v2/tenant/$tenant/session"; - -create_cloudconfig_dir(); - -$session_id_file = "$cloudconfig_dir/$tenant/deploy-session-id"; - -my $command = shift; -$command ||= "help"; - -# The '--insecure' parameter is sadly required as it is not possible to disable or alter hostname verification with curl -my $curl_command = $VESPA_HOME . '/libexec/vespa/vespa-curl-wrapper -A vespa-deploy --silent --show-error --connect-timeout 30 --max-time 1200'; - -my $CURL_PUT = $curl_command . ' --write-out \%{http_code} --request PUT'; -my $CURL_GET = $curl_command . ' --request GET'; -my $GZIP = "gzip"; -my $CURL_POST_WITH_HEADERS = $curl_command . ' -i --request POST --header "Content-Type: application/x-gzip" --data-binary @- -D /tmp/http-headers'; -my $CURL_POST = $curl_command . ' --write-out \%{http_code} --request POST --header "Content-Type: application/x-gzip" --data-binary @-'; -my $CURL_POST_ZIP = $curl_command . ' --write-out \%{http_code} --request POST --header "Content-Type: application/zip" --data-binary @-'; - -if ($command eq "upload") { - my $application_package = shift; - if (!$opt_F) { - if (!$application_package) { - print "Command failed. No application package specified\n"; - usage("upload"); - exit 1; - } - if (!(-e $application_package)) { - print "Command failed. No such directory found: '$application_package'\n"; - exit 1; - } - check_application_directory($application_package); - } - - do_http_request("upload", $application_package); -} elsif ($command eq "prepare") { - my $arg = shift; - if ($arg && looks_like_number($arg) && !(-d $arg)) { - do_http_request("prepare", "", $arg); - } elsif ($arg) { - check_application_directory($arg); - do_http_request("upload", $arg); - do_http_request("prepare"); - } else { - do_http_request("prepare"); - } -} elsif ($command eq "activate") { - my $session_id = shift; - do_http_request("activate", "", $session_id); -} elsif ($command eq "fetch") { - my $arg = shift; - if ($arg) { - fetch_active_application($arg); - } else { - usage("fetch", $arg); - } -} elsif ($command eq "help") { - my $arg = shift; - usage($command, $arg); -} else { - usage($command); -} - - -sub check_application_directory { - my ($application_package) = shift; - - if (-d $application_package) { - # OK - } elsif ((-f $application_package) && ($application_package =~ /.*\.zip/ )) { - # OK - } else { - print "Command failed. No directory or zip file found: '$application_package'\n"; - exit 1; - } -} - -sub usage { - my ($command, $arg) = @_; - - if ($command && $command eq "help") { - $command = $arg; - } - $command ||= "help"; - - if ($command eq "upload") { - usage_upload(); - } elsif ($command eq "prepare") { - usage_prepare(); - } elsif ($command eq "activate") { - usage_activate(); - } elsif ($command eq "fetch") { - usage_fetch(); - } else { - print "Usage: vespa-deploy [-h] [-v] [-f] [-t] [-c] [-p] [-z] [-V] [<command>] [args]\n"; - print "Supported commands: 'upload', 'prepare', 'activate', 'fetch' and 'help'\n"; - print "Supported options: '-h' (help), '-v' (verbose), '-f' (force/ignore validation errors), '-t' (timeout in seconds), '-p' (config server http port)\n"; - print " '-h' (help)\n"; - print " '-v' (verbose)\n"; - print " '-n' (dry-run)\n"; - print " '-f' (force/ignore validation errors)\n"; - print " '-t <timeout>' (timeout in seconds)\n"; - print " '-c <server>' (config server hostname)\n"; - print " '-p <port>' (config server http port)\n"; - - print "Try 'vespa-deploy help <command>' to get more help\n"; - } -} - -sub usage_upload { - print "Usage: vespa-deploy upload <application package>\n"; -} - -sub usage_prepare { - print "Usage: vespa-deploy prepare [<session_id> | <application package>]\n"; -} - -sub usage_activate { - print "Usage: vespa-deploy activate [<session_id>]\n"; -} - -sub usage_fetch { - print "Usage: vespa-deploy fetch <output directory>\n"; -} - -sub fetch_active_application { - my ($outputdir) = @_; - my ($configsource_url, @configsources) = get_configsource_url("fetch"); - my $url = "$configsource_url$pathPrefix/active/content/"; - if ($version eq "v2") { - $url = $configsource_url - . "/application/v2" - . "/tenant/${tenant}" - . "/application/${application}" - . "/environment/${environment}" - . "/region/${region}" - . "/instance/${instance}" - . "/content/"; - } - my $output = http_content($url); - my $exitcode = $? >> 8; - if ($exitcode != 0) { - print_request_failed($exitcode, $configsource_url); - exit 1; - } else { - print "Writing active application to $outputdir\n"; - `mkdir -p $outputdir`; - die "$outputdir is not writeable. Please check permissions\n" if (! -w $outputdir); - my $json_text = get_json($output); - if(ref($json_text) eq 'ARRAY'){ - fetch_directory($json_text, $outputdir); - } else { - print "Error response: $json_text->{message}\n"; - exit 1; - } - } -} - -sub fetch_directory { - my ($json, $outputdir) = @_; - `mkdir -p $outputdir`; - foreach my $entry (@{$json}) { - my $name = "$outputdir/"; - if ($entry =~ /\/([^\/]+\/?)$/) { - $name .= $1; - } - if ($name =~ /(.*)\/$/) { - my $dir = $1; - my $output = http_content($entry); - my $json_text = get_json($output); - fetch_directory($json_text, "$dir"); - } else { - my $output = http_content($entry); - open(FH, ">$name"); - print FH $output; - close(FH); - } - } -} - -sub get_configsource_url { - my ($command) = @_; - - my @configsources; - if ($configserver and $configserver ne "") { - @configsources = ('http://' . $configserver . ':' . $port . '/'); - } else { - @configsources = split(' ', `$VESPA_HOME/bin/vespa-print-default configservers_http`); - } - - my $configsource_url = shift(@configsources); - if (!$configsource_url) { - die "Could not get url to config server, make sure that VESPA_HOME and VESPA_CONFIGSERVERS are set\n"; - } - chomp($configsource_url); - my @temp = split(':', $configsource_url, 3); - $configsource_url = $temp[0] . ":" . $temp[1] . ":" . $port; - if (!$configsource_url) { - print "Could not get url to config server, make sure that VESPA_CONFIGSERVERS is set\n"; - exit 1; - } - - # configsource_url to be used by prepare and activate - if ($command eq "prepare" || $command eq "activate") { - my $temp = get_configsource_url_used(); - if ($temp and $temp ne "") { - $configsource_url = $temp; - debug("Using config server URL " . $configsource_url . " read from file\n"); - } else { - print "Could not read config server URL used for previous upload of an application package, trying to use $configsource_url\n"; - } - } - return ($configsource_url, @configsources); -} - -sub do_http_request { - my ($command, $application_package, $supplied_session_id) = @_; - my ($configsource_url, @configsources) = get_configsource_url($command); - - my $output; - my $exitcode = 1; - if ($command eq "upload") { - ($exitcode, $output) = http_upload(\@configsources, $configsource_url, $application_package); - } elsif ($command eq "prepare") { - $output = http_prepare($configsource_url, $supplied_session_id); - $exitcode = $? >> 8; - } elsif ($command eq "activate") { - $output = http_activate($configsource_url, $supplied_session_id); - $exitcode = $? >> 8; - } - - my $response; - if ($exitcode != 0) { - print_request_failed($exitcode, $configsource_url); - exit 1; - } else { - my $status_code; - ($status_code, $response) = parse_http_response($output); - if ($status_code != 200) { - print "Request failed. HTTP status code: $status_code\n"; - print_response($response); - exit 1; - } - print_response($response); - } -} - -sub http_upload { - my ($temp, $configsource_url, $application_package) = @_; - my @configsources = @{$temp}; - - my $output; - my $exitcode = 0; - my $retry = 0; - my $configsource_url_used = $configsource_url; - LOOP: { - do { - my $status_code; - my $response; - $output = http_upload_lowlevel($configsource_url, $application_package); - ($status_code, $response) = parse_http_response($output); - $exitcode = $? >> 8; - last LOOP if ($exitcode == 0 && $status_code == 200); - - debug("exitcode=$exitcode\n"); - debug("output=$output\n"); - $configsource_url = shift(@configsources); - if ($configsource_url) { - $configsource_url =~ s/\/$//; # Remove last / from configsource_url - $configsource_url_used = $configsource_url; - $retry = 1; - print_request_failed($exitcode, $configsource_url_used); - print "Retrying with another config server\n"; - } else { - if ($exitcode != 0) { - print_request_failed($exitcode, $configsource_url_used); - exit 1; - } else { # Non-200 HTTP status code - print "Request failed. HTTP status code: $status_code\n"; - print_response($response); - exit 1; - } - } - } while ($retry); - } - - write_session_id($output); - write_configsource_url_used($configsource_url_used); - - return ($exitcode, $output); -} - -sub http_upload_lowlevel { - my ($source, $app) = @_; - - my $url = $source . $pathPrefix; - $url = add_url_property_from_flag($url, $opt_v, "verbose"); - if ($opt_F) { - $url = add_url_property_from_option($url, $opt_F, "from"); - `$CURL_POST $url`; - } else { - my $TAR="tar -C $app --dereference --exclude='.[a-zA-Z0-9]*' --exclude=ext -cf - ."; - print "Uploading application '$app' using $url\n"; - if (-f $app) { - `cat $app | $CURL_POST_ZIP $url`; - } else { - `$TAR | $GZIP | $CURL_POST $url`; - } - } -} - -sub http_prepare { - my $source = shift; - my $session_id = shift || get_session_id(); - - my $url = $source . $pathPrefix . "/$session_id/prepared"; - $url = add_url_property_from_flag($url, $opt_f, "ignoreValidationErrors"); - $url = add_url_property_from_flag($url, $opt_n, "dryRun"); - $url = add_url_property_from_flag($url, $opt_v, "verbose"); - $url = add_url_property_from_flag($url, $opt_H, "hostedVespa"); - $url = add_url_property_from_option($url, $opt_a, "applicationName"); - $url = add_url_property_from_option($url, $opt_i, "instance"); - $url = add_url_property_from_option($url, $opt_t, "timeout"); - $url = add_url_property_from_option($url, $opt_R, "rotations"); - $url = add_url_property_from_option($url, $opt_V, "vespaVersion"); - print "Preparing session $session_id using $url\n"; - `$CURL_PUT \"$url\"`; -} - -sub http_content { - my $url = shift; - print "Getting content using $url\n"; - `$CURL_GET \"$url\"`; -} - -sub http_activate { - my $source = shift; - my $session_id = shift || get_session_id(); - - my $url = $source . $pathPrefix . "/$session_id/active"; - $url = add_url_property_from_flag($url, $opt_v, "verbose"); - $url = add_url_property_from_option($url, $opt_t, "timeout"); - print "Activating session $session_id using $url\n"; - `$CURL_PUT \"$url\"`; -} - -sub get_session_id { - my $session_id = `cat $session_id_file 2>/dev/null`; - unless ($session_id) { - print "Could not read session id from file, and no session id supplied as argument. Exiting.\n"; - exit 1 - } - $session_id; -} - -sub get_session_id_from_response { - my ($response) = @_; - - my $new_session_id; - if ($response =~ /.*"session-id":"(\d+)".*/) { $new_session_id = $1; } - $new_session_id; -} - -sub print_response { - my ($response) = @_; - chomp($response); - debug("$response\n"); - if ($response) { - my $json_text = get_json($response); - my $error = $json_text->{error}; - if ($error) { - print "$json_text->{error}\n"; - } - my $status = $json_text->{status}; - foreach my $log_message (@{$json_text->{log}}) { - print "$log_message->{level}: $log_message->{message}\n"; - } - my $message = $json_text->{message}; - if ($message) { - print "$message\n"; - } - my $metadata_deploy = $json_text->{deploy}; - if ($metadata_deploy) { - my $timestamp = $metadata_deploy->{timestamp}; - my $metadata_application = $json_text->{application}; - my $checksum = $metadata_application->{checksum}; - my $generation = $metadata_application->{generation}; - print "Checksum: $checksum\n"; - print "Timestamp: $timestamp\n"; - print "Generation: $generation\n"; - } - } else { - print "Empty response"; - } -} - -sub get_json { - my ($response) = @_; - - my $json_dir = JSON->new; - return $json_dir->utf8->decode($response); -} - -# extend $url with $url_property=true if $flag is set -sub add_url_property_from_flag { - my ($url, $flag, $url_property) = @_; - return $url unless $flag; - return add_url_property($url, "$url_property=true"); -} - -# extend $url with $url_property=$opt if $opt is set -sub add_url_property_from_option { - my ($url, $opt, $url_property) = @_; - return $url unless $opt; - add_url_property($url, "$url_property=$opt"); -} - -sub add_url_property { - my ($url, $url_property) = @_; - if ($url =~ /\?/) { - $url = $url . "&" . $url_property; - } else { - $url = $url . "?" . $url_property; - } - $url; -} - -sub write_session_id { - my ($response) = @_; - - my $new_session_id = get_session_id_from_response($response); - if ($new_session_id) { - open(my $fh, '>', $session_id_file) or die "Could not open file '$session_id_file' $!"; - print $fh $new_session_id; - close $fh; - } -} - -sub write_configsource_url_used { - my ($configsource_url) = @_; - - if ($configsource_url) { - open(my $fh2, '>', $configsource_url_used_file) or die "Could not open file '$configsource_url_used_file' $!"; - print $fh2 $configsource_url; - close $fh2; - } -} - -sub get_configsource_url_used { - my $configsource_url = `cat $configsource_url_used_file 2>/dev/null` || ""; - $configsource_url; -} - -sub print_request_failed { - my ($exitcode, $configsource_url) = @_; - - my $message = "HTTP request failed"; - if ($exitcode == 7) { - $message .= ". Could not connect to $configsource_url"; - } else { - $message .= " with curl exit code $exitcode"; - } - - print $message . "\n"; -} - - -sub debug { - my ($message) = @_; - - if ($opt_v) { - print "$message"; - } -} - -sub parse_http_response { - my ($response) = @_; - - my $message = ""; - my $status_code = 500; - - if ($response =~ /(.*)(\d\d\d)/) { - $message = $1; - $status_code = int($2); - } - return ($status_code, $message); -} - -sub create_cloudconfig_dir { - my $path = "$cloudconfig_dir/$tenant"; - if (-e $path) { - check_dir_permissions($path); - } else { - make_path($path); - } -} - - -sub check_dir_permissions { - my ($dir) = @_; - - if (!(-d $dir)) { - print "$dir is not a directory, please fix\n"; - } - if (!(-r $dir)) { - print "$dir is not readable, please fix\n"; - } -} diff --git a/config-model/src/test/derived/bolding_dynamic_summary/documenttypes.cfg b/config-model/src/test/derived/bolding_dynamic_summary/documenttypes.cfg new file mode 100644 index 00000000000..f88a22d8979 --- /dev/null +++ b/config-model/src/test/derived/bolding_dynamic_summary/documenttypes.cfg @@ -0,0 +1,104 @@ +ignoreundefinedfields false +usev8geopositions false +doctype[].name "document" +doctype[].idx 10000 +doctype[].internalid 8 +doctype[].contentstruct 10001 +doctype[].primitivetype[].idx 10002 +doctype[].primitivetype[].name "bool" +doctype[].primitivetype[].idx 10003 +doctype[].primitivetype[].name "byte" +doctype[].primitivetype[].idx 10004 +doctype[].primitivetype[].name "double" +doctype[].primitivetype[].idx 10005 +doctype[].primitivetype[].name "float" +doctype[].primitivetype[].idx 10006 +doctype[].primitivetype[].name "float16" +doctype[].primitivetype[].idx 10007 +doctype[].primitivetype[].name "int" +doctype[].primitivetype[].idx 10008 +doctype[].primitivetype[].name "long" +doctype[].primitivetype[].idx 10010 +doctype[].primitivetype[].name "predicate" +doctype[].primitivetype[].idx 10011 +doctype[].primitivetype[].name "raw" +doctype[].primitivetype[].idx 10012 +doctype[].primitivetype[].name "string" +doctype[].primitivetype[].idx 10014 +doctype[].primitivetype[].name "uri" +doctype[].wsettype[].idx 10013 +doctype[].wsettype[].elementtype 10012 +doctype[].wsettype[].createifnonexistent true +doctype[].wsettype[].removeifzero true +doctype[].wsettype[].internalid 18 +doctype[].structtype[].idx 10001 +doctype[].structtype[].name "document.header" +doctype[].structtype[].internalid -284186494 +doctype[].structtype[].idx 10009 +doctype[].structtype[].name "position" +doctype[].structtype[].field[].name "x" +doctype[].structtype[].field[].internalid 914677694 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].field[].name "y" +doctype[].structtype[].field[].internalid 900009410 +doctype[].structtype[].field[].type 10007 +doctype[].structtype[].internalid 1381038251 +doctype[].name "test" +doctype[].idx 10015 +doctype[].internalid -877171244 +doctype[].inherits[].idx 10000 +doctype[].contentstruct 10016 +doctype[].fieldsets{[]}.fields[] "arr_1" +doctype[].fieldsets{[]}.fields[] "arr_2" +doctype[].fieldsets{[]}.fields[] "arr_3" +doctype[].fieldsets{[]}.fields[] "arr_4" +doctype[].fieldsets{[]}.fields[] "str_1" +doctype[].fieldsets{[]}.fields[] "str_2" +doctype[].fieldsets{[]}.fields[] "str_3" +doctype[].fieldsets{[]}.fields[] "str_4" +doctype[].arraytype[].idx 10017 +doctype[].arraytype[].elementtype 10012 +doctype[].arraytype[].internalid -1486737430 +doctype[].arraytype[].idx 10018 +doctype[].arraytype[].elementtype 10012 +doctype[].arraytype[].internalid -1486737430 +doctype[].arraytype[].idx 10019 +doctype[].arraytype[].elementtype 10012 +doctype[].arraytype[].internalid -1486737430 +doctype[].arraytype[].idx 10020 +doctype[].arraytype[].elementtype 10012 +doctype[].arraytype[].internalid -1486737430 +doctype[].structtype[].idx 10016 +doctype[].structtype[].name "test.header" +doctype[].structtype[].field[].name "str_1" +doctype[].structtype[].field[].internalid 91206053 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "str_2" +doctype[].structtype[].field[].internalid 1236519725 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "str_3" +doctype[].structtype[].field[].internalid 1257861515 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "str_4" +doctype[].structtype[].field[].internalid 2009526185 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "arr_1" +doctype[].structtype[].field[].internalid 618314164 +doctype[].structtype[].field[].type 10017 +doctype[].structtype[].field[].name "arr_2" +doctype[].structtype[].field[].internalid 1363829203 +doctype[].structtype[].field[].type 10018 +doctype[].structtype[].field[].name "arr_3" +doctype[].structtype[].field[].internalid 815179933 +doctype[].structtype[].field[].type 10019 +doctype[].structtype[].field[].name "arr_4" +doctype[].structtype[].field[].internalid 142340927 +doctype[].structtype[].field[].type 10020 +doctype[].structtype[].field[].name "str_3_dyn" +doctype[].structtype[].field[].internalid 1779429789 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].field[].name "str_4_bold" +doctype[].structtype[].field[].internalid 1385662257 +doctype[].structtype[].field[].type 10012 +doctype[].structtype[].internalid 306916075 + diff --git a/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg b/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg new file mode 100644 index 00000000000..f9fdab87297 --- /dev/null +++ b/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg @@ -0,0 +1,19 @@ +maxtermoccurrences 100 +fieldmatchmaxlength 1000000 +ilscript[].doctype "test" +ilscript[].docfield[] "str_1" +ilscript[].docfield[] "str_2" +ilscript[].docfield[] "str_3" +ilscript[].docfield[] "str_4" +ilscript[].docfield[] "arr_1" +ilscript[].docfield[] "arr_2" +ilscript[].docfield[] "arr_3" +ilscript[].docfield[] "arr_4" +ilscript[].content[] "clear_state | guard { input str_1 | tokenize normalize stem:\"BEST\" | index str_1 | summary str_1; }" +ilscript[].content[] "clear_state | guard { input str_2 | tokenize normalize stem:\"BEST\" | index str_2 | summary str_2; }" +ilscript[].content[] "clear_state | guard { input str_3 | tokenize normalize stem:\"BEST\" | index str_3 | summary str_3 | summary str_3_dyn; }" +ilscript[].content[] "clear_state | guard { input str_4 | tokenize normalize stem:\"BEST\" | index str_4 | summary str_4 | summary str_4_bold; }" +ilscript[].content[] "clear_state | guard { input arr_1 | for_each { tokenize normalize stem:\"BEST\" } | index arr_1 | summary arr_1; }" +ilscript[].content[] "clear_state | guard { input arr_2 | for_each { tokenize normalize stem:\"BEST\" } | index arr_2 | summary arr_2; }" +ilscript[].content[] "clear_state | guard { input arr_3 | for_each { tokenize normalize stem:\"BEST\" } | index arr_3 | summary arr_3; }" +ilscript[].content[] "clear_state | guard { input arr_4 | for_each { tokenize normalize stem:\"BEST\" } | index arr_4 | summary arr_4; }" diff --git a/config-model/src/test/derived/bolding_dynamic_summary/summary.cfg b/config-model/src/test/derived/bolding_dynamic_summary/summary.cfg new file mode 100644 index 00000000000..b056ada8bd5 --- /dev/null +++ b/config-model/src/test/derived/bolding_dynamic_summary/summary.cfg @@ -0,0 +1,92 @@ +defaultsummaryid 1128441658 +usev8geopositions true +classes[].id 1128441658 +classes[].name "default" +classes[].omitsummaryfeatures false +classes[].fields[].name "str_1" +classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "str_1" +classes[].fields[].name "str_2" +classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "str_2" +classes[].fields[].name "str_3" +classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" +classes[].fields[].name "str_4" +classes[].fields[].type "longstring" +classes[].fields[].command "" +classes[].fields[].source "" +classes[].fields[].name "arr_1" +classes[].fields[].type "jsonstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "arr_1" +classes[].fields[].name "arr_2" +classes[].fields[].type "jsonstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "arr_2" +classes[].fields[].name "arr_3" +classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" +classes[].fields[].name "arr_4" +classes[].fields[].type "jsonstring" +classes[].fields[].command "" +classes[].fields[].source "" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" +classes[].fields[].name "str_3_dyn" +classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "str_3_dyn" +classes[].fields[].name "arr_3_dyn" +classes[].fields[].type "jsonstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "arr_3" +classes[].fields[].name "str_4_bold" +classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "str_4_bold" +classes[].fields[].name "arr_4_bold" +classes[].fields[].type "jsonstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "arr_4" +classes[].fields[].name "documentid" +classes[].fields[].type "longstring" +classes[].fields[].command "documentid" +classes[].fields[].source "" +classes[].id 2139497711 +classes[].name "dyn" +classes[].omitsummaryfeatures false +classes[].fields[].name "str_3_dyn" +classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "str_3_dyn" +classes[].fields[].name "arr_3_dyn" +classes[].fields[].type "jsonstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "arr_3" +classes[].fields[].name "str_4_bold" +classes[].fields[].type "longstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "str_4_bold" +classes[].fields[].name "arr_4_bold" +classes[].fields[].type "jsonstring" +classes[].fields[].command "dynamicteaser" +classes[].fields[].source "arr_4" +classes[].fields[].name "rankfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].command "rankfeatures" +classes[].fields[].source "" +classes[].fields[].name "summaryfeatures" +classes[].fields[].type "featuredata" +classes[].fields[].command "summaryfeatures" +classes[].fields[].source "" diff --git a/config-model/src/test/derived/bolding_dynamic_summary/test.sd b/config-model/src/test/derived/bolding_dynamic_summary/test.sd new file mode 100644 index 00000000000..caa7ca2cd2e --- /dev/null +++ b/config-model/src/test/derived/bolding_dynamic_summary/test.sd @@ -0,0 +1,50 @@ +schema test { + document test { + field str_1 type string { + indexing: index | summary + summary: dynamic + } + field str_2 type string { + indexing: index | summary + bolding: on + } + field str_3 type string { + indexing: index | summary + } + field str_4 type string { + indexing: index | summary + } + field arr_1 type array<string> { + indexing: index | summary + summary: dynamic + } + field arr_2 type array<string> { + indexing: index | summary + bolding: on + } + field arr_3 type array<string> { + indexing: index | summary + } + field arr_4 type array<string> { + indexing: index | summary + } + } + document-summary dyn { + summary str_3_dyn type string { + source: str_3 + dynamic + } + summary arr_3_dyn type array<string> { + source: arr_3 + dynamic + } + summary str_4_bold type string { + source: str_4 + bolding: on + } + summary arr_4_bold type array<string> { + source: arr_4 + bolding: on + } + } +} diff --git a/config-model/src/test/java/com/yahoo/schema/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/ExportingTestCase.java index b5b075ae260..c1e65abb5a5 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/ExportingTestCase.java @@ -192,4 +192,9 @@ public class ExportingTestCase extends AbstractExportingTestCase { assertCorrectDeriving("structandfieldset"); } + @Test + void testBoldingAndDynamicSummary() throws IOException, ParseException { + assertCorrectDeriving("bolding_dynamic_summary"); + } + } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/BoldingTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/BoldingTestCase.java index fe38ea1f1b1..7162cf7717b 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/BoldingTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/BoldingTestCase.java @@ -39,10 +39,10 @@ public class BoldingTestCase extends AbstractSchemaTestCase { } } - private final String boldonarray = - "search boldonarray {\n" + - " document boldonarray {\n" + - " field myarray type array<string> {\n" + + private final String boldonwset = + "search test {\n" + + " document test {\n" + + " field mywset type weightedset<string> {\n" + " indexing: summary | index\n" + " bolding: on\n" + " }\n" + @@ -50,12 +50,12 @@ public class BoldingTestCase extends AbstractSchemaTestCase { "}\n"; @Test - void testBoldOnArray() throws ParseException { + void testBoldOnWsetThrowsException() throws ParseException { try { - ApplicationBuilder.createFromString(boldonarray); + ApplicationBuilder.createFromString(boldonwset); fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("'bolding: on' for non-text field 'myarray' (datatype Array<string> (code: -1486737430)) is not allowed", + assertEquals("'bolding: on' for non-text field 'mywset' (datatype WeightedSet<string> (code: 1328286588)) is not allowed", e.getMessage()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UriBindingsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UriBindingsValidatorTest.java index 7c16d3c99cf..ff9596f2062 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UriBindingsValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/UriBindingsValidatorTest.java @@ -58,6 +58,11 @@ public class UriBindingsValidatorTest { } @Test + void allows_portbinding_when_restricting_data_plane() throws IOException, SAXException { + runUriBindingValidator(new TestProperties().setHostedVespa(true).setUseRestrictedDataPlaneBindings(true), createServicesXmlWithHandler("http://*:4443/my-handler")); + } + + @Test void allows_user_binding_with_wildcard_port() throws IOException, SAXException { runUriBindingValidator(true, createServicesXmlWithHandler("http://*:*/my-handler")); } @@ -68,12 +73,16 @@ public class UriBindingsValidatorTest { } private void runUriBindingValidator(boolean isHosted, String servicesXml) throws IOException, SAXException { + runUriBindingValidator(new TestProperties().setHostedVespa(isHosted), servicesXml); + } + + private void runUriBindingValidator(TestProperties testProperties, String servicesXml) throws IOException, SAXException { ApplicationPackage app = new MockApplicationPackage.Builder() .withServices(servicesXml) .build(); DeployState deployState = new DeployState.Builder() .applicationPackage(app) - .properties(new TestProperties().setHostedVespa(isHosted)) + .properties(testProperties) .build(); VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); new UriBindingsValidator().validate(model, deployState); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/HandlerBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/HandlerBuilderTest.java index 291a5f21305..186842ecbf1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/HandlerBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/HandlerBuilderTest.java @@ -1,6 +1,8 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.container.ComponentsConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; import com.yahoo.container.usability.BindingsOverviewHandler; @@ -10,10 +12,14 @@ import com.yahoo.vespa.model.container.component.Handler; import org.junit.jupiter.api.Test; import org.w3c.dom.Element; +import java.util.List; +import java.util.Map; + import static com.yahoo.vespa.model.container.ContainerCluster.ROOT_HANDLER_BINDING; import static com.yahoo.vespa.model.container.ContainerCluster.STATE_HANDLER_BINDING_1; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasItem; import static org.junit.jupiter.api.Assertions.*; @@ -100,6 +106,111 @@ public class HandlerBuilderTest extends ContainerModelBuilderTestBase { assertTrue(handler.getInjectedComponentIds().contains("threadpool@default-handler-common")); } + @Test + void restricts_default_bindings_in_hosted_vespa() { + DeployState deployState = new DeployState.Builder() + .properties(new TestProperties().setHostedVespa(true).setUseRestrictedDataPlaneBindings(true)) + .build(); + verifyDefaultBindings(deployState, "http://*:4443"); + } + + @Test + void does_not_restrict_default_bindings_in_hosted_vespa_when_disabled() { + DeployState deployState = new DeployState.Builder() + .properties(new TestProperties().setHostedVespa(true).setUseRestrictedDataPlaneBindings(false)) + .build(); + verifyDefaultBindings(deployState, "http://*"); + } + + @Test + void restricts_custom_bindings_in_hosted_vespa() { + DeployState deployState = new DeployState.Builder() + .properties(new TestProperties().setHostedVespa(true).setUseRestrictedDataPlaneBindings(true)) + .build(); + verifyCustomSearchBindings(deployState, "http://*:4443"); + } + + @Test + void does_not_restrict_default_bindings_in_self_hosted() { + DeployState deployState = new DeployState.Builder() + .properties(new TestProperties().setHostedVespa(false).setUseRestrictedDataPlaneBindings(false)) + .build(); + verifyDefaultBindings(deployState, "http://*"); + } + + @Test + void does_not_restrict_custom_bindings_in_self_hosted() { + DeployState deployState = new DeployState.Builder() + .properties(new TestProperties().setHostedVespa(false).setUseRestrictedDataPlaneBindings(false)) + .build(); + verifyCustomSearchBindings(deployState, "http://*"); + } + + private void verifyDefaultBindings(DeployState deployState, String bindingPrefix) { + Element clusterElem = DomBuilderTest.parse( + "<container id='default' version='1.0'>", + " <search/>", + " <document-api/>", + " <handler id='FooHandler'>", + " <binding>http://*/foo</binding>", + " </handler>", + nodesXml, + "</container>"); + + createModel(root, deployState, null, clusterElem); + JdiscBindingsConfig bindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default"); + + // Verify /search /feed /document and custom handler are bound correctly + Map<String, JdiscBindingsConfig.Handlers> handlers = bindingsConfig.handlers(); + Map<String, List<String>> expectedHandlerMappings = Map.of( + "com.yahoo.search.handler.SearchHandler", List.of("/search/*"), + "com.yahoo.document.restapi.resource.DocumentV1ApiHandler", List.of("/document/v1/*", "/document/v1/*/"), + "com.yahoo.vespa.http.server.FeedHandler", List.of("/reserved-for-internal-use/feedapi", "/reserved-for-internal-use/feedapi/"), + "FooHandler", List.of("/foo")); + expectedHandlerMappings.forEach((handler, bindings) -> validateHandler(handlers.get(handler), bindingPrefix, bindings)); + + // All other handlers should be bound to default (http://*/...) + handlers.entrySet().stream() + .filter(e -> ! expectedHandlerMappings.containsKey(e.getKey())) + .forEach(e -> assertTrue(e.getValue().serverBindings().stream().allMatch(s -> s.startsWith("http://*/")))); + } + + private void verifyCustomSearchBindings(DeployState deployState, String bindingPrefix) { + Element clusterElem = DomBuilderTest.parse( + "<container id='default' version='1.0'>", + " <search>", + " <binding>http://*/search-binding/</binding>", + " </search>", + " <document-api>", + " <binding>http://*/docapi-binding/</binding>", + " </document-api>", + nodesXml, + "</container>"); + + createModel(root, deployState, null, clusterElem); + JdiscBindingsConfig bindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default"); + + // Verify search feed and document handler are bound correctly + Map<String, JdiscBindingsConfig.Handlers> handlers = bindingsConfig.handlers(); + Map<String, List<String>> expectedHandlerMappings = Map.of( + "com.yahoo.search.handler.SearchHandler", List.of("/search-binding/"), + "com.yahoo.document.restapi.resource.DocumentV1ApiHandler", List.of("/docapi-binding/document/v1/*", "/docapi-binding/document/v1/*/"), + "com.yahoo.vespa.http.server.FeedHandler", List.of("/docapi-binding/reserved-for-internal-use/feedapi", "/docapi-binding/reserved-for-internal-use/feedapi/")); + expectedHandlerMappings.forEach((handler, bindings) -> validateHandler(handlers.get(handler), bindingPrefix, bindings)); + + // All other handlers should be bound to default (http://*/...) + handlers.entrySet().stream() + .filter(e -> ! expectedHandlerMappings.containsKey(e.getKey())) + .forEach(e -> assertTrue(e.getValue().serverBindings().stream().allMatch(s -> s.startsWith("http://*/")))); + + } + + private void validateHandler(JdiscBindingsConfig.Handlers handler, String bindingPrefix, List<String> expectedBindings) { + assertNotNull(handler); + assertEquals(expectedBindings.size(), handler.serverBindings().size()); + assertThat(handler.serverBindings(), containsInAnyOrder(expectedBindings.stream().map(s->bindingPrefix+s).toArray())); + } + private void createClusterWithJDiscHandler() { Element clusterElem = DomBuilderTest.parse( "<container id='default' version='1.0'>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index ac291fc578f..132255570ac 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1036,6 +1036,27 @@ public class ContentClusterTest extends ContentBaseTest { } } + private boolean coverageIsComputedFromTargetActive(Boolean coverageFromTargetActive) { + TestProperties properties = new TestProperties(); + if (coverageFromTargetActive != null) { + properties.setComputeCoverageFromTargetActiveDocs(coverageFromTargetActive); + } + VespaModel model = createEnd2EndOneNode(properties); + + ContentCluster cc = model.getContentClusters().get("storage"); + DispatchConfig.Builder builder = new DispatchConfig.Builder(); + cc.getSearch().getConfig(builder); + + return (new DispatchConfig(builder)).computeCoverageFromTargetActiveDocs(); + } + + @Test + public void coverage_from_target_active_dispatch_config_is_controlled_by_properties() { + assertFalse(coverageIsComputedFromTargetActive(null)); // TODO update when default changes + assertFalse(coverageIsComputedFromTargetActive(false)); + assertTrue(coverageIsComputedFromTargetActive(true)); + } + private boolean resolveThreePhaseUpdateConfigWithFeatureFlag(boolean flagEnableThreePhase) { VespaModel model = createEnd2EndOneNode(new TestProperties().setUseThreePhaseUpdates(flagEnableThreePhase)); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java index 1104ac7477a..ee5cf57a396 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java @@ -15,6 +15,7 @@ import java.util.List; import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; /** @@ -63,6 +64,7 @@ public class ClusterTest { assertEquals(0.23, config.minWaitAfterCoverageFactor(), DELTA); assertEquals(0.58, config.maxWaitAfterCoverageFactor(), DELTA); assertEquals(2, config.searchableCopies()); + assertEquals(3, config.redundancy()); assertEquals(DispatchConfig.DistributionPolicy.ADAPTIVE, config.distributionPolicy()); } @@ -80,6 +82,7 @@ public class ClusterTest { cluster.getSearch().getConfig(builder); DispatchConfig config = new DispatchConfig(builder); assertEquals(2, config.searchableCopies()); + assertEquals(3, config.redundancy()); assertEquals(93.0, config.minActivedocsPercentage(), DELTA); assertEquals(DispatchConfig.DistributionPolicy.ROUNDROBIN, config.distributionPolicy()); assertEquals(77, config.maxHitsPerNode()); @@ -94,6 +97,8 @@ public class ClusterTest { cluster.getSearch().getConfig(builder); DispatchConfig config = new DispatchConfig(builder); assertEquals(2, config.searchableCopies()); + assertEquals(3, config.redundancy()); + assertFalse(config.computeCoverageFromTargetActiveDocs()); assertEquals(DispatchConfig.DistributionPolicy.ADAPTIVE, config.distributionPolicy()); assertEquals(1.0, config.maxWaitAfterCoverageFactor(), DELTA); assertEquals(0, config.minWaitAfterCoverageFactor(), DELTA); diff --git a/config/CMakeLists.txt b/config/CMakeLists.txt index 24d5513bbfc..1c04aa3eaa8 100644 --- a/config/CMakeLists.txt +++ b/config/CMakeLists.txt @@ -54,5 +54,3 @@ vespa_define_module( src/tests/subscription src/tests/configparser ) - -vespa_install_script(src/apps/vespa-config/vespa-config.pl libexec/vespa) diff --git a/config/src/apps/vespa-config/vespa-config.pl b/config/src/apps/vespa-config/vespa-config.pl deleted file mode 100755 index 79c0a352e37..00000000000 --- a/config/src/apps/vespa-config/vespa-config.pl +++ /dev/null @@ -1,286 +0,0 @@ -#!/usr/bin/env perl -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# -# Various small functions used when bootstrapping the config system - -# BEGIN perl environment bootstrap section -# Do not edit between here and END as this section should stay identical in all scripts - -use File::Basename; -use File::Path; - -sub findpath { - my $myfullname = ${0}; - my($myname, $mypath) = fileparse($myfullname); - - return $mypath if ( $mypath && -d $mypath ); - $mypath=`pwd`; - - my $pwdfullname = $mypath . "/" . $myname; - return $mypath if ( -f $pwdfullname ); - return 0; -} - -# Returns the argument path if it seems to point to VESPA_HOME, 0 otherwise -sub is_vespa_home { - my($VESPA_HOME) = shift; - my $COMMON_ENV="libexec/vespa/common-env.sh"; - if ( $VESPA_HOME && -d $VESPA_HOME ) { - my $common_env = $VESPA_HOME . "/" . $COMMON_ENV; - return $VESPA_HOME if -f $common_env; - } - return 0; -} - -# Returns the home of Vespa, or dies if it cannot -sub findhome { - # Try the VESPA_HOME env variable - return $ENV{'VESPA_HOME'} if is_vespa_home($ENV{'VESPA_HOME'}); - if ( $ENV{'VESPA_HOME'} ) { # was set, but not correctly - die "FATAL: bad VESPA_HOME value '" . $ENV{'VESPA_HOME'} . "'\n"; - } - - # Try the ROOT env variable - my $ROOT = $ENV{'ROOT'}; - return $ROOT if is_vespa_home($ROOT); - - # Try the script location or current dir - my $mypath = findpath(); - if ($mypath) { - while ( $mypath =~ s|/[^/]*$|| ) { - return $mypath if is_vespa_home($mypath); - } - } - die "FATAL: Missing VESPA_HOME environment variable\n"; -} - -sub findhost { - my $tmp = $ENV{'VESPA_HOSTNAME'}; - my $bin = $ENV{'VESPA_HOME'} . "/bin"; - if (!defined $tmp) { - $tmp = `${bin}/vespa-detect-hostname || hostname -f || hostname || echo "localhost"`; - chomp $tmp; - } - my $validate = "${bin}/vespa-validate-hostname"; - if (-f "${validate}") { - system("${validate} $tmp"); - ( $? == 0 ) or die "Could not validate hostname\n"; - } - return $tmp; -} - -BEGIN { - my $tmp = findhome(); - $ENV{'VESPA_HOME'} = $tmp; - $tmp = findhost(); - $ENV{'VESPA_HOSTNAME'} = $tmp; -} -my $VESPA_HOME = $ENV{'VESPA_HOME'}; - -use lib $ENV{'VESPA_HOME'} . "/lib/perl5/site_perl"; - -# END perl environment bootstrap section - -use Yahoo::Vespa::Defaults; -readConfFile(); - -use strict; -use warnings; -use File::Copy; -use File::Temp; - -my $myHostname = `vespa-print-default hostname`; -chomp $myHostname; -my $default_configproxy_port = "19090"; -my $default_configserver_port = "19070"; - -my $base_cfg_dir = $VESPA_HOME . "/conf/vespa"; - -# Set this to 1 to look up values (see getValue) in config files instead -# of in environment variables -my $lookupInConfig = 0; - - -sub getValue { - my ($varname, $prefix) = @_; - if ($lookupInConfig) { - return getConfigValue($varname, $prefix); - } - else { - return getEnvironmentValue($varname, $prefix); - } -} - -sub getConfigValue { - my ($varname, $config) = @_; - my $path = "$base_cfg_dir/$config.conf"; - if (open(CFG, "<$path")) { - while (<CFG>) { - chomp; - if ( m{^(\w+)\s(.+)} ) { - return $2 if $1 eq $varname; - } - } - close(CFG); - } - return; -} - -sub getEnvironmentValue { - my ($varname, $prefix) = @_; - my $value = $ENV{$prefix . "__" . $varname}; - if (defined $value && $value =~ m{^\s*(\S.*)\s*}) { - return $1; - } - return $value; -} - -sub getCCSVar { - my ($varname, $default) = @_; - my $value = getValue($varname, "cloudconfig_server"); - if (defined($value)) { - return $value; - } - return $default; -} - -sub getServicesVar { - my ($varname, $default, $warn) = @_; - # print "GET var '$varname'\n"; - my $cloud = getValue($varname, "services"); - my $plain = $ENV{$varname}; - if (defined($cloud)) { - return $cloud; - } elsif (defined($plain)) { - return $plain; - } elsif ($warn > 0) { - print STDERR "No value found for 'services.$varname'; using '$default'\n"; - } - return $default; -} - -sub getConfigServerPort { - my $port = getServicesVar('port_configserver_rpc', $default_configserver_port, 0); - return $port; -} - -sub printConfigServerPort { - my $port = getConfigServerPort(); - print "$port\n"; -} - -sub getConfigServers { - my @ret; - - my $addr = $ENV{'VESPA_CONFIGSERVERS'}; - if (! defined($addr)) { - $addr = getServicesVar('addr_configserver', $myHostname, 1); - } - my $port = getConfigServerPort(); - - my $h; - foreach $h (split(/,|\s+/, $addr)) { - if ($h =~ m{(\S+:\d+)}) { - push @ret, $1; - } else { - push @ret, "${h}:${port}"; - } - } - return @ret; -} - - -sub printAllConfigSourcesWithPort { - my $cfport = getConfigServerPort(); - my $cpport = getServicesVar('port_configproxy_rpc', $default_configproxy_port, 0); - my $addr = "localhost"; - my $out = "tcp/${addr}:${cpport}"; - foreach $addr (getConfigServers()) { - if ($addr =~ m{\/}) { - if ($addr =~ m{\:}) { - $out .= ",${addr}"; - } else { - $out .= ",${addr}:${cfport}"; - } - } else { - if ($addr =~ m{\:}) { - $out .= ",tcp/${addr}"; - } else { - $out .= ",tcp/${addr}:${cfport}"; - } - } - } - print $out . "\n"; -} - -sub printConfigSources { - my $out; - my $addr; - foreach $addr (getConfigServers()) { - $out .= "tcp/${addr},"; - } - chop($out); # last comma - print $out . "\n"; -} - -sub printConfigHttpSources { - my $out; - my $addr; - foreach $addr (getConfigServers()) { - my $host = ""; - my $port = 0; - if ($addr =~ /(.*):(\d+)$/) { - $host = $1; - $port = $2; - } - $port++; # HTTP is rpc + 1 - $out .= "http://$host:$port "; - } - chop($out); # last space - print $out . "\n"; -} - -# Perl trim function to remove whitespace from the start and end of the string -sub trim($) { - my $string = shift; - $string =~ s/^\s+//; - $string =~ s/\s+$//; - return $string; -} - -sub getLastLine { - my ($file) = @_; - `grep -v \"\^\$\" $file | tail -n 1` # skip blank lines -} - -sub usage { - print "usage: "; - print "vespa-config [-configsources | -confighttpsources | -configserverport]\n"; -} - -if ( @ARGV == 0 ) { - usage(); - exit 1; -} - -if ( $ARGV[0] eq "-allconfigsources" ) { - printAllConfigSourcesWithPort(); - exit 0; -} -if ( $ARGV[0] eq "-configsources" ) { - printConfigSources(); - exit 0; -} -if ( $ARGV[0] eq "-confighttpsources" ) { - $lookupInConfig = 1; - printConfigHttpSources(); - exit 0; -} -if ( $ARGV[0] eq "-configserverport" ) { - $lookupInConfig = 1; - printConfigServerPort(); - exit 0; -} - -usage(); -exit 1; diff --git a/configdefinitions/src/vespa/dispatch.def b/configdefinitions/src/vespa/dispatch.def index e26a136d245..fb3fc4a331a 100644 --- a/configdefinitions/src/vespa/dispatch.def +++ b/configdefinitions/src/vespa/dispatch.def @@ -33,9 +33,13 @@ useMultilevelDispatch bool default=false # Dispatch only to local nodes. DEPRECATED: The container will automatically do this when it is appropriate. useLocalNode bool default=false -# Number of document copies +# Number of document replicas _per group_ that will be indexed in a stable cluster. searchableCopies long default=1 +# Number of document replicas _per group_ that will be present in a stable cluster. +# Should always be >= searchableCopies. +redundancy long default=1 + # Minimum search coverage required before returning the results of a query minSearchCoverage double default=100 @@ -69,3 +73,7 @@ node[].port int # TODO(bjorncs) Remove after May 2022 # Temporary feature flag mergeGroupingResultInSearchInvokerEnabled bool default=false + +# Whether degraded coverage computation will take target active docs into +# account, not just currently active docs. +computeCoverageFromTargetActiveDocs bool default=false diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index cdbf66f1734..d59b5da84da 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -225,6 +225,7 @@ public class ModelContextImpl implements ModelContext { private final int rpc_events_before_wakeup; private final int clusterControllerStateGatherCount; private final boolean useRestrictedDataPlaneBindings; + private final boolean computeCoverageFromTargetActiveDocs; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -286,6 +287,7 @@ public class ModelContextImpl implements ModelContext { this.phraseOptimization = flagValue(source, appId, version, Flags.PHRASE_OPTIMIZATION); this.clusterControllerStateGatherCount = flagValue(source, appId, version, Flags.CLUSTER_CONTROLLER_STATE_GATHER_COUNT); this.useRestrictedDataPlaneBindings = flagValue(source, appId, version, Flags.RESTRICT_DATA_PLANE_BINDINGS); + this.computeCoverageFromTargetActiveDocs = flagValue(source, appId, version, Flags.COMPUTE_COVERAGE_FROM_TARGET_ACTIVE_DOCS); } @Override public String queryDispatchPolicy() { return queryDispatchPolicy; } @@ -355,6 +357,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean useTwoPhaseDocumentGc() { return useTwoPhaseDocumentGc; } @Override public int clusterControllerStateGatherCount() { return clusterControllerStateGatherCount; } @Override public boolean useRestrictedDataPlaneBindings() { return useRestrictedDataPlaneBindings; } + @Override public boolean computeCoverageFromTargetActiveDocs() { return computeCoverageFromTargetActiveDocs; } private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java index 480ee96393a..df4f4bcb9ea 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java @@ -83,7 +83,7 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { @Override protected Response newResponse() { Response response = new Response(Response.Status.OK); - response.headers().add(HttpHeaders.Names.CONTENT_TYPE, "application/json"); + response.headers().add(HttpHeaders.Names.CONTENT_TYPE, getContentType(request.getUri().getQuery())); return response; } @@ -225,4 +225,11 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { } } + private String getContentType(String query) { + if ("format=prometheus".equals(query)) { + return "text/plain;charset=utf-8"; + } + return "application/json"; + } + } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/PrometheusHelper.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/PrometheusHelper.java index ca12e8161a9..43aaba555ab 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/PrometheusHelper.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/PrometheusHelper.java @@ -13,7 +13,7 @@ import static com.yahoo.container.jdisc.state.JsonUtil.sanitizeDouble; */ public class PrometheusHelper { - private static final String HELP_LINE = "# HELP %s\n# TYPE %s untyped\n"; + private static final String HELP_LINE = "# HELP %s \n# TYPE %s untyped\n"; private static final String METRIC_LINE = "%s{%s} %s %d\n"; protected static byte[] buildPrometheusOutput(MetricSnapshot metricSnapshot, String application, long timestamp) throws IOException { diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java index 56efa396297..0072b32ad7d 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java @@ -169,16 +169,16 @@ public class MetricsPacketsHandlerTest extends StateHandlerTestBase { snapshotProvider.setSnapshot(snapshot); var response = requestAsString("http://localhost/metrics-packets?format=prometheus"); var expectedResponse = """ - # HELP gauge_metric_last + # HELP gauge_metric_last\s # TYPE gauge_metric_last untyped gauge_metric_last{dim1="value1",vespa_service="state-handler-test-base",} 0.2 0 - # HELP gauge_metric_average + # HELP gauge_metric_average\s # TYPE gauge_metric_average untyped gauge_metric_average{dim1="value1",vespa_service="state-handler-test-base",} 0.2 0 - # HELP gauge_metric_max + # HELP gauge_metric_max\s # TYPE gauge_metric_max untyped gauge_metric_max{dim1="value1",vespa_service="state-handler-test-base",} 0.2 0 - # HELP counter_metric_count + # HELP counter_metric_count\s # TYPE counter_metric_count untyped counter_metric_count{dim1="value1",vespa_service="state-handler-test-base",} 5 0 """; diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java index 44c490b8ed1..4fc3f7919cf 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java @@ -11,6 +11,11 @@ import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; import com.yahoo.component.chain.dependencies.Provides; +import com.yahoo.data.access.ArrayTraverser; +import com.yahoo.data.access.Inspectable; +import com.yahoo.data.access.Inspector; +import com.yahoo.data.access.Type; +import com.yahoo.data.access.simple.Value; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.search.Searcher; @@ -104,14 +109,51 @@ public class JuniperSearcher extends Searcher { for (Index index : indexFacts.getIndexes(searchDefinitionField.toString())) { if (index.getDynamicSummary() || index.getHighlightSummary()) { - HitField fieldValue = fastHit.buildHitField(index.getName(), true); - if (fieldValue != null) - insertTags(fieldValue, bolding, index.getDynamicSummary()); + var field = fastHit.getField(index.getName()); + if (StringArrayConverter.shouldHandleField(field)) { + new StringArrayConverter(fastHit, index, field, bolding); + } else { + HitField fieldValue = fastHit.buildHitField(index.getName(), true); + if (fieldValue != null) { + insertTags(fieldValue, bolding, index.getDynamicSummary()); + } + } } } } } + private class StringArrayConverter implements ArrayTraverser { + + private Index index; + private boolean bolding; + private Value.ArrayValue convertedField = new Value.ArrayValue(); + + /** + * This converts the backend binary highlighting of each item in an array of string field, + * and creates a new field that replaces the original. + */ + StringArrayConverter(FastHit hit, Index index, Object field, boolean bolding) { + this.index = index; + this.bolding = bolding; + ((Inspectable)field).inspect().traverse(this); + hit.setField(index.getName(), convertedField); + } + + static boolean shouldHandleField(Object field) { + return (field instanceof Inspectable) && + (((Inspectable)field).inspect().type() == Type.ARRAY); + } + + @Override + public void entry(int idx, Inspector inspector) { + // This is how HitField is instantiated in Hit.buildHitField() when forceNoPreTokenize=true. + var hitField = new HitField(index.getName(), inspector.asString(), false); + insertTags(hitField, bolding, index.getDynamicSummary()); + convertedField.add(hitField.getContent()); + } + } + private void insertTags(HitField field, boolean bolding, boolean dynteaser) { boolean insideHighlight = false; for (ListIterator<FieldPart> i = field.listIterator(); i.hasNext();) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/AdaptiveTimeoutHandler.java b/container-search/src/main/java/com/yahoo/search/dispatch/AdaptiveTimeoutHandler.java new file mode 100644 index 00000000000..fbc179a10fa --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/AdaptiveTimeoutHandler.java @@ -0,0 +1,69 @@ +package com.yahoo.search.dispatch; + +import com.yahoo.concurrent.Timer; +import com.yahoo.search.Query; +import com.yahoo.vespa.config.search.DispatchConfig; + +import static com.yahoo.container.handler.Coverage.DEGRADED_BY_ADAPTIVE_TIMEOUT; + +/** + * Computes next timeout based on how many responses has been received so far + * + * @author baldersheim + */ +class AdaptiveTimeoutHandler implements TimeoutHandler { + private final double minimumCoverage; + private final int askedNodes; + private final int minimumResponses; + private final Query query; + private final Timer timer; + private final DispatchConfig config; + + private long deadline; + private long adaptiveTimeoutMin; + private long adaptiveTimeoutMax; + boolean adaptiveTimeoutCalculated = false; + + AdaptiveTimeoutHandler(Timer timer, DispatchConfig config, int askedNodes, Query query) { + minimumCoverage = config.minSearchCoverage(); + this.config = config; + this.askedNodes = askedNodes; + this.query = query; + this.timer = timer; + minimumResponses = (int) Math.ceil(askedNodes * minimumCoverage / 100.0); + deadline = timer.milliTime() + query.getTimeLeft(); + } + + @Override + public long nextTimeoutMS(int answeredNodes) { + if (askedNodes == answeredNodes) return query.getTimeLeft(); // All nodes have responded - done + if (answeredNodes < minimumResponses) return query.getTimeLeft(); // Minimum responses have not been received yet + + if (!adaptiveTimeoutCalculated) { + // Recompute timeout when minimum responses have been received + long timeLeftMs = query.getTimeLeft(); + adaptiveTimeoutMin = (long) (timeLeftMs * config.minWaitAfterCoverageFactor()); + adaptiveTimeoutMax = (long) (timeLeftMs * config.maxWaitAfterCoverageFactor()); + adaptiveTimeoutCalculated = true; + } + long now = timer.milliTime(); + int pendingQueries = askedNodes - answeredNodes; + double missWidth = ((100.0 - minimumCoverage) * askedNodes) / 100.0 - 1.0; + double slopedWait = adaptiveTimeoutMin; + if (pendingQueries > 1 && missWidth > 0.0) { + slopedWait += ((adaptiveTimeoutMax - adaptiveTimeoutMin) * (pendingQueries - 1)) / missWidth; + } + long nextAdaptive = (long) slopedWait; + if (now + nextAdaptive >= deadline) { + return deadline - now; + } + deadline = now + nextAdaptive; + + return nextAdaptive; + } + + @Override + public int reason() { + return DEGRADED_BY_ADAPTIVE_TIMEOUT; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/CoverageAggregator.java b/container-search/src/main/java/com/yahoo/search/dispatch/CoverageAggregator.java new file mode 100644 index 00000000000..877412d2e9b --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/CoverageAggregator.java @@ -0,0 +1,97 @@ +package com.yahoo.search.dispatch; + +import com.yahoo.search.result.Coverage; + +import static com.yahoo.container.handler.Coverage.*; + +/** + * Aggregates coverage as responses are added. + * + * @author baldersheim + */ +public class CoverageAggregator { + private final int askedNodes; + private int answeredNodes = 0; + private int answeredNodesParticipated = 0; + private int failedNodes = 0; + private long answeredDocs = 0; + private long answeredActiveDocs = 0; + private long answeredTargetActiveDocs = 0; + private boolean timedOut = false; + private boolean degradedByMatchPhase = false; + CoverageAggregator(int askedNodes) { + this.askedNodes = askedNodes; + } + CoverageAggregator(CoverageAggregator rhs) { + askedNodes = rhs.askedNodes; + answeredNodes = rhs.answeredNodes; + answeredNodesParticipated = rhs.answeredNodesParticipated; + failedNodes = rhs.failedNodes; + answeredDocs = rhs.answeredDocs; + answeredActiveDocs = rhs.answeredActiveDocs; + answeredTargetActiveDocs = rhs.answeredTargetActiveDocs; + timedOut = rhs.timedOut; + degradedByMatchPhase = rhs.degradedByMatchPhase; + } + void add(Coverage source) { + answeredDocs += source.getDocs(); + answeredActiveDocs += source.getActive(); + answeredTargetActiveDocs += source.getTargetActive(); + answeredNodesParticipated += source.getNodes(); + answeredNodes++; + degradedByMatchPhase |= source.isDegradedByMatchPhase(); + timedOut |= source.isDegradedByTimeout(); + } + public int getAskedNodes() { + return askedNodes; + } + public int getAnsweredNodes() { + return answeredNodes; + } + public boolean hasNoAnswers() { return answeredNodes == 0; } + public void setTimedOut() { timedOut = true; } + public void setFailedNodes(int failedNodes) { + this.failedNodes = failedNodes; + } + + public Coverage createCoverage(TimeoutHandler timeoutHandler) { + Coverage coverage = new Coverage(answeredDocs, answeredActiveDocs, answeredNodesParticipated, 1); + coverage.setNodesTried(askedNodes); + coverage.setTargetActive(answeredTargetActiveDocs); + int degradedReason = 0; + if (timedOut) { + degradedReason |= timeoutHandler.reason(); + } + if (degradedByMatchPhase) { + degradedReason |= DEGRADED_BY_MATCH_PHASE; + } + coverage.setDegradedReason(degradedReason); + return coverage; + } + public CoverageAggregator adjustedDegradedCoverage(int searchableCopies, TimeoutHandler timeoutHandler) { + int askedAndFailed = askedNodes + failedNodes; + if (askedAndFailed == answeredNodesParticipated) { + return this; + } + int notAnswered = askedAndFailed - answeredNodesParticipated; + + if ((timeoutHandler.reason() == DEGRADED_BY_ADAPTIVE_TIMEOUT) && answeredNodesParticipated > 0) { + CoverageAggregator clone = new CoverageAggregator(this); + clone.answeredActiveDocs += (notAnswered * answeredActiveDocs / answeredNodesParticipated); + clone.answeredTargetActiveDocs += (notAnswered * answeredTargetActiveDocs / answeredNodesParticipated); + return clone; + } else { + if (askedAndFailed > answeredNodesParticipated) { + int missingNodes = notAnswered - (searchableCopies - 1); + if (answeredNodesParticipated > 0) { + CoverageAggregator clone = new CoverageAggregator(this); + clone.answeredActiveDocs += (missingNodes * answeredActiveDocs / answeredNodesParticipated); + clone.answeredTargetActiveDocs += (missingNodes * answeredTargetActiveDocs / answeredNodesParticipated); + clone.timedOut = true; + return clone; + } + } + } + return this; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java index 8c92e8b0270..dcf9b56aa33 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; +import com.yahoo.concurrent.Timer; import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.dispatch.searchcluster.Group; import com.yahoo.search.dispatch.searchcluster.SearchCluster; -import com.yahoo.search.result.Coverage; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; @@ -25,10 +25,6 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Logger; import java.util.stream.Collectors; -import static com.yahoo.container.handler.Coverage.DEGRADED_BY_ADAPTIVE_TIMEOUT; -import static com.yahoo.container.handler.Coverage.DEGRADED_BY_MATCH_PHASE; -import static com.yahoo.container.handler.Coverage.DEGRADED_BY_TIMEOUT; - /** * InterleavedSearchInvoker uses multiple {@link SearchInvoker} objects to interface with content * nodes in parallel. Operationally it first sends requests to all contained invokers and then @@ -40,38 +36,35 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM private static final Logger log = Logger.getLogger(InterleavedSearchInvoker.class.getName()); + private final Timer timer; private final Set<SearchInvoker> invokers; private final SearchCluster searchCluster; private final Group group; private final LinkedBlockingQueue<SearchInvoker> availableForProcessing; private final Set<Integer> alreadyFailedNodes; + private final CoverageAggregator coverageAggregator; private Query query; - private boolean adaptiveTimeoutCalculated = false; - private long adaptiveTimeoutMin = 0; - private long adaptiveTimeoutMax = 0; - private long deadline = 0; - - private long answeredDocs = 0; - private long answeredActiveDocs = 0; - private long answeredTargetActiveDocs = 0; - private int askedNodes = 0; - private int answeredNodes = 0; - private int answeredNodesParticipated = 0; - private boolean timedOut = false; - private boolean degradedByMatchPhase = false; - - public InterleavedSearchInvoker(Collection<SearchInvoker> invokers, + private TimeoutHandler timeoutHandler; + public InterleavedSearchInvoker(Timer timer, Collection<SearchInvoker> invokers, SearchCluster searchCluster, Group group, Set<Integer> alreadyFailedNodes) { super(Optional.empty()); + this.timer = timer; this.invokers = Collections.newSetFromMap(new IdentityHashMap<>()); this.invokers.addAll(invokers); this.searchCluster = searchCluster; this.group = group; this.availableForProcessing = newQueue(); this.alreadyFailedNodes = alreadyFailedNodes; + coverageAggregator = new CoverageAggregator(invokers.size()); + } + + private TimeoutHandler createTimeoutHandler(DispatchConfig config, int askedNodes, Query query) { + return (config.minSearchCoverage() < 100.0D) + ? new AdaptiveTimeoutHandler(timer, config, askedNodes, query) + : new SimpleTimeoutHandler(query); } /** @@ -83,7 +76,6 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM protected Object sendSearchRequest(Query query, Object unusedContext) throws IOException { this.query = query; invokers.forEach(invoker -> invoker.setMonitor(this)); - deadline = currentTime() + query.getTimeLeft(); int originalHits = query.getHits(); int originalOffset = query.getOffset(); @@ -101,8 +93,8 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM Object context = null; for (SearchInvoker invoker : invokers) { context = invoker.sendSearchRequest(query, context); - askedNodes++; } + timeoutHandler = createTimeoutHandler(searchCluster.dispatchConfig(), invokers.size(), query); query.setHits(originalHits); query.setOffset(originalOffset); @@ -119,14 +111,15 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM while (!invokers.isEmpty() && nextTimeout >= 0) { SearchInvoker invoker = availableForProcessing.poll(nextTimeout, TimeUnit.MILLISECONDS); if (invoker == null) { - log.fine(() -> "Search timed out with " + askedNodes + " requests made, " + answeredNodes + " responses received"); + log.fine(() -> "Search timed out with " + coverageAggregator.getAskedNodes() + " requests made, " + + coverageAggregator.getAnsweredNodes() + " responses received"); break; } else { InvokerResult toMerge = invoker.getSearchResult(execution); merged = mergeResult(result.getResult(), toMerge, merged, groupingResultAggregator); ejectInvoker(invoker); } - nextTimeout = nextTimeout(); + nextTimeout = timeoutHandler.nextTimeoutMS(coverageAggregator.getAnsweredNodes()); } } catch (InterruptedException e) { throw new RuntimeException("Interrupted while waiting for search results", e); @@ -134,7 +127,8 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM groupingResultAggregator.toAggregatedHit().ifPresent(h -> result.getResult().hits().add(h)); insertNetworkErrors(result.getResult()); - result.getResult().setCoverage(createCoverage()); + CoverageAggregator adjusted = coverageAggregator.adjustedDegradedCoverage((int)searchCluster.dispatchConfig().searchableCopies(), timeoutHandler); + result.getResult().setCoverage(adjusted.createCoverage(timeoutHandler)); int needed = query.getOffset() + query.getHits(); for (int index = query.getOffset(); (index < merged.size()) && (index < needed); index++) { @@ -146,7 +140,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM private void insertNetworkErrors(Result result) { // Network errors will be reported as errors only when all nodes fail, otherwise they are just traced - boolean asErrors = answeredNodes == 0; + boolean asErrors = coverageAggregator.hasNoAnswers(); if (!invokers.isEmpty()) { String keys = invokers.stream().map(SearchInvoker::distributionKey).map(dk -> dk.map(i -> i.toString()).orElse("(unspecified)")) @@ -158,7 +152,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM } else { query.trace("Backend communication timeout on nodes with distribution-keys: " + keys, 2); } - timedOut = true; + coverageAggregator.setTimedOut(); } if (alreadyFailedNodes != null) { var message = "Connection failure on nodes with distribution-keys: " @@ -168,51 +162,13 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM } else { query.trace(message, 2); } - int failed = alreadyFailedNodes.size(); - askedNodes += failed; - answeredNodes += failed; - } - } - - private long nextTimeout() { - DispatchConfig config = searchCluster.dispatchConfig(); - double minimumCoverage = config.minSearchCoverage(); - - if (askedNodes == answeredNodes || minimumCoverage >= 100.0) { - return query.getTimeLeft(); + coverageAggregator.setFailedNodes(alreadyFailedNodes.size()); } - int minimumResponses = (int) Math.ceil(askedNodes * minimumCoverage / 100.0); - - if (answeredNodes < minimumResponses) { - return query.getTimeLeft(); - } - - long timeLeft = query.getTimeLeft(); - if (!adaptiveTimeoutCalculated) { - adaptiveTimeoutMin = (long) (timeLeft * config.minWaitAfterCoverageFactor()); - adaptiveTimeoutMax = (long) (timeLeft * config.maxWaitAfterCoverageFactor()); - adaptiveTimeoutCalculated = true; - } - - long now = currentTime(); - int pendingQueries = askedNodes - answeredNodes; - double missWidth = ((100.0 - config.minSearchCoverage()) * askedNodes) / 100.0 - 1.0; - double slopedWait = adaptiveTimeoutMin; - if (pendingQueries > 1 && missWidth > 0.0) { - slopedWait += ((adaptiveTimeoutMax - adaptiveTimeoutMin) * (pendingQueries - 1)) / missWidth; - } - long nextAdaptive = (long) slopedWait; - if (now + nextAdaptive >= deadline) { - return deadline - now; - } - deadline = now + nextAdaptive; - - return nextAdaptive; } private List<LeanHit> mergeResult(Result result, InvokerResult partialResult, List<LeanHit> current, GroupingResultAggregator groupingResultAggregator) { - collectCoverage(partialResult.getResult().getCoverage(true)); + coverageAggregator.add(partialResult.getResult().getCoverage(true)); result.mergeWith(partialResult.getResult()); List<Hit> partialNonLean = partialResult.getResult().hits().asUnorderedHits(); @@ -265,55 +221,6 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM return merged; } - private void collectCoverage(Coverage source) { - answeredDocs += source.getDocs(); - answeredActiveDocs += source.getActive(); - answeredTargetActiveDocs += source.getTargetActive(); - answeredNodesParticipated += source.getNodes(); - answeredNodes++; - degradedByMatchPhase |= source.isDegradedByMatchPhase(); - timedOut |= source.isDegradedByTimeout(); - } - - private Coverage createCoverage() { - adjustDegradedCoverage(); - - Coverage coverage = new Coverage(answeredDocs, answeredActiveDocs, answeredNodesParticipated, 1); - coverage.setNodesTried(askedNodes); - coverage.setTargetActive(answeredTargetActiveDocs); - int degradedReason = 0; - if (timedOut) { - degradedReason |= (adaptiveTimeoutCalculated ? DEGRADED_BY_ADAPTIVE_TIMEOUT : DEGRADED_BY_TIMEOUT); - } - if (degradedByMatchPhase) { - degradedReason |= DEGRADED_BY_MATCH_PHASE; - } - coverage.setDegradedReason(degradedReason); - return coverage; - } - - private void adjustDegradedCoverage() { - if (askedNodes == answeredNodesParticipated) { - return; - } - int notAnswered = askedNodes - answeredNodesParticipated; - - if (adaptiveTimeoutCalculated && answeredNodesParticipated > 0) { - answeredActiveDocs += (notAnswered * answeredActiveDocs / answeredNodesParticipated); - answeredTargetActiveDocs += (notAnswered * answeredTargetActiveDocs / answeredNodesParticipated); - } else { - if (askedNodes > answeredNodesParticipated) { - int searchableCopies = (int) searchCluster.dispatchConfig().searchableCopies(); - int missingNodes = notAnswered - (searchableCopies - 1); - if (answeredNodesParticipated > 0) { - answeredActiveDocs += (missingNodes * answeredActiveDocs / answeredNodesParticipated); - answeredTargetActiveDocs += (missingNodes * answeredTargetActiveDocs / answeredNodesParticipated); - timedOut = true; - } - } - } - } - private void ejectInvoker(SearchInvoker invoker) { invokers.remove(invoker); invoker.release(); @@ -340,11 +247,6 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM } // For overriding in tests - protected long currentTime() { - return System.currentTimeMillis(); - } - - // For overriding in tests protected LinkedBlockingQueue<SearchInvoker> newQueue() { return new LinkedBlockingQueue<>(); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java index eb379a51ed4..02cf11c9fe7 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; +import com.yahoo.concurrent.Timer; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -89,7 +90,7 @@ public abstract class InvokerFactory { if (invokers.size() == 1 && failed == null) { return Optional.of(invokers.get(0)); } else { - return Optional.of(new InterleavedSearchInvoker(invokers, searchCluster, group, failed)); + return Optional.of(new InterleavedSearchInvoker(Timer.monotonic, invokers, searchCluster, group, failed)); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SimpleTimeoutHandler.java b/container-search/src/main/java/com/yahoo/search/dispatch/SimpleTimeoutHandler.java new file mode 100644 index 00000000000..90b6bf87a0b --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SimpleTimeoutHandler.java @@ -0,0 +1,26 @@ +package com.yahoo.search.dispatch; + +import com.yahoo.search.Query; + +import static com.yahoo.container.handler.Coverage.DEGRADED_BY_TIMEOUT; + +/** + * Computes the timeout based solely on the query timeout + * + * @author baldersheim + */ +public class SimpleTimeoutHandler implements TimeoutHandler { + private final Query query; + SimpleTimeoutHandler(Query query) { + this.query = query; + } + @Override + public long nextTimeoutMS(int answeredNodes) { + return query.getTimeLeft(); + } + + @Override + public int reason() { + return DEGRADED_BY_TIMEOUT; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/TimeoutHandler.java b/container-search/src/main/java/com/yahoo/search/dispatch/TimeoutHandler.java new file mode 100644 index 00000000000..a8ac0d14ddc --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/TimeoutHandler.java @@ -0,0 +1,15 @@ +package com.yahoo.search.dispatch; + +/** + * Computes next timeout in milliseconds + * + * @author baldersheim + */ +public interface TimeoutHandler { + long nextTimeoutMS(int answeredNodes); + + /** + * Return a bitmask from com.yahoo.container.handler.Coverage.DEGRADED.... set + */ + int reason(); +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 82d3d98d9ef..25617e48aa1 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -307,6 +307,7 @@ public class SearchCluster implements NodeManager<Node> { if (fullCoverage) { log.info("Cluster " + clusterId + ": " + group + " has full coverage. " + "Active documents: " + group.activeDocuments() + "/" + medianDocuments + ", " + + "Target active documents: " + group.targetActiveDocuments() + ", " + "working nodes: " + group.workingNodes() + "/" + group.nodes().size()); } else { StringBuilder unresponsive = new StringBuilder(); @@ -316,6 +317,7 @@ public class SearchCluster implements NodeManager<Node> { } log.warning("Cluster " + clusterId + ": " + group + " has reduced coverage: " + "Active documents: " + group.activeDocuments() + "/" + medianDocuments + ", " + + "Target active documents: " + group.targetActiveDocuments() + ", " + "working nodes: " + group.workingNodes() + "/" + group.nodes().size() + ", unresponsive nodes: " + (unresponsive.toString().isEmpty() ? " none" : unresponsive)); } diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java index bd72ed9fd08..d374bfdeb7b 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java @@ -7,6 +7,7 @@ import com.google.common.collect.ImmutableList; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.Chain; import com.yahoo.container.QrSearchersConfig; +import com.yahoo.data.access.simple.Value; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; @@ -40,7 +41,7 @@ public class JuniperSearcherTestCase { * @param sdName the search definition type of the returned hit * @param content the content of the "dynteaser" field of the returned hit */ - private Chain<Searcher> createSearchChain(String sdName, String content) { + private Chain<Searcher> createSearchChain(String sdName, Object content) { JuniperSearcher searcher = new JuniperSearcher(new ComponentId("test"), new QrSearchersConfig(new QrSearchersConfig.Builder())); @@ -50,7 +51,7 @@ public class JuniperSearcherTestCase { return new Chain<>(searcher, docsource); } - private void addResult(Query query, String sdName, String content, DocumentSourceSearcher docsource) { + private void addResult(Query query, String sdName, Object content, DocumentSourceSearcher docsource) { Result r = new Result(query); FastHit hit = new FastHit(); hit.setId("http://abc.html"); @@ -62,7 +63,7 @@ public class JuniperSearcherTestCase { } /** Creates a result of the search definiton "one" */ - private Result createResult(String content) { + private Result createResult(Object content) { return createResult("one", content, true); } @@ -70,7 +71,7 @@ public class JuniperSearcherTestCase { return createResult(sdName, content, true); } - private Result createResult(String sdName, String content, boolean bolding) { + private Result createResult(String sdName, Object content, boolean bolding) { Chain<Searcher> chain = createSearchChain(sdName, content); Query query = new Query("?query=12"); if ( ! bolding) @@ -123,6 +124,17 @@ public class JuniperSearcherTestCase { } @Test + void test_field_rewriting_for_array_of_string_field() { + var content = new Value.ArrayValue() + .add("\u001Faaa\u001F\u001Ebbb\u001Fccc\u001Fddd") + .add("\u001Feee\u001F\u001Efff\u001Fggg\u001Fhhh"); + Result check = createResult(content); + assertEquals(1, check.getHitCount()); + assertEquals("[\"<hi>aaa</hi><sep />bbb<hi>ccc</hi>ddd\",\"<hi>eee</hi><sep />fff<hi>ggg</hi>hhh\"]", + check.hits().get(0).getField("dynteaser").toString()); + } + + @Test void testNoRewritingDueToSearchDefinition() { Result check = createResult("two", "\u001FXYZ\u001F\u001EQWE\u001FJKL\u001FASD&"); assertEquals(1, check.getHitCount()); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java index a88046197e0..7fe2d2c2d9d 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; +import com.yahoo.concurrent.Timer; import com.yahoo.document.GlobalId; import com.yahoo.document.idstring.IdString; import com.yahoo.prelude.fastsearch.FastHit; @@ -23,6 +24,7 @@ import com.yahoo.test.ManualClock; import org.junit.jupiter.api.Test; import java.io.IOException; +import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -33,6 +35,7 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +import java.util.function.ToLongFunction; import java.util.stream.StreamSupport; import static com.yahoo.container.handler.Coverage.DEGRADED_BY_MATCH_PHASE; @@ -347,7 +350,7 @@ public class InterleavedSearchInvokerTest { .addAggregationResult(new MinAggregationResult().setMin(new IntegerResultNode(6)).setTag(3)))); invokers.add(new MockInvoker(0).setHits(List.of(new GroupingListHit(List.of(grouping2))))); - InterleavedSearchInvoker invoker = new InterleavedSearchInvoker(invokers, cluster, new Group(0, List.of()), Collections.emptySet()); + InterleavedSearchInvoker invoker = new InterleavedSearchInvoker(Timer.monotonic, invokers, cluster, new Group(0, List.of()), Collections.emptySet()); invoker.responseAvailable(invokers.get(0)); invoker.responseAvailable(invokers.get(1)); Result result = invoker.search(query, null); @@ -360,7 +363,7 @@ public class InterleavedSearchInvokerTest { List<SearchInvoker> invokers = new ArrayList<>(); invokers.add(createInvoker(a, 0)); invokers.add(createInvoker(b, 1)); - InterleavedSearchInvoker invoker = new InterleavedSearchInvoker(invokers, cluster, group, Collections.emptySet()); + InterleavedSearchInvoker invoker = new InterleavedSearchInvoker(Timer.monotonic, invokers, cluster, group, Collections.emptySet()); invoker.responseAvailable(invokers.get(0)); invoker.responseAvailable(invokers.get(1)); return invoker; @@ -411,12 +414,7 @@ public class InterleavedSearchInvokerTest { invokers.add(new MockInvoker(i)); } - return new InterleavedSearchInvoker(invokers, searchCluster, group,null) { - - @Override - protected long currentTime() { - return clock.millis(); - } + return new InterleavedSearchInvoker(Timer.wrap(clock), invokers, searchCluster, group,null) { @Override protected LinkedBlockingQueue<SearchInvoker> newQueue() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java index 0c9ff7863bd..5b20c57fcca 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackage.java @@ -25,6 +25,8 @@ import com.yahoo.yolean.Exceptions; import javax.security.auth.x500.X500Principal; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; import java.math.BigInteger; import java.security.KeyPair; import java.security.cert.X509Certificate; @@ -81,8 +83,9 @@ public class TestPackage { entries.put("tests/.ignore-" + UUID.randomUUID(), new byte[0]); entries.put(servicesFile, - servicesXml( ! isPublicSystem, + servicesXml(! isPublicSystem, certificateValidFrom != null, + hasLegacyTests(testPackage), testerResourcesFor(id.type().zone(), spec.requireInstance(id.application().instance())), testerApp)); @@ -114,6 +117,12 @@ public class TestPackage { this.applicationPackage = new ApplicationPackage(buffer.toByteArray()); } + static boolean hasLegacyTests(byte[] testPackage) { + return ZipEntries.from(testPackage, __ -> true, 0, false).asList().stream() + .anyMatch(file -> file.name().startsWith("artifacts/") && file.name().endsWith("-tests.jar")); + + } + public ApplicationPackage asApplicationPackage() { return applicationPackage; } @@ -210,9 +219,9 @@ public class TestPackage { } /** Returns the generated services.xml content for the tester application. */ - public static byte[] servicesXml(boolean systemUsesAthenz, boolean useTesterCertificate, + public static byte[] servicesXml(boolean systemUsesAthenz, boolean useTesterCertificate, boolean hasLegacyTests, NodeResources resources, ControllerConfig.Steprunner.Testerapp config) { - int jdiscMemoryGb = 2; // 2Gb memory for tester application (excessive?). + int jdiscMemoryGb = 2; // 2Gb memory for tester application which uses Maven. int jdiscMemoryPct = (int) Math.ceil(100 * jdiscMemoryGb / resources.memoryGb()); // Of the remaining memory, split 50/50 between Surefire running the tests and the rest @@ -260,7 +269,7 @@ public class TestPackage { " </component>\n" + "\n" + " <nodes count=\"1\">\n" + - " <jvm allocated-memory=\"" + jdiscMemoryPct + "%\"/>\n" + + (hasLegacyTests ? " <jvm allocated-memory=\"" + jdiscMemoryPct + "%\"/>\n" : "" ) + " " + resourceString + "\n" + " </nodes>\n" + " </container>\n" + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java index 8392a77bad5..6bbcd551924 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ZipEntries.java @@ -67,6 +67,7 @@ public class ZipEntries { /** Read ZIP entries from inputStream */ public static ZipEntries from(byte[] zip, Predicate<String> entryNameMatcher, int maxEntrySizeInBytes, boolean throwIfEntryExceedsMaxSize) { + Options options = Options.standard() .pathPredicate(entryNameMatcher) .maxSize(2 * (long) Math.pow(1024, 3)) // 2 GB diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index 6a3a44c0140..52eeaae1297 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -542,7 +542,10 @@ public class DeploymentStatus { existingRevision = Optional.of(target.targetRevision()); } List<Job> toRun = new ArrayList<>(); - List<Change> changes = deployingCompatibilityChange ? List.of(change) : changes(job, step, change); + List<Change> changes = deployingCompatibilityChange + || allJobs.get(job).flatMap(status -> status.lastCompleted()).isEmpty() + ? List.of(change) + : changes(job, step, change); for (Change partial : changes) { Job jobToRun = new Job(job.type(), Versions.from(partial, application, existingPlatform, existingRevision, fallbackPlatform(partial, job)), @@ -656,6 +659,7 @@ public class DeploymentStatus { /** The test jobs that need to run prior to the given production deployment jobs. */ public Map<JobId, List<Job>> testJobs(Map<JobId, List<Job>> jobs) { Map<JobId, List<Job>> testJobs = new LinkedHashMap<>(); + // First, look for a declared test in the instance of each production job. jobs.forEach((job, versionsList) -> { for (JobType testType : List.of(systemTest(job.type()), stagingTest(job.type()))) { if (job.type().isProduction() && job.type().isDeployment()) { @@ -673,6 +677,7 @@ public class DeploymentStatus { } } }); + // If no declared test in the right instance was triggered, pick one from a different instance. jobs.forEach((job, versionsList) -> { for (JobType testType : List.of(systemTest(job.type()), stagingTest(job.type()))) { for (Job productionJob : versionsList) @@ -680,7 +685,8 @@ public class DeploymentStatus { && allJobs.successOn(testType, productionJob.versions()).asList().isEmpty() && testJobs.keySet().stream() .noneMatch(test -> test.type().equals(testType) && test.type().zone().equals(testType.zone()) - && testJobs.get(test).stream().anyMatch(testJob -> testJob.versions().equals(productionJob.versions())))) { + && testJobs.get(test).stream().anyMatch(testJob -> test.type().isSystemTest() ? testJob.versions().targetsMatch(productionJob.versions()) + : testJob.versions().equals(productionJob.versions())))) { JobId testJob = firstDeclaredOrElseImplicitTest(testType); testJobs.merge(testJob, List.of(new Job(testJob.type(), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index 85588d2cf0f..d1f6ddc1644 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -120,7 +120,8 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { Map<ZoneId, Double> deploymentCosts = snapshotsByInstance.getOrDefault(instanceName, List.of()).stream() .collect(Collectors.toUnmodifiableMap( ResourceSnapshot::getZoneId, - snapshot -> cost(snapshot.allocation(), systemName))); + snapshot -> cost(snapshot.allocation(), systemName), + Double::sum)); locked = locked.with(instanceName, i -> i.withDeploymentCosts(deploymentCosts)); updateCostMetrics(tenantAndApplication.instance(instanceName), deploymentCosts); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 1f9a9b07f71..341cba60519 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -1028,6 +1028,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { var awsRegion = request.getProperty("aws-region"); var parameterName = request.getProperty("parameter-name"); var applicationId = ApplicationId.fromFullString(request.getProperty("application-id")); + if (!applicationId.tenant().equals(TenantName.from(tenantName))) + return ErrorResponse.badRequest("Invalid application id"); var zoneId = requireZone(ZoneId.from(request.getProperty("zone"))); var deploymentId = new DeploymentId(applicationId, zoneId); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java index 5bc1c386134..bff0ccc8ae1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/TestPackageTest.java @@ -147,8 +147,17 @@ public class TestPackageTest { @Test void generates_correct_services_xml() throws IOException { assertEquals(Files.readString(Paths.get("src/test/resources/test_runner_services.xml-cd")), + new String(TestPackage.servicesXml(true, + false, + false, + new NodeResources(2, 12, 75, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local), + new ControllerConfig.Steprunner.Testerapp.Builder().build()), + UTF_8)); + + assertEquals(Files.readString(Paths.get("src/test/resources/test_runner_services_with_legacy_tests.xml-cd")), new String(TestPackage.servicesXml(true, false, + true, new NodeResources(2, 12, 75, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local), new ControllerConfig.Steprunner.Testerapp.Builder().build()), UTF_8)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index b533a0c7797..89b6f6ca606 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -63,7 +63,6 @@ import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.tes import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testUsWest1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.ALL; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.PLATFORM; -import static java.util.Collections.copy; import static java.util.Collections.emptyList; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -2563,12 +2562,18 @@ public class DeploymentTriggerTest { assertEquals(Set.of(), tests.deploymentStatus().jobsToRun().keySet()); } - @Test void testInstancesWithMultipleClouds() { String spec = """ <deployment> <parallel> + <instance id='separate'> + <test /> + <staging /> + <prod> + <region>alpha-centauri</region> + </prod> + </instance> <instance id='independent'> <test /> </instance> @@ -2599,10 +2604,9 @@ public class DeploymentTriggerTest { </prod> </instance> </steps> - <instance id='separate'> - <staging /> + <instance id='dependent'> <prod> - <region>alpha-centauri</region> + <region>us-east-3</region> </prod> </instance> </parallel> @@ -2620,17 +2624,18 @@ public class DeploymentTriggerTest { tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.notController()); ApplicationPackage appPackage = ApplicationPackageBuilder.fromDeploymentXml(spec); - DeploymentContext app = tester.newDeploymentContext("tenant", "application", "alpha").submit(appPackage); + DeploymentContext app = tester.newDeploymentContext("tenant", "application", "alpha").submit(appPackage).deploy(); + app.submit(appPackage); Map<JobId, List<DeploymentStatus.Job>> jobs = app.deploymentStatus().jobsToRun(); JobType centauriTest = JobType.systemTest(tester.controller().zoneRegistry(), CloudName.from("centauri")); JobType centauriStaging = JobType.stagingTest(tester.controller().zoneRegistry(), CloudName.from("centauri")); + assertQueued("separate", jobs, centauriTest); + assertQueued("separate", jobs, stagingTest, centauriStaging); assertQueued("independent", jobs, systemTest, centauriTest); assertQueued("alpha", jobs, systemTest); assertQueued("beta", jobs, centauriTest); assertQueued("gamma", jobs, centauriTest); - assertQueued("nu", jobs, stagingTest); - assertQueued("separate", jobs, centauriStaging); // Once alpha runs its default system test, it also runs the centauri system test, as omega depends on it. app.runJob(systemTest); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index cfe25232408..af0a85f1a90 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -323,6 +323,15 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { request("/application/v4/tenant/scoober/secret-store/secret-foo/validate?aws-region=us-west-1¶meter-name=foo&application-id=scoober.albums.default&zone=prod.aws-us-east-1c", GET) .roles(Set.of(Role.developer(tenantName))); tester.assertResponse(secretStoreRequest, "{\"target\":\"scoober.albums in prod.aws-us-east-1c\",\"result\":{\"settings\":{\"name\":\"foo\",\"role\":\"vespa-secretstore-access\",\"awsId\":\"892075328880\",\"externalId\":\"*****\",\"region\":\"us-east-1\"},\"status\":\"ok\"}}", 200); + + secretStoreRequest = + request("/application/v4/tenant/scoober/secret-store/secret-foo/validate?aws-region=us-west-1¶meter-name=foo&application-id=scober.albums.default&zone=prod.aws-us-east-1c", GET) + .roles(Set.of(Role.developer(tenantName))); + tester.assertResponse(secretStoreRequest, "{" + + "\"error-code\":\"BAD_REQUEST\"," + + "\"message\":\"Invalid application id\"" + + "}", 400); + } @Test diff --git a/controller-server/src/test/resources/test_runner_services.xml-cd b/controller-server/src/test/resources/test_runner_services.xml-cd index 526fd12965b..4bf3a78801d 100644 --- a/controller-server/src/test/resources/test_runner_services.xml-cd +++ b/controller-server/src/test/resources/test_runner_services.xml-cd @@ -33,7 +33,6 @@ </component> <nodes count="1"> - <jvm allocated-memory="17%"/> <resources vcpu="2.00" memory="12.00Gb" disk="75.00Gb" disk-speed="fast" storage-type="local"/> </nodes> </container> diff --git a/controller-server/src/test/resources/test_runner_services_with_legacy_tests.xml-cd b/controller-server/src/test/resources/test_runner_services_with_legacy_tests.xml-cd new file mode 100644 index 00000000000..526fd12965b --- /dev/null +++ b/controller-server/src/test/resources/test_runner_services_with_legacy_tests.xml-cd @@ -0,0 +1,40 @@ +<?xml version='1.0' encoding='UTF-8'?> +<services xmlns:deploy='vespa' version='1.0'> + <container version='1.0' id='tester'> + + <component id="com.yahoo.vespa.hosted.testrunner.TestRunner" bundle="vespa-testrunner-components"> + <config name="com.yahoo.vespa.hosted.testrunner.test-runner"> + <artifactsPath>artifacts</artifactsPath> + <surefireMemoryMb>5120</surefireMemoryMb> + <useAthenzCredentials>true</useAthenzCredentials> + <useTesterCertificate>false</useTesterCertificate> + </config> + </component> + + <handler id="com.yahoo.vespa.testrunner.TestRunnerHandler" bundle="vespa-osgi-testrunner"> + <binding>http://*/tester/v1/*</binding> + </handler> + + <component id="ai.vespa.hosted.cd.cloud.impl.VespaTestRuntimeProvider" bundle="cloud-tenant-cd" /> + + <component id="com.yahoo.vespa.testrunner.JunitRunner" bundle="vespa-osgi-testrunner"> + <config name="com.yahoo.vespa.testrunner.junit-test-runner"> + <artifactsPath>artifacts</artifactsPath> + <useAthenzCredentials>true</useAthenzCredentials> + </config> + </component> + + <component id="com.yahoo.vespa.testrunner.VespaCliTestRunner" bundle="vespa-osgi-testrunner"> + <config name="com.yahoo.vespa.testrunner.vespa-cli-test-runner"> + <artifactsPath>artifacts</artifactsPath> + <testsPath>tests</testsPath> + <useAthenzCredentials>true</useAthenzCredentials> + </config> + </component> + + <nodes count="1"> + <jvm allocated-memory="17%"/> + <resources vcpu="2.00" memory="12.00Gb" disk="75.00Gb" disk-speed="fast" storage-type="local"/> + </nodes> + </container> +</services> diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/ScheduledEventQueueTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/ScheduledEventQueueTestCase.java index 4f937aa26ba..ad73aa92d7c 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/ScheduledEventQueueTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/ScheduledEventQueueTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi.messagebus; -import com.yahoo.concurrent.Timer; +import com.yahoo.concurrent.ManualTimer; import org.junit.Test; import java.util.concurrent.RejectedExecutionException; @@ -13,7 +13,7 @@ import static org.junit.Assert.fail; public class ScheduledEventQueueTestCase { - class TestTask implements Runnable { + static class TestTask implements Runnable { public long timestamp = 0; public void run() { @@ -46,42 +46,34 @@ public class ScheduledEventQueueTestCase { assertNull(queue.popTask()); } - class TestTimer implements Timer { - public long milliTime = 0; - - public long milliTime() { - return milliTime; - } - } - @Test public void testPushTaskWithTime() { - TestTimer timer = new TestTimer(); + ManualTimer timer = new ManualTimer(); ScheduledEventQueue queue = new ScheduledEventQueue(timer); TestTask task = new TestTask(); queue.pushTask(task, 1000); assertNull(queue.popTask()); - timer.milliTime = 1000; + timer.set(1000); assertEquals(task, queue.popTask()); } @Test public void testTwoTasksWithSameTime() { - TestTimer timer = new TestTimer(); + ManualTimer timer = new ManualTimer(); ScheduledEventQueue queue = new ScheduledEventQueue(timer); TestTask task1 = new TestTask(); queue.pushTask(task1, 1000); TestTask task2 = new TestTask(); queue.pushTask(task2, 1000); assertNull(queue.popTask()); - timer.milliTime = 1000; + timer.set(1000); assertEquals(task1, queue.popTask()); assertEquals(task2, queue.popTask()); } @Test public void testThreeTasksWithDifferentTime() { - TestTimer timer = new TestTimer(); + ManualTimer timer = new ManualTimer(); ScheduledEventQueue queue = new ScheduledEventQueue(timer); TestTask task1 = new TestTask(); queue.pushTask(task1, 1000); @@ -91,17 +83,17 @@ public class ScheduledEventQueueTestCase { queue.pushTask(task3); assertEquals(task3, queue.popTask()); assertNull(queue.popTask()); - timer.milliTime = 1000; + timer.set(1000); assertEquals(task2, queue.popTask()); assertEquals(task1, queue.popTask()); } - class ClockSetterThread implements Runnable { + static class ClockSetterThread implements Runnable { ScheduledEventQueue queue; - TestTimer timer; + ManualTimer timer; long newTime; - public ClockSetterThread(ScheduledEventQueue queue, TestTimer timer, long newTime) { + public ClockSetterThread(ScheduledEventQueue queue, ManualTimer timer, long newTime) { this.queue = queue; this.timer = timer; this.newTime = newTime; @@ -114,14 +106,14 @@ public class ScheduledEventQueueTestCase { } } catch (InterruptedException e) { } - timer.milliTime = newTime; + timer.set(newTime); queue.wakeTasks(); } } @Test public void testPushAndWaitForTask() { - TestTimer timer = new TestTimer(); + ManualTimer timer = new ManualTimer(); ScheduledEventQueue queue = new ScheduledEventQueue(timer); TestTask task = new TestTask(); queue.pushTask(task, 50); @@ -131,7 +123,7 @@ public class ScheduledEventQueueTestCase { assertEquals(50, timer.milliTime()); } - class TaskPusherThread implements Runnable { + static class TaskPusherThread implements Runnable { ScheduledEventQueue queue; TestTask task; @@ -162,7 +154,7 @@ public class ScheduledEventQueueTestCase { @Test public void testPushAndWaitMultiple() { - TestTimer timer = new TestTimer(); + ManualTimer timer = new ManualTimer(); ScheduledEventQueue queue = new ScheduledEventQueue(timer); TestTask lastTask = new TestTask(); queue.pushTask(lastTask, 250); @@ -188,11 +180,11 @@ public class ScheduledEventQueueTestCase { } } - class ShutdownThread implements Runnable { + static class ShutdownThread implements Runnable { ScheduledEventQueue queue; - TestTimer timer; + ManualTimer timer; - public ShutdownThread(ScheduledEventQueue queue, TestTimer timer) { + public ShutdownThread(ScheduledEventQueue queue, ManualTimer timer) { this.queue = queue; this.timer = timer; } @@ -205,14 +197,14 @@ public class ScheduledEventQueueTestCase { } catch (InterruptedException e) { } queue.shutdown(); - timer.milliTime = 100; + timer.set(100); queue.wakeTasks(); } } @Test public void testShutdownInGetNext() { - TestTimer timer = new TestTimer(); + ManualTimer timer = new ManualTimer(); ScheduledEventQueue queue = new ScheduledEventQueue(timer); TestTask task = new TestTask(); queue.pushTask(task, 100); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index b3257ec0052..12bf24eb737 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -515,6 +515,14 @@ public class Flags { "Lock scheme for tenant-related controller locks (valid values: OLD, BOTH, NEW)", "Takes effect immediately"); + public static final UnboundBooleanFlag COMPUTE_COVERAGE_FROM_TARGET_ACTIVE_DOCS = defineFeatureFlag( + "compute-coverage-from-target-active-docs", false, + List.of("baldersheim", "vekterli"), "2022-09-13", "2022-11-01", + "Whether degraded coverage computation will take target active docs into " + + "account, not just currently active docs", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/messagebus/src/test/java/com/yahoo/messagebus/CustomTimer.java b/messagebus/src/test/java/com/yahoo/messagebus/CustomTimer.java deleted file mode 100644 index 539897d6485..00000000000 --- a/messagebus/src/test/java/com/yahoo/messagebus/CustomTimer.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.messagebus; - -import com.yahoo.concurrent.Timer; - -/** - * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> - */ -class CustomTimer implements Timer { - - long millis = 0; - - @Override - public long milliTime() { - return millis; - } -} diff --git a/messagebus/src/test/java/com/yahoo/messagebus/DynamicThrottlePolicyTest.java b/messagebus/src/test/java/com/yahoo/messagebus/DynamicThrottlePolicyTest.java index 8f9ed2323d7..8414f6588ea 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/DynamicThrottlePolicyTest.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/DynamicThrottlePolicyTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.messagebus; +import com.yahoo.concurrent.ManualTimer; import com.yahoo.messagebus.test.SimpleMessage; import com.yahoo.messagebus.test.SimpleReply; import org.junit.jupiter.api.Test; @@ -46,7 +47,7 @@ public class DynamicThrottlePolicyTest { { // This setup is lucky with the artificial local maxima for latency, and gives good results. See below for counter-examples. int workPerSuccess = 8; - CustomTimer timer = new CustomTimer(); + ManualTimer timer = new ManualTimer(); DynamicThrottlePolicy policy = new DynamicThrottlePolicy(timer).setMinWindowSize(1) .setWindowSizeIncrement(0.1) .setResizeRate(100); @@ -64,7 +65,7 @@ public class DynamicThrottlePolicyTest { { // This setup is not so lucky, and the artificial behaviour pushes it into overload. int workPerSuccess = 5; - CustomTimer timer = new CustomTimer(); + ManualTimer timer = new ManualTimer(); DynamicThrottlePolicy policy = new DynamicThrottlePolicy(timer).setMinWindowSize(1) .setWindowSizeIncrement(0.1) .setResizeRate(100); @@ -80,7 +81,7 @@ public class DynamicThrottlePolicyTest { { // This setup is not so lucky either, and the artificial behaviour keeps it far below a good throughput. int workPerSuccess = 4; - CustomTimer timer = new CustomTimer(); + ManualTimer timer = new ManualTimer(); DynamicThrottlePolicy policy = new DynamicThrottlePolicy(timer).setMinWindowSize(1) .setWindowSizeIncrement(0.1) .setResizeRate(100); @@ -98,7 +99,7 @@ public class DynamicThrottlePolicyTest { @Test void singlePolicySingleWorkerWithIncreasingParallelism() { for (int exponent = 0; exponent < 4; exponent++) { - CustomTimer timer = new CustomTimer(); + ManualTimer timer = new ManualTimer(); DynamicThrottlePolicy policy = new DynamicThrottlePolicy(timer); int scaleFactor = (int) Math.pow(10, exponent); long operations = 3_000L * scaleFactor; @@ -121,7 +122,7 @@ public class DynamicThrottlePolicyTest { @Test void singlePolicyIncreasingWorkersWithNoParallelism() { for (int exponent = 0; exponent < 4; exponent++) { - CustomTimer timer = new CustomTimer(); + ManualTimer timer = new ManualTimer(); DynamicThrottlePolicy policy = new DynamicThrottlePolicy(timer); int scaleFactor = (int) Math.pow(10, exponent); long operations = 2_000L * scaleFactor; @@ -156,7 +157,7 @@ public class DynamicThrottlePolicyTest { int numberOfWorkers = 1 + (int) (10 * Math.random()); int maximumTasksPerWorker = 100_000; int workerParallelism = 32; - CustomTimer timer = new CustomTimer(); + ManualTimer timer = new ManualTimer(); DynamicThrottlePolicy policy1 = new DynamicThrottlePolicy(timer); DynamicThrottlePolicy policy2 = new DynamicThrottlePolicy(timer).setWeight(0.5); Summary summary = run(operations, workPerSuccess, numberOfWorkers, maximumTasksPerWorker, workerParallelism, timer, policy1, policy2); @@ -180,7 +181,7 @@ public class DynamicThrottlePolicyTest { int numberOfWorkers = 6; int maximumTasksPerWorker = 180 + (int) (120 * Math.random()); int workerParallelism = 60 + (int) (40 * Math.random()); - CustomTimer timer = new CustomTimer(); + ManualTimer timer = new ManualTimer(); int p = 10; DynamicThrottlePolicy[] policies = IntStream.range(0, p) .mapToObj(j -> new DynamicThrottlePolicy(timer) @@ -213,7 +214,7 @@ public class DynamicThrottlePolicyTest { } private Summary run(long operations, int workPerSuccess, int numberOfWorkers, int maximumTasksPerWorker, - int workerParallelism, CustomTimer timer, DynamicThrottlePolicy... policies) { + int workerParallelism, ManualTimer timer, DynamicThrottlePolicy... policies) { System.err.printf("\n### Running %d operations of %d ticks each against %d workers with parallelism %d and queue size %d\n", operations, workPerSuccess, numberOfWorkers, workerParallelism, maximumTasksPerWorker); @@ -250,7 +251,7 @@ public class DynamicThrottlePolicyTest { ++ticks; totalPending += resource.pending(); resource.tick(); - ++timer.millis; + timer.advance(1); } for (int i = 0; i < windows.length; i++) diff --git a/messagebus/src/test/java/com/yahoo/messagebus/RateThrottlingTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/RateThrottlingTestCase.java index 5e579f5d622..b4ca923c4ff 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/RateThrottlingTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/RateThrottlingTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.messagebus; +import com.yahoo.concurrent.ManualTimer; import com.yahoo.messagebus.test.SimpleMessage; import org.junit.jupiter.api.Test; @@ -11,7 +12,7 @@ public class RateThrottlingTestCase { @Test void testPending() { - CustomTimer timer = new CustomTimer(); + ManualTimer timer = new ManualTimer(); RateThrottlingPolicy policy = new RateThrottlingPolicy(5.0, timer); policy.setMaxPendingCount(200); @@ -20,7 +21,7 @@ public class RateThrottlingTestCase { } public int getActualRate(double desiredRate) { - CustomTimer timer = new CustomTimer(); + ManualTimer timer = new ManualTimer(); RateThrottlingPolicy policy = new RateThrottlingPolicy(desiredRate, timer); int ok = 0; @@ -28,7 +29,7 @@ public class RateThrottlingTestCase { if (policy.canSend(new SimpleMessage("test"), 0)) { ok++; } - timer.millis += 10; + timer.advance(10); } return ok; diff --git a/messagebus/src/test/java/com/yahoo/messagebus/ThrottlerTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/ThrottlerTestCase.java index 045dc1177db..2e50d561778 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/ThrottlerTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/ThrottlerTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.messagebus; +import com.yahoo.concurrent.ManualTimer; import com.yahoo.jrt.ListenFailedException; import com.yahoo.jrt.slobrok.server.Slobrok; import com.yahoo.messagebus.network.rpc.test.TestServer; @@ -14,7 +15,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.util.Arrays; +import java.util.List; import static org.junit.jupiter.api.Assertions.*; @@ -29,8 +30,8 @@ public class ThrottlerTestCase { @BeforeEach public void setUp() throws ListenFailedException { RoutingTableSpec table = new RoutingTableSpec(SimpleProtocol.NAME); - table.addHop("dst", "test/dst/session", Arrays.asList("test/dst/session")); - table.addRoute("test", Arrays.asList("dst")); + table.addHop("dst", "test/dst/session", List.of("test/dst/session")); + table.addRoute("test", List.of("dst")); slobrok = new Slobrok(); src = new TestServer("test/src", table, slobrok, null); dst = new TestServer("test/dst", table, slobrok, null); @@ -127,7 +128,7 @@ public class ThrottlerTestCase { @Test void testDynamicWindowSize() { - CustomTimer timer = new CustomTimer(); + ManualTimer timer = new ManualTimer(); DynamicThrottlePolicy policy = new DynamicThrottlePolicy(timer); policy.setWindowSizeIncrement(5) @@ -151,7 +152,7 @@ public class ThrottlerTestCase { @Test void testIdleTimePeriod() { - CustomTimer timer = new CustomTimer(); + ManualTimer timer = new ManualTimer(); DynamicThrottlePolicy policy = new DynamicThrottlePolicy(timer); policy.setWindowSizeIncrement(5) @@ -162,15 +163,15 @@ public class ThrottlerTestCase { assertTrue(windowSize >= 90 && windowSize <= 110); Message msg = new SimpleMessage("foo"); - timer.millis += 30 * 1000; + timer.advance(30 * 1000); assertTrue(policy.canSend(msg, 0)); assertTrue(windowSize >= 90 && windowSize <= 110); - timer.millis += 60 * 1000 + 1; + timer.advance(60 * 1000 + 1); assertTrue(policy.canSend(msg, 50)); assertEquals(55, policy.getMaxPendingCount()); - timer.millis += 60 * 1000 + 1; + timer.advance(60 * 1000 + 1); assertTrue(policy.canSend(msg, 0)); assertEquals(5, policy.getMaxPendingCount()); @@ -178,7 +179,7 @@ public class ThrottlerTestCase { @Test void testMinWindowSize() { - CustomTimer timer = new CustomTimer(); + ManualTimer timer = new ManualTimer(); DynamicThrottlePolicy policy = new DynamicThrottlePolicy(timer); policy.setWindowSizeIncrement(5) @@ -191,7 +192,7 @@ public class ThrottlerTestCase { @Test void testMaxWindowSize() { - CustomTimer timer = new CustomTimer(); + ManualTimer timer = new ManualTimer(); DynamicThrottlePolicy policy = new DynamicThrottlePolicy(timer); policy.setWindowSizeIncrement(5); @@ -202,7 +203,7 @@ public class ThrottlerTestCase { assertTrue(windowSize >= 40 && windowSize <= 50); } - private int getWindowSize(DynamicThrottlePolicy policy, CustomTimer timer, int maxPending) { + private int getWindowSize(DynamicThrottlePolicy policy, ManualTimer timer, int maxPending) { Message msg = new SimpleMessage("foo"); Reply reply = new SimpleReply("bar"); reply.setContext(1); @@ -213,8 +214,8 @@ public class ThrottlerTestCase { ++numPending; } - long tripTime = (numPending < maxPending) ? 1000 : 1000 + (numPending - maxPending) * 1000; - timer.millis += tripTime; + long tripTime = (numPending < maxPending) ? 1000L : 1000 + (numPending - maxPending) * 1000L; + timer.advance(tripTime); while (--numPending >= 0) { policy.processReply(reply); diff --git a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/TargetPoolTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/TargetPoolTestCase.java index afeaa1304a1..9fb817ad12f 100755 --- a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/TargetPoolTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/TargetPoolTestCase.java @@ -6,6 +6,7 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; import com.yahoo.jrt.slobrok.server.Slobrok; import com.yahoo.concurrent.Timer; +import com.yahoo.concurrent.ManualTimer; import com.yahoo.messagebus.network.rpc.test.TestServer; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -46,7 +47,7 @@ public class TargetPoolTestCase { // Necessary setup to be able to resolve targets. RPCServiceAddress adr1 = registerServer(); - PoolTimer timer = new PoolTimer(); + Timer timer = new ManualTimer(); RPCTargetPool pool1 = new RPCTargetPool(timer, 0.666, 1); RPCTarget target1 = pool1.getTarget(orb, adr1); @@ -80,7 +81,7 @@ public class TargetPoolTestCase { RPCServiceAddress adr2 = registerServer(); RPCServiceAddress adr3 = registerServer(); - PoolTimer timer = new PoolTimer(); + ManualTimer timer = new ManualTimer(); RPCTargetPool pool = new RPCTargetPool(timer, 0.666, 1); // Assert that all connections expire. @@ -96,7 +97,7 @@ public class TargetPoolTestCase { pool.flushTargets(false); assertEquals(3, pool.size()); } - timer.millis += 999; + timer.advance(999); pool.flushTargets(false); assertEquals(0, pool.size()); @@ -108,7 +109,7 @@ public class TargetPoolTestCase { assertNotNull(target = pool.getTarget(orb, adr3)); target.subRef(); assertEquals(3, pool.size()); - timer.millis += 444; + timer.advance(444); pool.flushTargets(false); assertEquals(3, pool.size()); assertNotNull(target = pool.getTarget(orb, adr2)); @@ -116,15 +117,15 @@ public class TargetPoolTestCase { assertNotNull(target = pool.getTarget(orb, adr3)); target.subRef(); assertEquals(3, pool.size()); - timer.millis += 444; + timer.advance(444); pool.flushTargets(false); assertEquals(2, pool.size()); assertNotNull(target = pool.getTarget(orb, adr3)); target.subRef(); - timer.millis += 444; + timer.advance(444); pool.flushTargets(false); assertEquals(1, pool.size()); - timer.millis += 444; + timer.advance(444); pool.flushTargets(false); assertEquals(0, pool.size()); @@ -132,12 +133,12 @@ public class TargetPoolTestCase { assertNotNull(target = pool.getTarget(orb, adr1)); assertEquals(1, pool.size()); for (int i = 0; i < 10; ++i) { - timer.millis += 999; + timer.advance(999); pool.flushTargets(false); assertEquals(1, pool.size()); } target.subRef(); - timer.millis += 999; + timer.advance(999); pool.flushTargets(false); assertEquals(0, pool.size()); } @@ -147,13 +148,4 @@ public class TargetPoolTestCase { return new RPCServiceAddress("foo/bar", servers.get(servers.size() - 1).mb.getConnectionSpec()); } - private static class PoolTimer implements Timer { - long millis = 0; - - @Override - public long milliTime() { - return millis; - } - } - } diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index 6abaec6ec93..6001fb0c000 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -323,6 +323,9 @@ public: void insert_juniper_field(vespalib::stringref input, vespalib::slime::Inserter&) override { _result = input; } + void insert_juniper_field(const document::StringFieldValue& input, vespalib::slime::Inserter&) override { + _result = input.getValueRef(); + } const vespalib::string& get_result() const noexcept { return _result; } }; diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.h b/searchcore/src/vespa/searchcore/proton/matching/match_tools.h index 8012d7b1ec5..bc147e9c787 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.h @@ -19,7 +19,7 @@ #include <vespa/vespalib/util/clock.h> namespace vespalib { class ExecutionProfiler; } -namespace vespalib { class ThreadBundle; } +namespace vespalib { struct ThreadBundle; } namespace search::engine { class Trace; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.h b/searchcore/src/vespa/searchcore/proton/matching/query.h index 8517ec2153f..1f9499b02d8 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.h +++ b/searchcore/src/vespa/searchcore/proton/matching/query.h @@ -10,7 +10,7 @@ #include <vespa/searchlib/queryeval/blueprint.h> #include <vespa/searchlib/queryeval/irequestcontext.h> -namespace vespalib { class ThreadBundle; } +namespace vespalib { struct ThreadBundle; } namespace search::engine { class Trace; } namespace proton::matching { diff --git a/searchcore/util/showschema.pl b/searchcore/util/showschema.pl deleted file mode 100644 index 1fbdc1cfec7..00000000000 --- a/searchcore/util/showschema.pl +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/perl -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -while ( <STDIN> ) { - if ( m/^indexfield\[(\d+)\]$/ ) { - $numfields = $1; - print "$numfields fields\n"; - next; - } - if ( m/^indexfield\[(\d+)\]\.(\w+) (.*)$/ ) { - $ifield[$1]{$2} = $3; - next; - } - if ( m/^fieldcollection\[(\d+)\]$/ ) { - $numfc = $1; - } - if ( m/^fieldcollection\[(\d+)\]\.name (.*)$/ ) { - $fieldcoll[$1]{name} = $2; - next; - } - if ( m/^fieldcollection\[(\d+)\]\.field\[\d+\]$/ ) { -# $fieldcoll[$1]{fields} = [ ]; - next; - } - if ( m/^fieldcollection\[(\d+)\]\.field\[\d+\]\.name (.*)$/ ) { - push(@{$fieldcoll[$1]{fields}}, $2); - next; - } -} -for ( $f = 0; $f < $numfields; ++$f) { - printf "indexfield %s %s %s %s %s %s\n", - $ifield[$f]{name}, $ifield[$f]{datatype}, $ifield[$f]{collectiontype}, - $ifield[$f]{prefix}, $ifield[$f]{phrases}, $ifield[$f]{positions}; -} -for ($fc = 0; $fc < $numfc; ++$fc) { - printf "fieldcoll %s -> %s\n", $fieldcoll[$fc]{name}, - join(' ', @{$fieldcoll[$fc]{fields}}); -} diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 4d0f520f666..80a02b0d928 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -89,9 +89,9 @@ vespa_define_module( src/tests/attribute/multi_value_mapping src/tests/attribute/multi_value_read_view src/tests/attribute/posting_list_merger + src/tests/attribute/posting_store src/tests/attribute/postinglist src/tests/attribute/postinglistattribute - src/tests/attribute/posting_store src/tests/attribute/reference_attribute src/tests/attribute/save_target src/tests/attribute/searchable @@ -112,8 +112,8 @@ vespa_define_module( src/tests/common/summaryfeatures src/tests/diskindex/bitvector src/tests/diskindex/diskindex - src/tests/diskindex/fieldwriter src/tests/diskindex/field_length_scanner + src/tests/diskindex/fieldwriter src/tests/diskindex/fusion src/tests/diskindex/pagedict4 src/tests/docstore/chunk @@ -193,6 +193,7 @@ vespa_define_module( src/tests/queryeval/equiv src/tests/queryeval/fake_searchable src/tests/queryeval/getnodeweight + src/tests/queryeval/global_filter src/tests/queryeval/matching_elements_search src/tests/queryeval/monitoring_search_iterator src/tests/queryeval/multibitvectoriterator diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index ebe96035fc8..3dda2eb6d95 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -1114,7 +1114,7 @@ TEST_F("NN blueprint handles empty filter (post-filtering)", NearestNeighborBlue TEST_F("NN blueprint handles strong filter (pre-filtering)", NearestNeighborBlueprintFixture) { auto bp = f.make_blueprint(); - auto filter = search::BitVector::create(11); + auto filter = search::BitVector::create(1,11); filter->setBit(3); filter->invalidateCachedCount(); auto strong_filter = GlobalFilter::create(std::move(filter)); @@ -1128,7 +1128,7 @@ TEST_F("NN blueprint handles strong filter (pre-filtering)", NearestNeighborBlue TEST_F("NN blueprint handles weak filter (pre-filtering)", NearestNeighborBlueprintFixture) { auto bp = f.make_blueprint(); - auto filter = search::BitVector::create(11); + auto filter = search::BitVector::create(1,11); filter->setBit(1); filter->setBit(3); filter->setBit(5); @@ -1147,7 +1147,7 @@ TEST_F("NN blueprint handles weak filter (pre-filtering)", NearestNeighborBluepr TEST_F("NN blueprint handles strong filter triggering exact search", NearestNeighborBlueprintFixture) { auto bp = f.make_blueprint(true, 0.2); - auto filter = search::BitVector::create(11); + auto filter = search::BitVector::create(1,11); filter->setBit(3); filter->invalidateCachedCount(); auto strong_filter = GlobalFilter::create(std::move(filter)); diff --git a/searchlib/src/tests/queryeval/global_filter/CMakeLists.txt b/searchlib/src/tests/queryeval/global_filter/CMakeLists.txt new file mode 100644 index 00000000000..2f768bf9d88 --- /dev/null +++ b/searchlib/src/tests/queryeval/global_filter/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_queryeval_global_filter_test_app TEST + SOURCES + global_filter_test.cpp + DEPENDS + searchlib + GTest::GTest +) +vespa_add_test(NAME searchlib_queryeval_global_filter_test_app COMMAND searchlib_queryeval_global_filter_test_app) diff --git a/searchlib/src/tests/queryeval/global_filter/global_filter_test.cpp b/searchlib/src/tests/queryeval/global_filter/global_filter_test.cpp new file mode 100644 index 00000000000..a64e75cecd6 --- /dev/null +++ b/searchlib/src/tests/queryeval/global_filter/global_filter_test.cpp @@ -0,0 +1,139 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/util/require.h> +#include <vespa/searchlib/queryeval/global_filter.h> +#include <vespa/searchlib/common/bitvector.h> + +#include <gmock/gmock.h> +#include <vector> + +using namespace testing; + +using search::BitVector; +using search::queryeval::GlobalFilter; +using vespalib::RequireFailedException; + +TEST(GlobalFilterTest, create_can_make_inactive_filter) { + auto filter = GlobalFilter::create(); + EXPECT_FALSE(filter->is_active()); +} + +void verify(const GlobalFilter &filter) { + EXPECT_TRUE(filter.is_active()); + EXPECT_EQ(filter.size(), 100); + EXPECT_EQ(filter.count(), 3); + for (size_t i = 1; i < 100; ++i) { + if (i == 11 || i == 22 || i == 33) { + EXPECT_TRUE(filter.check(i)); + } else { + EXPECT_FALSE(filter.check(i)); + } + } +} + +TEST(GlobalFilterTest, create_can_make_test_filter) { + auto docs = std::vector<uint32_t>({11,22,33}); + auto filter = GlobalFilter::create(docs, 100); + verify(*filter); +} + +TEST(GlobalFilterTest, test_filter_requires_docs_in_order) { + auto docs = std::vector<uint32_t>({11,33,22}); + EXPECT_THAT([&](){ GlobalFilter::create(docs, 100); }, Throws<RequireFailedException>()); +} + +TEST(GlobalFilterTest, test_filter_requires_docs_in_range) { + auto docs = std::vector<uint32_t>({11,22,133}); + EXPECT_THAT([&](){ GlobalFilter::create(docs, 100); }, Throws<RequireFailedException>()); +} + +TEST(GlobalFilterTest, test_filter_docid_0_not_allowed) { + auto docs = std::vector<uint32_t>({0,22,33}); + EXPECT_THAT([&](){ GlobalFilter::create(docs, 100); }, Throws<RequireFailedException>()); +} + +TEST(GlobalFilterTest, create_can_make_single_bitvector_filter) { + auto bits = BitVector::create(1, 100); + bits->setBit(11); + bits->setBit(22); + bits->setBit(33); + bits->invalidateCachedCount(); + EXPECT_EQ(bits->countTrueBits(), 3); + auto filter = GlobalFilter::create(std::move(bits)); + verify(*filter); +} + +TEST(GlobalFilterTest, global_filter_pointer_guard) { + auto inactive = GlobalFilter::create(); + auto active = GlobalFilter::create(BitVector::create(1,100)); + EXPECT_TRUE(active->is_active()); + EXPECT_FALSE(inactive->is_active()); + EXPECT_TRUE(active->ptr_if_active() == active.get()); + EXPECT_TRUE(inactive->ptr_if_active() == nullptr); +} + +TEST(GlobalFilterTest, create_can_make_multi_bitvector_filter) { + std::vector<std::unique_ptr<BitVector>> bits; + bits.push_back(BitVector::create(1, 11)); + bits.push_back(BitVector::create(11, 23)); + bits.push_back(BitVector::create(23, 25)); + bits.push_back(BitVector::create(25, 100)); + bits[1]->setBit(11); + bits[1]->setBit(22); + bits[3]->setBit(33); + for (const auto &v: bits) { + v->invalidateCachedCount(); + } + auto filter = GlobalFilter::create(std::move(bits)); + verify(*filter); +} + +TEST(GlobalFilterTest, multi_bitvector_filter_with_empty_vectors) { + std::vector<std::unique_ptr<BitVector>> bits; + bits.push_back(BitVector::create(1, 11)); + bits.push_back(BitVector::create(11, 23)); + bits.push_back(BitVector::create(23, 23)); + bits.push_back(BitVector::create(23, 23)); + bits.push_back(BitVector::create(23, 25)); + bits.push_back(BitVector::create(25, 100)); + bits[1]->setBit(11); + bits[1]->setBit(22); + bits[5]->setBit(33); + for (const auto &v: bits) { + v->invalidateCachedCount(); + } + auto filter = GlobalFilter::create(std::move(bits)); + verify(*filter); +} + +TEST(GlobalFilterTest, multi_bitvector_filter_with_no_vectors) { + std::vector<std::unique_ptr<BitVector>> bits; + auto filter = GlobalFilter::create(std::move(bits)); + EXPECT_TRUE(filter->is_active()); + EXPECT_EQ(filter->size(), 0); + EXPECT_EQ(filter->count(), 0); +} + +TEST(GlobalFilterTest, multi_bitvector_filter_requires_no_gaps) { + std::vector<std::unique_ptr<BitVector>> bits; + bits.push_back(BitVector::create(1, 11)); + bits.push_back(BitVector::create(12, 100)); + EXPECT_THAT([&](){ GlobalFilter::create(std::move(bits)); }, Throws<RequireFailedException>()); +} + +TEST(GlobalFilterTest, multi_bitvector_filter_requires_no_overlap) { + std::vector<std::unique_ptr<BitVector>> bits; + bits.push_back(BitVector::create(1, 11)); + bits.push_back(BitVector::create(10, 100)); + EXPECT_THAT([&](){ GlobalFilter::create(std::move(bits)); }, Throws<RequireFailedException>()); +} + +TEST(GlobalFilterTest, multi_bitvector_filter_requires_correct_order) { + std::vector<std::unique_ptr<BitVector>> bits; + bits.push_back(BitVector::create(11, 100)); + bits.push_back(BitVector::create(1, 11)); + EXPECT_THAT([&](){ GlobalFilter::create(std::move(bits)); }, Throws<RequireFailedException>()); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp index 33435f43618..f02681908d6 100644 --- a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp +++ b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp @@ -96,12 +96,7 @@ struct Fixture void setFilter(std::vector<uint32_t> docids) { uint32_t sz = _attr->getNumDocs(); - auto bit_vector = BitVector::create(sz); - for (uint32_t id : docids) { - EXPECT_LT(id, sz); - bit_vector->setBit(id); - } - _global_filter = GlobalFilter::create(std::move(bit_vector)); + _global_filter = GlobalFilter::create(docids, sz); } void setTensor(uint32_t docId, const Value &tensor) { diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp index 193bb04843c..7877b488065 100644 --- a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp @@ -82,10 +82,6 @@ public: ~HnswIndexTest() {} - const GlobalFilter *global_filter_ptr() const { - return global_filter->is_active() ? global_filter.get() : nullptr; - } - void init(bool heuristic_select_neighbors) { auto generator = std::make_unique<LevelGenerator>(); level_generator = generator.get(); @@ -110,12 +106,7 @@ public: } void set_filter(std::vector<uint32_t> docids) { uint32_t sz = 10; - auto bit_vector = BitVector::create(sz); - for (uint32_t id : docids) { - EXPECT_LT(id, sz); - bit_vector->setBit(id); - } - global_filter = GlobalFilter::create(std::move(bit_vector)); + global_filter = GlobalFilter::create(docids, sz); } GenerationHandler::Guard take_read_guard() { return gen_handler.takeGuard(); @@ -149,7 +140,7 @@ public: void expect_top_3(uint32_t docid, std::vector<uint32_t> exp_hits) { uint32_t k = 3; auto qv = vectors.get_vector(docid); - auto rv = index->top_k_candidates(qv, k, global_filter_ptr()).peek(); + auto rv = index->top_k_candidates(qv, k, global_filter->ptr_if_active()).peek(); std::sort(rv.begin(), rv.end(), LesserDistance()); size_t idx = 0; for (const auto & hit : rv) { @@ -170,7 +161,7 @@ public: void check_with_distance_threshold(uint32_t docid) { auto qv = vectors.get_vector(docid); uint32_t k = 3; - auto rv = index->top_k_candidates(qv, k, global_filter_ptr()).peek(); + auto rv = index->top_k_candidates(qv, k, global_filter->ptr_if_active()).peek(); std::sort(rv.begin(), rv.end(), LesserDistance()); EXPECT_EQ(rv.size(), 3); EXPECT_LE(rv[0].distance, rv[1].distance); diff --git a/searchlib/src/vespa/searchlib/queryeval/global_filter.cpp b/searchlib/src/vespa/searchlib/queryeval/global_filter.cpp index 1a5d3d3dacd..2aff91974a7 100644 --- a/searchlib/src/vespa/searchlib/queryeval/global_filter.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/global_filter.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "global_filter.h" +#include <vespa/vespalib/util/require.h> namespace search::queryeval { @@ -23,6 +24,31 @@ struct BitVectorFilter : public GlobalFilter { bool check(uint32_t docid) const override { return vector->testBit(docid); } }; +struct MultiBitVectorFilter : public GlobalFilter { + std::vector<std::unique_ptr<BitVector>> vectors; + std::vector<uint32_t> splits; + uint32_t total_size; + uint32_t total_count; + MultiBitVectorFilter(std::vector<std::unique_ptr<BitVector>> vectors_in, + std::vector<uint32_t> splits_in, + uint32_t total_size_in, + uint32_t total_count_in) + : vectors(std::move(vectors_in)), + splits(std::move(splits_in)), + total_size(total_size_in), + total_count(total_count_in) {} + bool is_active() const override { return true; } + uint32_t size() const override { return total_size; } + uint32_t count() const override { return total_count; } + bool check(uint32_t docid) const override { + size_t i = 0; + while ((i < splits.size()) && (docid >= splits[i])) { + ++i; + } + return vectors[i]->testBit(docid); + } +}; + } GlobalFilter::GlobalFilter() = default; @@ -34,8 +60,44 @@ GlobalFilter::create() { } std::shared_ptr<GlobalFilter> -GlobalFilter::create(std::unique_ptr<BitVector> vector) { +GlobalFilter::create(std::vector<uint32_t> docids, uint32_t size) +{ + uint32_t prev = 0; + auto bits = BitVector::create(1, size); + for (uint32_t docid: docids) { + REQUIRE(docid > prev); + REQUIRE(docid < size); + bits->setBit(docid); + prev = docid; + } + bits->invalidateCachedCount(); + return create(std::move(bits)); +} + +std::shared_ptr<GlobalFilter> +GlobalFilter::create(std::unique_ptr<BitVector> vector) +{ return std::make_shared<BitVectorFilter>(std::move(vector)); } +std::shared_ptr<GlobalFilter> +GlobalFilter::create(std::vector<std::unique_ptr<BitVector>> vectors) +{ + uint32_t total_size = 0; + uint32_t total_count = 0; + std::vector<uint32_t> splits; + for (size_t i = 0; i < vectors.size(); ++i) { + bool last = ((i + 1) == vectors.size()); + total_count += vectors[i]->countTrueBits(); + if (last) { + total_size = vectors[i]->size(); + } else { + REQUIRE_EQ(vectors[i]->size(), vectors[i + 1]->getStartIndex()); + splits.push_back(vectors[i]->size()); + } + } + return std::make_shared<MultiBitVectorFilter>(std::move(vectors), std::move(splits), + total_size, total_count); +} + } diff --git a/searchlib/src/vespa/searchlib/queryeval/global_filter.h b/searchlib/src/vespa/searchlib/queryeval/global_filter.h index c6e08d5018d..e93864db2c8 100644 --- a/searchlib/src/vespa/searchlib/queryeval/global_filter.h +++ b/searchlib/src/vespa/searchlib/queryeval/global_filter.h @@ -4,6 +4,7 @@ #include <memory> #include <vespa/searchlib/common/bitvector.h> +#include <vector> namespace search::queryeval { @@ -26,8 +27,14 @@ public: virtual bool check(uint32_t docid) const = 0; virtual ~GlobalFilter(); + const GlobalFilter *ptr_if_active() const { + return is_active() ? this : nullptr; + } + static std::shared_ptr<GlobalFilter> create(); + static std::shared_ptr<GlobalFilter> create(std::vector<uint32_t> docids, uint32_t size); static std::shared_ptr<GlobalFilter> create(std::unique_ptr<BitVector> vector); + static std::shared_ptr<GlobalFilter> create(std::vector<std::unique_ptr<BitVector>> vectors); }; } // namespace diff --git a/searchsummary/src/vespa/juniper/query.h b/searchsummary/src/vespa/juniper/query.h index 2b45909e0ac..f75949c6d06 100644 --- a/searchsummary/src/vespa/juniper/query.h +++ b/searchsummary/src/vespa/juniper/query.h @@ -35,9 +35,8 @@ const char* creator_text(ItemCreator); class IQueryVisitor; -/** Opaque datatype implemented by provider - */ -struct QueryItem; +// Interface class for juniper query items +class QueryItem; /** This is the basic query type, implemented by the query provider */ diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp index 4fb20bb1458..fcdb81defbf 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp @@ -5,6 +5,7 @@ #include "i_docsum_store_document.h" #include "i_juniper_converter.h" #include "juniper_query_adapter.h" +#include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/vespalib/objects/hexdump.h> #include <vespa/juniper/config.h> #include <vespa/juniper/result.h> @@ -114,6 +115,7 @@ public: JuniperConverter(const DynamicTeaserDFW& writer, uint32_t doc_id, GetDocsumsState& state); ~JuniperConverter() override; void insert_juniper_field(vespalib::stringref input, vespalib::slime::Inserter& inserter) override; + void insert_juniper_field(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) override; }; JuniperConverter::JuniperConverter(const DynamicTeaserDFW& writer, uint32_t doc_id, GetDocsumsState& state) @@ -132,6 +134,12 @@ JuniperConverter::insert_juniper_field(vespalib::stringref input, vespalib::slim _writer.insert_juniper_field(_doc_id, input, _state, inserter); } +void +JuniperConverter::insert_juniper_field(const document::StringFieldValue& input, vespalib::slime::Inserter& inserter) +{ + _writer.insert_juniper_field(_doc_id, input.getValueRef(), _state, inserter); +} + } void diff --git a/searchsummary/src/vespa/searchsummary/docsummary/i_juniper_converter.h b/searchsummary/src/vespa/searchsummary/docsummary/i_juniper_converter.h index f0b8ec4309f..00751082567 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/i_juniper_converter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/i_juniper_converter.h @@ -4,6 +4,7 @@ #include <vespa/vespalib/stllike/string.h> +namespace document { class StringFieldValue; } namespace vespalib::slime { struct Inserter; } namespace search::docsummary { @@ -11,12 +12,17 @@ namespace search::docsummary { /** * Interface class for inserting a dynamic string based on an * annotated full string and query context. + * + * For streaming search we use the same interface in an adapter that + * calls a snippet modifier (vsm::SnippetModifier) to add the annotation + * needed by juniper. */ class IJuniperConverter { public: virtual ~IJuniperConverter() = default; virtual void insert_juniper_field(vespalib::stringref input, vespalib::slime::Inserter& inserter) = 0; + virtual void insert_juniper_field(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) = 0; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.cpp b/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.cpp index 725ebff1f77..b3d3fde7150 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.cpp @@ -189,9 +189,9 @@ SlimeFiller::visit(const StringFieldValue& value) } } else { if (_juniper_converter != nullptr) { - _juniper_converter->insert_juniper_field(value.getValue(), _inserter); + _juniper_converter->insert_juniper_field(value, _inserter); } else { - _inserter.insertString(Memory(value.getValue())); + _inserter.insertString(Memory(value.getValueRef())); } } } diff --git a/storage/src/tests/pstack_testrunner b/storage/src/tests/pstack_testrunner index 151aaec7e1a..4d4d5105d5a 100755 --- a/storage/src/tests/pstack_testrunner +++ b/storage/src/tests/pstack_testrunner @@ -1,15 +1,9 @@ -#!/usr/bin/perl -w +#!/bin/sh # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -use strict; - -my @pids = `ps auxww | grep "./testrunner" | grep -v grep`; -foreach (@pids) { - s/^\S+\s+(\d+)\s+.*$/$1/; - chomp; -} - -foreach my $pid (@pids) { - my $cmd = "pstack $pid"; - system($cmd) == 0 or die "Failed to run '$cmd'"; -} +ps auxww | grep "./testrunner" | grep -v grep | while read username pid restofline; do + if pstack $pid; then :; else + echo "Failed to run 'pstack $pid'" + exit 1 + fi +done diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp index 32a9da08a51..69a38e205c0 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp @@ -5,6 +5,7 @@ #include <vespa/juniper/juniper_separators.h> #include <vespa/searchsummary/docsummary/check_undefined_value_visitor.h> #include <vespa/searchsummary/docsummary/i_docsum_store_document.h> +#include <vespa/searchsummary/docsummary/i_juniper_converter.h> #include <vespa/searchsummary/docsummary/summaryfieldconverter.h> #include <vespa/document/base/exceptions.h> #include <vespa/document/fieldvalue/iteratorhandler.h> @@ -20,6 +21,49 @@ namespace vsm { namespace { +bool is_struct_or_multivalue_field_type(const document::DataType& data_type) +{ + return (data_type.isStructured() || data_type.isArray() || data_type.isWeightedSet() || data_type.isMap()); +} + +/* + * This class creates a modified field value which is then passed to + * the original juniper converter. + */ +class SnippetModifierJuniperConverter : public IJuniperConverter +{ + IJuniperConverter& _orig_converter; + FieldModifier& _modifier; + FieldPath _empty_field_path; +public: + SnippetModifierJuniperConverter(IJuniperConverter& orig_converter, FieldModifier& modifier) + : IJuniperConverter(), + _orig_converter(orig_converter), + _modifier(modifier), + _empty_field_path() + { + } + ~SnippetModifierJuniperConverter() override = default; + void insert_juniper_field(vespalib::stringref input, vespalib::slime::Inserter& inserter) override; + void insert_juniper_field(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) override; +}; + + +void +SnippetModifierJuniperConverter::insert_juniper_field(vespalib::stringref input, vespalib::slime::Inserter& inserter) +{ + _orig_converter.insert_juniper_field(input, inserter); +} + +void +SnippetModifierJuniperConverter::insert_juniper_field(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) +{ + auto fv = _modifier.modify(input, _empty_field_path); + assert(fv); + auto& modified_input = dynamic_cast<const document::StringFieldValue &>(*fv); + _orig_converter.insert_juniper_field(modified_input.getValueRef(), inserter); +} + /** * Class providing access to a document retrieved from an IDocsumStore * (vsm::DocsumFilter). VSM specific transforms might be applied when @@ -114,6 +158,18 @@ DocsumStoreVsmDocument::insert_juniper_field(const vespalib::string& field_name, // Markup for juniper has already been added due to FLATTENJUNIPER command in vsm summary config. auto field_value = get_field_value(field_name); if (field_value) { + if (is_struct_or_multivalue_field_type(*field_value->getDataType())) { + auto entry_idx = _result_class.GetIndexFromName(field_name.c_str()); + if (entry_idx >= 0) { + assert((uint32_t) entry_idx < _result_class.GetNumEntries()); + auto modifier = _docsum_filter.get_field_modifier(entry_idx); + if (modifier != nullptr) { + SnippetModifierJuniperConverter stacked_converter(converter, *modifier); + SummaryFieldConverter::insert_juniper_field(*field_value, inserter, false, stacked_converter); + return; + } + } + } SummaryFieldConverter::insert_juniper_field(*field_value, inserter, false, converter); } } @@ -396,4 +452,16 @@ DocsumFilter::insert_summary_field(uint32_t entry_idx, const Document& doc, vesp } } +FieldModifier* +DocsumFilter::get_field_modifier(uint32_t entry_idx) +{ + if (_snippetModifiers == nullptr) { + return nullptr; + } + const auto& field_spec = _fields[entry_idx]; + auto& fieldId = field_spec.getOutputField(); + FieldIdT fId = fieldId.getId(); + return _snippetModifiers->getModifier(fId); +} + } diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.h b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.h index f143c91d962..9ec6cb54f51 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.h +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.h @@ -84,6 +84,7 @@ public: search::docsummary::DocsumStoreFieldValue get_summary_field(uint32_t entry_idx, const Document& doc); void insert_summary_field(uint32_t entry_idx, const Document& doc, vespalib::slime::Inserter& inserter); + FieldModifier* get_field_modifier(uint32_t entry_idx); }; } diff --git a/streamingvisitors/src/vespa/vsm/vsm/slimefieldwriter.h b/streamingvisitors/src/vespa/vsm/vsm/slimefieldwriter.h index 0907783feaf..9f72e4f3687 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/slimefieldwriter.h +++ b/streamingvisitors/src/vespa/vsm/vsm/slimefieldwriter.h @@ -5,7 +5,7 @@ #include <vespa/vsm/common/storagedocument.h> #include <vespa/document/fieldvalue/fieldvalues.h> -namespace vespalib::slime { class Inserter; } +namespace vespalib::slime { struct Inserter; } namespace vsm { diff --git a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java index 7268e892c7d..6eba29fe9cb 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java @@ -2,7 +2,7 @@ package com.yahoo.vespafeeder; import com.yahoo.clientmetrics.RouteMetricSet; -import com.yahoo.concurrent.Timer; +import com.yahoo.concurrent.ManualTimer; import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; import com.yahoo.messagebus.EmptyReply; @@ -15,17 +15,10 @@ import static org.junit.jupiter.api.Assertions.assertTrue; public class BenchmarkProgressPrinterTest { - class DummyTimer implements Timer { - long ms; - - public long milliTime() { return ms; } - } - @Test void testSimple() { ByteArrayOutputStream output = new ByteArrayOutputStream(); - DummyTimer timer = new DummyTimer(); - timer.ms = 0; + ManualTimer timer = new ManualTimer(); BenchmarkProgressPrinter printer = new BenchmarkProgressPrinter(timer, new PrintStream(output)); RouteMetricSet metrics = new RouteMetricSet("foobar", printer); @@ -35,7 +28,7 @@ public class BenchmarkProgressPrinterTest { metrics.addReply(reply); } - timer.ms = 1200; + timer.set(1200); { EmptyReply reply = new EmptyReply(); @@ -49,7 +42,7 @@ public class BenchmarkProgressPrinterTest { metrics.addReply(reply); } - timer.ms = 2400; + timer.set(2400); { EmptyReply reply = new EmptyReply(); @@ -58,7 +51,7 @@ public class BenchmarkProgressPrinterTest { metrics.addReply(reply); } - timer.ms = 62000; + timer.set(62000); { EmptyReply reply = new EmptyReply(); diff --git a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/ProgressPrinterTest.java b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/ProgressPrinterTest.java index 495367ff4c3..2307e27b161 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/ProgressPrinterTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/ProgressPrinterTest.java @@ -2,7 +2,7 @@ package com.yahoo.vespafeeder; import com.yahoo.clientmetrics.RouteMetricSet; -import com.yahoo.concurrent.Timer; +import com.yahoo.concurrent.ManualTimer; import com.yahoo.documentapi.messagebus.protocol.DocumentIgnoredReply; import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; @@ -16,17 +16,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class ProgressPrinterTest { - class DummyTimer implements Timer { - long ms; - - public long milliTime() { return ms; } - } - @Test void testSimple() { ByteArrayOutputStream output = new ByteArrayOutputStream(); - DummyTimer timer = new DummyTimer(); - timer.ms = 0; + ManualTimer timer = new ManualTimer(); ProgressPrinter printer = new ProgressPrinter(timer, new PrintStream(output)); RouteMetricSet metrics = new RouteMetricSet("foobar", printer); @@ -36,7 +29,7 @@ public class ProgressPrinterTest { metrics.addReply(reply); } - timer.ms = 1200; + timer.set(1200); { EmptyReply reply = new EmptyReply(); @@ -50,7 +43,7 @@ public class ProgressPrinterTest { metrics.addReply(reply); } - timer.ms = 2400; + timer.set(2400); { DocumentIgnoredReply reply = new DocumentIgnoredReply(); @@ -65,7 +58,7 @@ public class ProgressPrinterTest { metrics.addReply(reply); } - timer.ms = 62000; + timer.set(62000); { EmptyReply reply = new EmptyReply(); diff --git a/vespaclient/CMakeLists.txt b/vespaclient/CMakeLists.txt index a427198a22c..86e30ab1051 100644 --- a/vespaclient/CMakeLists.txt +++ b/vespaclient/CMakeLists.txt @@ -18,10 +18,6 @@ vespa_define_module( src/vespa/vespaclient/vesparoute ) -vespa_install_script(src/sh/vespa-set-node-state vespa-set-node-state bin) -vespa_install_script(src/sh/vespa-get-node-state vespa-get-node-state bin) -vespa_install_script(src/sh/vespa-get-cluster-state vespa-get-cluster-state bin) - vespa_install_script(src/perl/bin/SetNodeState.pl libexec/vespa) vespa_install_script(src/perl/bin/GetNodeState.pl libexec/vespa) vespa_install_script(src/perl/bin/GetClusterState.pl libexec/vespa) diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/ManualTimer.java b/vespajlib/src/main/java/com/yahoo/concurrent/ManualTimer.java new file mode 100644 index 00000000000..ffa6acd446a --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/concurrent/ManualTimer.java @@ -0,0 +1,18 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.concurrent; + +/** + * Simple manual Timer for use in tests + * @author baldersheim + */ +public class ManualTimer implements Timer { + + private long millis = 0; + public void set(long ms) { millis = ms; } + public void advance(long ms) { millis += ms; } + + @Override + public long milliTime() { + return millis; + } +} diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/Timer.java b/vespajlib/src/main/java/com/yahoo/concurrent/Timer.java index 1793e860af8..282524c0d54 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/Timer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/Timer.java @@ -1,6 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.concurrent; +import java.time.Clock; +import java.util.concurrent.TimeUnit; + /** * This interface wraps access to some timer that can be used to measure elapsed time, in milliseconds. This * abstraction allows for unit testing the behavior of time-based constructs. @@ -16,5 +19,14 @@ public interface Timer { * @return The current value of the timer, in milliseconds. */ long milliTime(); + Timer monotonic = () -> TimeUnit.NANOSECONDS.toMillis(System.nanoTime()); + static Timer wrap(Clock original) { + return new Timer() { + private final Clock clock = original; + @Override + public long milliTime() { + return clock.millis(); + } + }; } } diff --git a/vespalog/src/vespa-logfmt/vespa-logfmt.pl b/vespalog/src/vespa-logfmt/vespa-logfmt.pl deleted file mode 100755 index 611af47715c..00000000000 --- a/vespalog/src/vespa-logfmt/vespa-logfmt.pl +++ /dev/null @@ -1,306 +0,0 @@ -#!/usr/bin/env perl -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -sub findhome { - # Try the VESPA_HOME env variable - return $ENV{'VESPA_HOME'} if defined $ENV{'VESPA_HOME'}; - if ( $0 =~ m{(.*)/bin[^/]*/[^/]*logfmt[^/]*$} ) { - return $1; - } - return "/opt/vespa"; -} - -my $VESPA_HOME = findhome(); - -use 5.006_001; -use strict; -use warnings; - -use Getopt::Long qw(:config no_ignore_case); - -my %showflags = ( - time => 1, - fmttime => 1, - msecs => 1, - usecs => 0, - host => 0, - level => 1, - pid => 0, - service => 1, - component => 1, - message => 1 -); - -my %levelflags = ( - fatal => 1, - error => 1, - warning => 1, - info => 1, - config => 0, - event => 0, - debug => 0, - spam => 0 -); - -# Do not buffer the output -$| = 1; - -my $compore; -my $msgtxre; -my $onlypid; -my $onlysvc; -my $onlyhst; -my $onlyint; - -my $shortsvc; -my $shortcmp; - -my @optlevels; -my @optshow; -my $optlevels; -my $optfollow; -my $optnldequote; -my $opthelp = ''; - -my $bad = 0; - -GetOptions ('level|l=s' => \@optlevels, - 'service|S=s' => \$onlysvc, - 'show|s=s' => \@optshow, - 'pid|p=s' => \$onlypid, - 'internal|i' => \$onlyint, - 'component|c=s' => \$compore, - 'message|m=s' => \$msgtxre, - 'help|h' => \$opthelp, - 'follow|f' => \$optfollow, - 'nldequote|N' => \$optnldequote, - 'host|H=s' => \$onlyhst, - 'truncateservice|ts' => \$shortsvc, - 'truncatecomponent|tc|t' => \$shortcmp, -) or $bad=1; - -if ( @ARGV == 0 and ! -p STDIN) { - push(@ARGV, "$VESPA_HOME/logs/vespa/vespa.log"); -} - -if ( $optfollow ) { - my $filearg = ""; - if ( @ARGV > 1 ) { - print STDERR "ERROR: Cannot follow more than one file\n\n"; - $bad=1; - } else { - $filearg = shift @ARGV if (@ARGV > 0); - open(STDIN, "tail -F $filearg |") - or die "cannot open 'tail -F $filearg' as input pipe\n"; - } -} - -if ( $opthelp || $bad ) { - print STDERR "Usage: $0 [options] [inputfile ...]\n", - "Options:\n", - " -l LEVELLIST\t--level=LEVELLIST\tselect levels to include\n", - " -s FIELDLIST\t--show=FIELDLIST\tselect fields to print\n", - " -p PID\t--pid=PID\t\tselect messages from given PID\n", - " -S SERVICE\t--service=SERVICE\tselect messages from given SERVICE\n", - " -H HOST\t--host=HOST\t\tselect messages from given HOST\n", - " -c REGEX\t--component=REGEX\tselect components matching REGEX\n", - " -m REGEX\t--message=REGEX\t\tselect message text matching REGEX\n", - " -f\t\t--follow\t\tinvoke tail -F to follow input file\n", - " -N\t\t--nldequote\t\tdequote newlines in message text field\n", - " -t\t--tc\t--truncatecomponent\tchop component to 15 chars\n", - " --ts\t\t--truncateservice\tchop service to 9 chars\n", - " -i\t\t--internal\t\tfilter out plugin-generated messages\n", - "\n", - "FIELDLIST is comma separated, available fields:\n", - "\t time fmttime msecs usecs host level pid service component message\n", - "Available levels for LEVELLIST:\n", - "\t fatal error warning info event debug spam\n", - "for both lists, use 'all' for all possible values, and -xxx to disable xxx.\n"; - exit $bad; -} - - -$optlevels = join(",", @optlevels ); -if ( $optlevels ) { - my $k; - unless ( $optlevels =~ s/^\+// or $optlevels =~ m/^-/ ) { - $levelflags{$_} = 0 foreach ( keys %levelflags ); - } - my @l = split(/,|(?=-)/, $optlevels); - my $l; - foreach $l ( @l ) { - my $v = 1; - my $minus = ""; - if ( $l =~ s/^-// ) { $v = 0; $minus = "-"; } - if ( $l eq "all" ) { - foreach $k ( keys %levelflags ) { - $levelflags{$k} = $v; - } - } elsif ( defined $levelflags{$l} ) { - $levelflags{$l} = $v; - } else { - print STDERR "bad level option '$minus$l'\n"; - exit 1; - } - } -# print STDERR "select level $_ => $levelflags{$_}\n" -# foreach ( keys %levelflags ); -} - -my $optshow; -$optshow = join(",", @optshow ); -if ( $optshow ) { - my $k; - unless ( $optshow =~ s/^\+// or $optshow =~ m/^-/ ) { - $showflags{$_} = 0 foreach ( keys %showflags ); - } - my @l = split(/,|(?=-)/, $optshow); - my $l; - foreach $l ( @l ) { - my $v = 1; - my $minus = ""; - if ( $l =~ s/^-// ) { $v = 0; $minus = "-"; } - if ( $l eq "all" ) { - foreach $k ( keys %showflags ) { - $showflags{$k} = $v; - } - } elsif ( defined $showflags{$l} ) { - $showflags{$l} = $v; - } else { - print STDERR "bad show option '$minus$l'\n"; - exit 1; - } - } -# print STDERR "show field $_ => $showflags{$_}\n" -# foreach ( keys %showflags ) ; -} - -my $only_internal_regexp = qr/^ - [^.]*[.] - ( ai[.]vespa[.] | - com[.]yahoo[.] - ( application | binaryprefix | clientmetrics | - collections | component | compress | - concurrent | config | configtest | - container | data | docproc | docprocs | - document | documentapi | documentmodel | - dummyreceiver | errorhandling | exception | - feedapi | feedhandler | filedistribution | - fs4 | fsa | geo | io | javacc | jdisc | - jrt | lang | language | log | logserver | - messagebus | metrics | net | osgi | path | - plugin | prelude | processing | protect | - reflection | restapi | search | - searchdefinition | searchlib | security | - slime | socket | statistics | stream | - system | tensor | test | text | - time | transaction | vdslib | vespa | - vespaclient | vespafeeder | vespaget | - vespastat | vespasummarybenchmark | - vespavisit | vespaxmlparser | yolean ) - )[.]/x ; - -while (<>) { - chomp; - if ( /^ - (\d+)\.?(\d*) # seconds, optional fractional seconds - \t - ([^\t]*) # host - \t - (\d+\/?\d*|\-\/\d+) # pid, optional tid - \t - ([^\t]*) # servicename - \t - ([^\t]*) # componentname - \t - (\w+) # level - \t - (.*) # message text - $/x ) - { - my $secs = $1; - my $usec = $2 . "000000"; # make sure we have atleast 6 digits - my $host = $3; - my $pidn = $4; - my $svcn = $5; - my $comp = $6; - my $levl = $7; - my $msgt = $8; - - if ( ! defined $levelflags{$levl} ) { - print STDERR "Warning: unknown level '$levl' in input\n"; - $levelflags{$levl} = 1; - } - next unless ( $levelflags{$levl} ); - - # for now, only filter plugins in "Container" - if ($onlyint && $comp =~ m/^Container[.]/) { - if ($comp !~ m/$only_internal_regexp/) { - next; - } - } - if ($compore && $comp !~ m/$compore/o) { next; } - if ($msgtxre && $msgt !~ m/$msgtxre/o) { next; } - if ($onlypid && $pidn ne $onlypid) { next; } - if ($onlysvc && $svcn ne $onlysvc) { next; } - if ($onlyhst && $host ne $onlyhst) { next; } - - $levl = "\U$levl"; - - my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday); - ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday)=localtime($secs); - my $datestr = sprintf("%04d-%02d-%02d", - 1900+$year, 1+$mon, $mday); - my $timestr = sprintf("%02d:%02d:%02d", - $hour, $min, $sec); - - if ( $showflags{"time"} || $showflags{"fmttime"} ) { - if ($showflags{"fmttime"} ) { - print "[$datestr $timestr"; - if ( $showflags{"usecs"} ) { - printf ".%.6s", $usec; - } elsif ( $showflags{"msecs"} ) { - printf ".%.3s", $usec; - } - print "] "; - } else { - printf "%s.%.6s ", $secs, $usec; - } - } - if ( $showflags{"host"} ) { - printf "%-8s ", $host; - } - if ( $showflags{"level"} ) { - printf "%-7s : ", $levl; - } - if ( $showflags{"pid"} ) { - printf "%5s ", $pidn; - } - if ( $showflags{"service"} ) { - if ( $shortsvc ) { - printf "%-9.9s ", $svcn; - } else { - printf "%-16s ", $svcn; - } - } - if ( $showflags{"component"} ) { - if ( $shortcmp ) { - printf "%-15.15s ", $comp; - } else { - printf "%s\t", $comp; - } - } - if ( $showflags{"message"} ) { - if ( $optnldequote ) { - my $did_dequote_1 = ( $msgt =~ s/\\n\\t/\n\t/g ); - my $did_dequote_2 = ( $msgt =~ s/\\n/\n\t/g ); - $msgt = "\n\t${msgt}" if ( $did_dequote_1 || $did_dequote_2 ); - $msgt =~ s/\\t/\t/; - } - print $msgt; - } - print "\n"; - } else { - print STDERR "bad log line: '$_'\n"; - } -} diff --git a/vespalog/src/vespa/log/.gitignore b/vespalog/src/vespa/log/.gitignore index 7b45206e869..290968e7bfd 100644 --- a/vespalog/src/vespa/log/.gitignore +++ b/vespalog/src/vespa/log/.gitignore @@ -3,6 +3,5 @@ .depend Makefile logctl -loglevelnames.cpp logtest logtest.logcontrol diff --git a/vespalog/src/vespa/log/CMakeLists.txt b/vespalog/src/vespa/log/CMakeLists.txt index 1b68a53ad3d..dbeba17334b 100644 --- a/vespalog/src/vespa/log/CMakeLists.txt +++ b/vespalog/src/vespa/log/CMakeLists.txt @@ -2,7 +2,7 @@ vespa_add_library(vespalog SOURCES exceptions.cpp - ${CMAKE_CURRENT_BINARY_DIR}/loglevelnames.cpp + loglevelnames.cpp log.cpp bufferedlogger.cpp log-target-fd.cpp @@ -18,11 +18,3 @@ vespa_add_library(vespalog reject-filter.cpp INSTALL lib64 ) - -add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/loglevelnames.cpp - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/mknm.pl < ${CMAKE_CURRENT_SOURCE_DIR}/log.h > lln.NEW && mv lln.NEW ${CMAKE_CURRENT_BINARY_DIR}/loglevelnames.cpp - MAIN_DEPENDENCY log.h - DEPENDENCIES mknm.pl -) - diff --git a/vespalog/src/vespa/log/create-multiarg-file.pl b/vespalog/src/vespa/log/create-multiarg-file.pl deleted file mode 100755 index f2d60afeba3..00000000000 --- a/vespalog/src/vespa/log/create-multiarg-file.pl +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env perl -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -# This script generates multi-argument functions for sending some events. - -require 5.006_001; -use warnings; -use strict; - -die "Usage: $0 <event-name> <event-version> <number of functions> <name>...\n" - unless $#ARGV > 2; - -my $event = $ARGV[0]; -my $event_version = $ARGV[1]; -my $noof_fns = $ARGV[2]; - -my @variables = @ARGV; -splice(@variables, 0, 3); - - -open(H, ">event-${event}-multiarg.h") - or die "Cannot open event-${event}-multiarg.h: $!\n"; -open(CPP, ">event-${event}-multiarg.cpp") - or die "Cannot open event-${event}-multiarg.cpp: $!\n"; - -my $called_as = "$0 $ARGV[0] $ARGV[1] $ARGV[2]"; -foreach my $x (@variables) { - $called_as .= " \"$x\""; -} - -print H << "EOF"; -// This file was generated like this: -// $called_as -// Don not modify this file manually! -EOF - -print CPP << "EOF"; -// This file was generated like this: -// $called_as -// Do not modify this file manually! -EOF - - -my $i; - -for ($i = 2; $i <= $noof_fns; $i++) { - print H "void doEvent\u${event}("; - print CPP "void\nLogger::doEvent\u${event}("; - my $n; - for ($n = 1; $n <= $i; $n++) { - print H "," unless $n == 1; - print H "\n\t"; - print CPP "," unless $n == 1; - print CPP "\n\t"; - my $first = 1; - foreach my $var (@variables) { - print H ", " unless $first; - print H "${var}${n}"; - print CPP ", " unless $first; - print CPP "${var}${n}"; - $first = 0; - } - } - print H ");\n\n"; - print CPP ")\n{\n"; - - print CPP "\tdoLog(event, \"\", 0, \"${event}/${event_version}\""; - for ($n = 1; $n <= $i; $n++) { - foreach my $var (@variables) { - my $type; - my $quot = ""; - if ($var =~ m=double =) { - $type = "%lf"; - } elsif ($var =~ m=int =) { - $type = "%d"; - } elsif ($var =~ m=const char ?\*=) { - $type = "%s"; - $quot = "\\\""; - } else { - die "Don't know printf format for variable $var\n"; - } - my $name = $var; - $name =~ s=.*[ *&]==; - print CPP "\n\t\t\" ${name}=${quot}${type}${quot}\""; - } - } - print CPP ","; - for ($n = 1; $n <= $i; $n++) { - my $first = 1; - print CPP "," unless $n == 1; - print CPP "\n\t\t"; - foreach my $var (@variables) { - print CPP ", " unless $first; - $first = 0; - my $name = $var; - $name =~ s=.*[ *&]==; - print CPP "${name}${n}"; - } - } - - print CPP ");\n"; - - print CPP "}\n\n"; -} diff --git a/vespalog/src/vespa/log/event-count-multiarg.cpp b/vespalog/src/vespa/log/event-count-multiarg.cpp deleted file mode 100644 index 139e050ecf4..00000000000 --- a/vespalog/src/vespa/log/event-count-multiarg.cpp +++ /dev/null @@ -1,274 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// This file was generated like this: -// ./create-multiarg-file.pl count 2 10 "const char *name" "double value" -// Do not modify this file manually! -void -Logger::doEventCount( - const char *name1, double value1, - const char *name2, double value2) -{ - doLog(event, "", 0, "count/2" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf", - name1, value1, - name2, value2); -} - -void -Logger::doEventCount( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3) -{ - doLog(event, "", 0, "count/2" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf", - name1, value1, - name2, value2, - name3, value3); -} - -void -Logger::doEventCount( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4) -{ - doLog(event, "", 0, "count/2" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf", - name1, value1, - name2, value2, - name3, value3, - name4, value4); -} - -void -Logger::doEventCount( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5) -{ - doLog(event, "", 0, "count/2" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf", - name1, value1, - name2, value2, - name3, value3, - name4, value4, - name5, value5); -} - -void -Logger::doEventCount( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5, - const char *name6, double value6) -{ - doLog(event, "", 0, "count/2" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf", - name1, value1, - name2, value2, - name3, value3, - name4, value4, - name5, value5, - name6, value6); -} - -void -Logger::doEventCount( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5, - const char *name6, double value6, - const char *name7, double value7) -{ - doLog(event, "", 0, "count/2" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf", - name1, value1, - name2, value2, - name3, value3, - name4, value4, - name5, value5, - name6, value6, - name7, value7); -} - -void -Logger::doEventCount( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5, - const char *name6, double value6, - const char *name7, double value7, - const char *name8, double value8) -{ - doLog(event, "", 0, "count/2" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf", - name1, value1, - name2, value2, - name3, value3, - name4, value4, - name5, value5, - name6, value6, - name7, value7, - name8, value8); -} - -void -Logger::doEventCount( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5, - const char *name6, double value6, - const char *name7, double value7, - const char *name8, double value8, - const char *name9, double value9) -{ - doLog(event, "", 0, "count/2" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf", - name1, value1, - name2, value2, - name3, value3, - name4, value4, - name5, value5, - name6, value6, - name7, value7, - name8, value8, - name9, value9); -} - -void -Logger::doEventCount( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5, - const char *name6, double value6, - const char *name7, double value7, - const char *name8, double value8, - const char *name9, double value9, - const char *name10, double value10) -{ - doLog(event, "", 0, "count/2" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf", - name1, value1, - name2, value2, - name3, value3, - name4, value4, - name5, value5, - name6, value6, - name7, value7, - name8, value8, - name9, value9, - name10, value10); -} - diff --git a/vespalog/src/vespa/log/event-count-multiarg.h b/vespalog/src/vespa/log/event-count-multiarg.h deleted file mode 100644 index 0019245329f..00000000000 --- a/vespalog/src/vespa/log/event-count-multiarg.h +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// This file was generated like this: -// ./create-multiarg-file.pl count 2 10 "const char *name" "double value" -// Don not modify this file manually! -void doEventCount( - const char *name1, double value1, - const char *name2, double value2); - -void doEventCount( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3); - -void doEventCount( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4); - -void doEventCount( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5); - -void doEventCount( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5, - const char *name6, double value6); - -void doEventCount( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5, - const char *name6, double value6, - const char *name7, double value7); - -void doEventCount( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5, - const char *name6, double value6, - const char *name7, double value7, - const char *name8, double value8); - -void doEventCount( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5, - const char *name6, double value6, - const char *name7, double value7, - const char *name8, double value8, - const char *name9, double value9); - -void doEventCount( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5, - const char *name6, double value6, - const char *name7, double value7, - const char *name8, double value8, - const char *name9, double value9, - const char *name10, double value10); - diff --git a/vespalog/src/vespa/log/event-value-multiarg.cpp b/vespalog/src/vespa/log/event-value-multiarg.cpp deleted file mode 100644 index faaf0a1fde9..00000000000 --- a/vespalog/src/vespa/log/event-value-multiarg.cpp +++ /dev/null @@ -1,274 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// This file was generated like this: -// ./create-multiarg-file.pl value 2 10 "const char *name" "double value" -// Do not modify this file manually! -void -Logger::doEventValue( - const char *name1, double value1, - const char *name2, double value2) -{ - doLog(event, "", 0, "value/2" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf", - name1, value1, - name2, value2); -} - -void -Logger::doEventValue( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3) -{ - doLog(event, "", 0, "value/2" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf", - name1, value1, - name2, value2, - name3, value3); -} - -void -Logger::doEventValue( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4) -{ - doLog(event, "", 0, "value/2" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf", - name1, value1, - name2, value2, - name3, value3, - name4, value4); -} - -void -Logger::doEventValue( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5) -{ - doLog(event, "", 0, "value/2" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf", - name1, value1, - name2, value2, - name3, value3, - name4, value4, - name5, value5); -} - -void -Logger::doEventValue( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5, - const char *name6, double value6) -{ - doLog(event, "", 0, "value/2" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf", - name1, value1, - name2, value2, - name3, value3, - name4, value4, - name5, value5, - name6, value6); -} - -void -Logger::doEventValue( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5, - const char *name6, double value6, - const char *name7, double value7) -{ - doLog(event, "", 0, "value/2" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf", - name1, value1, - name2, value2, - name3, value3, - name4, value4, - name5, value5, - name6, value6, - name7, value7); -} - -void -Logger::doEventValue( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5, - const char *name6, double value6, - const char *name7, double value7, - const char *name8, double value8) -{ - doLog(event, "", 0, "value/2" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf", - name1, value1, - name2, value2, - name3, value3, - name4, value4, - name5, value5, - name6, value6, - name7, value7, - name8, value8); -} - -void -Logger::doEventValue( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5, - const char *name6, double value6, - const char *name7, double value7, - const char *name8, double value8, - const char *name9, double value9) -{ - doLog(event, "", 0, "value/2" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf", - name1, value1, - name2, value2, - name3, value3, - name4, value4, - name5, value5, - name6, value6, - name7, value7, - name8, value8, - name9, value9); -} - -void -Logger::doEventValue( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5, - const char *name6, double value6, - const char *name7, double value7, - const char *name8, double value8, - const char *name9, double value9, - const char *name10, double value10) -{ - doLog(event, "", 0, "value/2" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf" - " name=\"%s\"" - " value=%lf", - name1, value1, - name2, value2, - name3, value3, - name4, value4, - name5, value5, - name6, value6, - name7, value7, - name8, value8, - name9, value9, - name10, value10); -} - diff --git a/vespalog/src/vespa/log/event-value-multiarg.h b/vespalog/src/vespa/log/event-value-multiarg.h deleted file mode 100644 index a84f56291f9..00000000000 --- a/vespalog/src/vespa/log/event-value-multiarg.h +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// This file was generated like this: -// ./create-multiarg-file.pl value 2 10 "const char *name" "double value" -// Don not modify this file manually! -void doEventValue( - const char *name1, double value1, - const char *name2, double value2); - -void doEventValue( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3); - -void doEventValue( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4); - -void doEventValue( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5); - -void doEventValue( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5, - const char *name6, double value6); - -void doEventValue( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5, - const char *name6, double value6, - const char *name7, double value7); - -void doEventValue( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5, - const char *name6, double value6, - const char *name7, double value7, - const char *name8, double value8); - -void doEventValue( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5, - const char *name6, double value6, - const char *name7, double value7, - const char *name8, double value8, - const char *name9, double value9); - -void doEventValue( - const char *name1, double value1, - const char *name2, double value2, - const char *name3, double value3, - const char *name4, double value4, - const char *name5, double value5, - const char *name6, double value6, - const char *name7, double value7, - const char *name8, double value8, - const char *name9, double value9, - const char *name10, double value10); - diff --git a/vespalog/src/vespa/log/loglevelnames.cpp b/vespalog/src/vespa/log/loglevelnames.cpp new file mode 100644 index 00000000000..4b571b23994 --- /dev/null +++ b/vespalog/src/vespa/log/loglevelnames.cpp @@ -0,0 +1,33 @@ +#include <string.h> +#include <vespa/log/log.h> + +namespace ns_log { + +enum Logger::LogLevel +Logger::parseLevel(const char *lname) +{ + if (strcmp(lname, "fatal") == 0) return fatal; + if (strcmp(lname, "error") == 0) return error; + if (strcmp(lname, "warning") == 0) return warning; + if (strcmp(lname, "config") == 0) return config; + if (strcmp(lname, "info") == 0) return info; + if (strcmp(lname, "event") == 0) return event; + if (strcmp(lname, "debug") == 0) return debug; + if (strcmp(lname, "spam") == 0) return spam; + // bad level name signaled by NUM_LOGLEVELS + return NUM_LOGLEVELS; +} + +const char *Logger::logLevelNames[] = { + "fatal", + "error", + "warning", + "config", + "info", + "event", + "debug", + "spam", + 0 // converting NUM_LOGLEVELS gives null pointer +}; + +} // namespace diff --git a/vespalog/src/vespa/log/mknm.pl b/vespalog/src/vespa/log/mknm.pl deleted file mode 100755 index 9aa2f6310ce..00000000000 --- a/vespalog/src/vespa/log/mknm.pl +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/perl -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -while (<>) { - if ( s/.*\benum\s+LogLevel\s*\{// ) { - chomp; - $t = $_; - while (<>) { - if ( s/\}.*// ) { - $t .= $_; - $t =~ s/,/ /g; - @t = split(" ", $t); - if ( $t[$#t] ne "NUM_LOGLEVELS" ) { - die "expected NUM_LOGLEVELS got '$t[$#t]'\n"; - } - pop @t; - makecpp(); - } - $t .= $_; - } - } -} -die "did not find enum\n"; - -sub makecpp -{ - print "#include <string.h>\n"; - print '#include <vespa/log/log.h>'; - print "\n\n" . "namespace ns_log {" . "\n\n"; - - print "enum Logger::LogLevel\n"; - print "Logger::parseLevel(const char *lname)\n{\n"; - foreach $l ( @t ) { - print " if (strcmp(lname, \"$l\") == 0) return $l;\n"; - } - print " // bad level name signaled by NUM_LOGLEVELS\n"; - print " return NUM_LOGLEVELS;\n"; - print "}\n\n"; - - print "const char *Logger::logLevelNames[] = {" . "\n "; - foreach $l ( @t ) { $l = "\"$l\""; } - push @t, "0 // converting NUM_LOGLEVELS gives null pointer\n"; - print join(",\n ", @t); - print "};\n\n} // namespace\n"; - exit(0); -} |