diff options
207 files changed, 1568 insertions, 7568 deletions
diff --git a/build_settings.cmake b/build_settings.cmake index 2bf6f93b815..a6e2490f117 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -60,7 +60,7 @@ if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" ST endif() endif() else() - set(CXX_SPECIFIC_WARN_OPTS "-Wnoexcept -Wsuggest-override -Wnon-virtual-dtor -Wformat-security") + set(CXX_SPECIFIC_WARN_OPTS "-Wnoexcept -Wsuggest-override -Wnon-virtual-dtor -Wformat-security -Wmismatched-tags") set(VESPA_GCC_LIB "gcc") set(VESPA_STDCXX_FS_LIB "stdc++fs") endif() diff --git a/client/go/cmd/version.go b/client/go/cmd/version.go index 0a47039a37d..3189b573059 100644 --- a/client/go/cmd/version.go +++ b/client/go/cmd/version.go @@ -27,7 +27,7 @@ func newVersionCmd(cli *CLI) *cobra.Command { SilenceUsage: true, Args: cobra.ExactArgs(0), RunE: func(cmd *cobra.Command, args []string) error { - log.Printf("vespa version %s compiled with %v on %v/%v", build.Version, runtime.Version(), runtime.GOOS, runtime.GOARCH) + log.Printf("Vespa CLI version %s compiled with %v on %v/%v", build.Version, runtime.Version(), runtime.GOOS, runtime.GOARCH) if !skipVersionCheck && cli.isTerminal() { return checkVersion(cli) } diff --git a/client/go/cmd/version_test.go b/client/go/cmd/version_test.go index 4354a2098c7..404aa9ac1e1 100644 --- a/client/go/cmd/version_test.go +++ b/client/go/cmd/version_test.go @@ -21,7 +21,7 @@ func TestVersion(t *testing.T) { t.Fatal(err) } assert.Equal(t, "", stderr.String()) - assert.Contains(t, stdout.String(), "vespa version 0.0.0-devel compiled with") + assert.Contains(t, stdout.String(), "Vespa CLI version 0.0.0-devel compiled with") assert.Contains(t, stdout.String(), "New release available: 1.2.3\nhttps://github.com/vespa-engine/vespa/releases/tag/v1.2.3") } @@ -38,7 +38,7 @@ func TestVersionCheckHomebrew(t *testing.T) { t.Fatal(err) } assert.Equal(t, "", stderr.String()) - assert.Contains(t, stdout.String(), "vespa version 0.0.0-devel compiled with") + assert.Contains(t, stdout.String(), "Vespa CLI version 0.0.0-devel compiled with") assert.Contains(t, stdout.String(), "New release available: 1.2.3\n"+ "https://github.com/vespa-engine/vespa/releases/tag/v1.2.3\n"+ "\nUpgrade by running:\nbrew update && brew upgrade vespa-cli\n") diff --git a/client/go/vespa/target_custom.go b/client/go/vespa/target_custom.go index c34f801641c..19fd56e7568 100644 --- a/client/go/vespa/target_custom.go +++ b/client/go/vespa/target_custom.go @@ -73,7 +73,7 @@ func (t *customTarget) Service(name string, timeout time.Duration, sessionOrRunI } func (t *customTarget) PrintLog(options LogOptions) error { - return fmt.Errorf("reading logs from non-cloud deployment is unsupported") + return fmt.Errorf("log access is only supported on cloud: run vespa-logfmt on the admin node instead") } func (t *customTarget) SignRequest(req *http.Request, sigKeyId string) error { return nil } diff --git a/config-lib/abi-spec.json b/config-lib/abi-spec.json index 573a8a24e0c..2d3092229ae 100644 --- a/config-lib/abi-spec.json +++ b/config-lib/abi-spec.json @@ -335,20 +335,19 @@ "public" ], "methods": [ - "public void <init>(java.nio.file.Path)", - "public void <init>(java.util.Optional, java.util.Optional, java.util.Optional)", "public java.util.Optional modelId()", "public java.util.Optional url()", "public java.util.Optional path()", - "public com.yahoo.config.ModelReference withModelId(java.util.Optional)", - "public com.yahoo.config.ModelReference withUrl(java.util.Optional)", - "public com.yahoo.config.ModelReference withPath(java.util.Optional)", "public java.nio.file.Path value()", "public boolean equals(java.lang.Object)", "public int hashCode()", "public java.lang.String toString()", - "public static com.yahoo.config.ModelReference valueOf(java.nio.file.Path)", - "public static com.yahoo.config.ModelReference valueOf(java.lang.String)" + "public static com.yahoo.config.ModelReference valueOf(java.lang.String)", + "public static com.yahoo.config.ModelReference unresolved(java.lang.String)", + "public static com.yahoo.config.ModelReference unresolved(com.yahoo.config.UrlReference)", + "public static com.yahoo.config.ModelReference unresolved(com.yahoo.config.FileReference)", + "public static com.yahoo.config.ModelReference unresolved(java.util.Optional, java.util.Optional, java.util.Optional)", + "public static com.yahoo.config.ModelReference resolved(java.nio.file.Path)" ], "fields": [] }, diff --git a/config-lib/src/main/java/com/yahoo/config/ModelReference.java b/config-lib/src/main/java/com/yahoo/config/ModelReference.java index 13bb5737c6f..25caad55b84 100644 --- a/config-lib/src/main/java/com/yahoo/config/ModelReference.java +++ b/config-lib/src/main/java/com/yahoo/config/ModelReference.java @@ -22,57 +22,41 @@ public class ModelReference { // Or: If resolved, this is set private final Path resolved; - public ModelReference(Path resolved) { - this.modelId = Optional.empty(); - this.url = Optional.empty(); - this.path = Optional.empty(); - this.resolved = resolved; - } - - public ModelReference(Optional<String> modelId, - Optional<UrlReference> url, - Optional<FileReference> path) { - if (modelId.isEmpty() && url.isEmpty() && path.isEmpty()) - throw new IllegalArgumentException("A model reference must have either a model id, url or path"); + private ModelReference(Optional<String> modelId, + Optional<UrlReference> url, + Optional<FileReference> path, + Path resolved) { this.modelId = modelId; this.url = url; this.path = path; - this.resolved = null; + this.resolved = resolved; } + /** Returns the id specified for this model, oor null if it is resolved. */ public Optional<String> modelId() { return modelId; } - public Optional<UrlReference> url() { return url; } - public Optional<FileReference> path() { return path; } - public ModelReference withModelId(Optional<String> modelId) { - return new ModelReference(modelId, url, path); - } - - public ModelReference withUrl(Optional<UrlReference> url) { - return new ModelReference(modelId, url, path); - } + /** Returns the url specified for this model, or null if it is resolved. */ + public Optional<UrlReference> url() { return url; } - public ModelReference withPath(Optional<FileReference> path) { - return new ModelReference(modelId, url, path); - } + /** Returns the path specified for this model, oor null if it is resolved. */ + public Optional<FileReference> path() { return path; } - /** Returns the path to the file containing this model, or null if not available. */ - public Path value() { - return resolved; - } + /** Returns the path to the file containing this model, or null if this is unresolved. */ + public Path value() { return resolved; } @Override public boolean equals(Object o) { if ( ! (o instanceof ModelReference other)) return false; - if ( ! this.modelId.equals(other.modelId)) return false; - if ( ! this.url.equals(other.url)) return false; - if ( ! this.path.equals(other.path)) return false; + if ( ! Objects.equals(this.modelId, other.modelId)) return false; + if ( ! Objects.equals(this.url, other.url)) return false; + if ( ! Objects.equals(this.path, other.path)) return false; + if ( ! Objects.equals(this.resolved, other.resolved)) return false; return true; } @Override public int hashCode() { - return Objects.hash(modelId, url, path); + return Objects.hash(modelId, url, path, resolved); } /** Returns this on the format accepted by valueOf */ @@ -84,26 +68,50 @@ public class ModelReference { path.map(v -> v.value()).orElse("\"\""); } - /** Creates a model reference resolved to a Path to the local file. */ - public static ModelReference valueOf(Path path) { - return new ModelReference(path); - } - /** - * Creates a model reference from a three-part string on the form - * <code>modelId url path</code> - * Each of the elements is either a value not containing space, or empty represented by "". + * Creates a model reference which is either a single string with no spaces if resolved, or if unresolved + * a three-part string on the form <code>modelId url path</code>, where + * each of the elements is either a value not containing space, or empty represented by "". */ public static ModelReference valueOf(String s) { String[] parts = s.split(" "); if (parts.length == 1) - return new ModelReference(Path.of(s)); + return resolved(Path.of(s)); else if (parts.length == 3) - return new ModelReference(parts[0].equals("\"\"") ? Optional.empty() : Optional.of(parts[0]), - parts[1].equals("\"\"") ? Optional.empty() : Optional.of(new UrlReference(parts[1])), - parts[2].equals("\"\"") ? Optional.empty() : Optional.of(new FileReference(parts[2]))); + return unresolved(parts[0].equals("\"\"") ? Optional.empty() : Optional.of(parts[0]), + parts[1].equals("\"\"") ? Optional.empty() : Optional.of(new UrlReference(parts[1])), + parts[2].equals("\"\"") ? Optional.empty() : Optional.of(new FileReference(parts[2]))); else - throw new IllegalArgumentException("Unexpected model string '" + s + "'"); + throw new IllegalArgumentException("Unexpected model reference string '" + s + "'"); + } + + /** Creates an unresolved reference from a model id only. */ + public static ModelReference unresolved(String modelId) { + return new ModelReference(Optional.of(modelId), Optional.empty(), Optional.empty(), null); + } + + /** Creates an unresolved reference from an url only. */ + public static ModelReference unresolved(UrlReference url) { + return new ModelReference(Optional.empty(), Optional.of(url), Optional.empty(), null); + } + + /** Creates an unresolved reference from a path only. */ + public static ModelReference unresolved(FileReference path) { + return new ModelReference(Optional.empty(), Optional.empty(), Optional.of(path), null); + } + + /** Creates an unresolved reference. */ + public static ModelReference unresolved(Optional<String> modelId, + Optional<UrlReference> url, + Optional<FileReference> path) { + if (modelId.isEmpty() && url.isEmpty() && path.isEmpty()) + throw new IllegalArgumentException("A model reference must have either a model id, url or path"); + return new ModelReference(modelId, url, path, null); + } + + /** Creates a nresolved reference. */ + public static ModelReference resolved(Path path) { + return new ModelReference(null, null, null, Objects.requireNonNull(path)); } } diff --git a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java index d1cd7678911..76871aaca42 100644 --- a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java +++ b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java @@ -169,9 +169,7 @@ public class ConfigInstanceBuilderTest { fileVal("etc"). pathVal(FileReference.mockFileReferenceForUnitTesting(new File("pom.xml"))). urlVal(new UrlReference("http://docs.vespa.ai")). - modelVal(new ModelReference(Optional.empty(), - Optional.empty(), - Optional.of(FileReference.mockFileReferenceForUnitTesting(new File("pom.xml"))))). + modelVal(ModelReference.unresolved(FileReference.mockFileReferenceForUnitTesting(new File("pom.xml")))). boolarr(false). longarr(9223372036854775807L). longarr(-9223372036854775808L). diff --git a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java index 1a05c08d8f2..db1509fba93 100644 --- a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java +++ b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java @@ -132,9 +132,9 @@ public class ConfigInstanceEqualsTest { fileVal("etc"). pathVal(FileReference.mockFileReferenceForUnitTesting(new File("pom.xml"))). urlVal(new UrlReference("http://docs.vespa.ai")). - modelVal(new ModelReference(Optional.of("my-model-id"), - Optional.of(new UrlReference("http://docs.vespa.ai")), - Optional.empty())). + modelVal(ModelReference.unresolved(Optional.of("my-model-id"), + Optional.of(new UrlReference("http://docs.vespa.ai")), + Optional.empty())). boolarr(false). longarr(9223372036854775807L). longarr(-9223372036854775808L). @@ -145,9 +145,9 @@ public class ConfigInstanceEqualsTest { refarr(Arrays.asList(":parent:", ":parent", "parent:")). // test collection based setter fileArr("bin"). urlArr(new UrlReference("http://docs.vespa.ai")). - modelArr(new ModelReference(Optional.empty(), - Optional.of(new UrlReference("http://docs.vespa.ai")), - Optional.of(FileReference.mockFileReferenceForUnitTesting(new File("pom.xml"))))). + modelArr(ModelReference.unresolved(Optional.empty(), + Optional.of(new UrlReference("http://docs.vespa.ai")), + Optional.of(FileReference.mockFileReferenceForUnitTesting(new File("pom.xml"))))). basicStruct(new BasicStruct.Builder(). foo("basicFoo"). diff --git a/config-lib/src/test/java/com/yahoo/config/ModelNodeTest.java b/config-lib/src/test/java/com/yahoo/config/ModelNodeTest.java index 696e0722714..328b27bf4c8 100644 --- a/config-lib/src/test/java/com/yahoo/config/ModelNodeTest.java +++ b/config-lib/src/test/java/com/yahoo/config/ModelNodeTest.java @@ -3,6 +3,7 @@ package com.yahoo.config; import org.junit.jupiter.api.Test; +import java.nio.file.Path; import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -18,12 +19,19 @@ public class ModelNodeTest { } @Test - void testReference() { - var reference = new ModelReference(Optional.of("myModelId"), - Optional.of(new UrlReference("https://host:my/path")), - Optional.of(new FileReference("foo.txt"))); + void testUnresolvedReference() { + var reference = ModelReference.unresolved(Optional.of("myModelId"), + Optional.of(new UrlReference("https://host:my/path")), + Optional.of(new FileReference("foo.txt"))); assertEquals("myModelId https://host:my/path foo.txt", reference.toString()); assertEquals(reference, ModelReference.valueOf(reference.toString())); } + @Test + void testResolvedReference() { + var reference = ModelReference.resolved(Path.of("dir/resolvedFile.txt")); + assertEquals("dir/resolvedFile.txt", reference.toString()); + assertEquals(reference, ModelReference.valueOf(reference.toString())); + } + } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java b/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java index 53633d1b7d4..c2cde72449b 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java @@ -6,7 +6,6 @@ import com.yahoo.config.FileReference; import java.nio.ByteBuffer; import java.util.List; - /** * @author Tony Vaagenes */ diff --git a/config-model/src/main/java/com/yahoo/schema/DistributableResource.java b/config-model/src/main/java/com/yahoo/schema/DistributableResource.java index 7a8a3963ba4..e7bdb68a03d 100644 --- a/config-model/src/main/java/com/yahoo/schema/DistributableResource.java +++ b/config-model/src/main/java/com/yahoo/schema/DistributableResource.java @@ -67,14 +67,9 @@ public class DistributableResource implements Comparable <DistributableResource> public void register(FileRegistry fileRegistry) { switch (pathType) { - case FILE: - fileReference = fileRegistry.addFile(path); - break; - case URI: - fileReference = fileRegistry.addUri(path); - break; - default: - throw new IllegalArgumentException("Unknown path type " + pathType); + case FILE -> fileReference = fileRegistry.addFile(path); + case URI -> fileReference = fileRegistry.addUri(path); + default -> throw new IllegalArgumentException("Unknown path type " + pathType); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index b25e8aaf2c0..9e15d43d92d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -330,6 +330,7 @@ public class VespaMetricSet { metrics.add(new Metric("query_hit_offset.count")); metrics.add(new Metric("documents_covered.count")); metrics.add(new Metric("documents_total.count")); + metrics.add(new Metric("documents_target_total.count")); metrics.add(new Metric("dispatch_internal.rate")); metrics.add(new Metric("dispatch_fdispatch.rate")); addMetric(metrics, "jdisc.render.latency", Set.of("min", "max", "count", "sum", "last", "average")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java index 1631bd228df..ee1771cfbfc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java @@ -3,6 +3,9 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.collections.Tuple2; import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.FileReference; +import com.yahoo.config.ModelReference; +import com.yahoo.config.UrlReference; import com.yahoo.text.XML; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; @@ -11,6 +14,7 @@ import com.yahoo.vespa.config.util.ConfigUtils; import com.yahoo.yolean.Exceptions; import org.w3c.dom.Element; import java.util.List; +import java.util.Optional; import java.util.regex.Pattern; /** @@ -133,19 +137,19 @@ public class DomConfigPayloadBuilder { } else if (element.hasAttribute("model-id") || element.hasAttribute("url") || element.hasAttribute("path")) { // special syntax for "model" fields - String modelString = modelElement("model-id", element); - modelString += " " + modelElement("url", element); - modelString += " " + modelElement("path", element); - payloadBuilder.setField(name, modelString); + var model = ModelReference.unresolved(modelElement("model-id", element), + modelElement("url", element).map(UrlReference::new), + modelElement("path", element).map(FileReference::new)); + payloadBuilder.setField(name, model.toString()); } else { // leaf value: <myValueName>value</myValue> payloadBuilder.setField(name, value); } } - private String modelElement(String attributeName, Element element) { - String value = XML.attribute(attributeName, element).orElse("\"\"").trim(); - if (value.contains(" ")) + private Optional<String> modelElement(String attributeName, Element element) { + Optional<String> value = XML.attribute(attributeName, element); + if (value.isPresent() && value.get().contains(" ")) throw new IllegalArgumentException("The value of " + attributeName + " on " + element.getTagName() + "cannot contain space"); return value; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index bb45c509b5b..16eac3bb34e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -191,14 +191,14 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addModelEvaluation(spec, cluster, context); addModelEvaluationBundles(cluster); - addProcessing(deployState, spec, cluster); - addSearch(deployState, spec, cluster); + addProcessing(deployState, spec, cluster, context); + addSearch(deployState, spec, cluster, context); addDocproc(deployState, spec, cluster); - addDocumentApi(deployState, spec, cluster); // NOTE: Must be done after addSearch + addDocumentApi(deployState, spec, cluster, context); // NOTE: Must be done after addSearch cluster.addDefaultHandlersExceptStatus(); addStatusHandlers(cluster, context.getDeployState().isHosted()); - addUserHandlers(deployState, cluster, spec); + addUserHandlers(deployState, cluster, spec, context); addHttp(deployState, spec, cluster, context); @@ -520,8 +520,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return http; } - private void addDocumentApi(DeployState deployState, Element spec, ApplicationContainerCluster cluster) { - ContainerDocumentApi containerDocumentApi = buildDocumentApi(deployState, cluster, spec); + private void addDocumentApi(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) { + ContainerDocumentApi containerDocumentApi = buildDocumentApi(deployState, cluster, spec, context); if (containerDocumentApi == null) return; cluster.setDocumentApi(containerDocumentApi); @@ -537,14 +537,14 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { docprocOptions.maxConcurrentFactor, docprocOptions.documentExpansionFactor, docprocOptions.containerCoreMemory)); } - private void addSearch(DeployState deployState, Element spec, ApplicationContainerCluster cluster) { + private void addSearch(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) { Element searchElement = XML.getChild(spec, "search"); if (searchElement == null) return; addIncludes(searchElement); cluster.setSearch(buildSearch(deployState, cluster, searchElement)); - addSearchHandler(deployState, cluster, searchElement); + addSearchHandler(deployState, cluster, searchElement, context); validateAndAddConfiguredComponents(deployState, cluster, searchElement, "renderer", ContainerModelBuilder::validateRendererElement); } @@ -588,14 +588,14 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.addPlatformBundle(ContainerModelEvaluation.MODEL_INTEGRATION_BUNDLE_FILE); } - private void addProcessing(DeployState deployState, Element spec, ApplicationContainerCluster cluster) { + private void addProcessing(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) { Element processingElement = XML.getChild(spec, "processing"); if (processingElement == null) return; cluster.addSearchAndDocprocBundles(); addIncludes(processingElement); cluster.setProcessingChains(new DomProcessingBuilder(null).build(deployState, cluster, processingElement), - serverBindings(deployState, processingElement, ProcessingChains.defaultBindings).toArray(BindingPattern[]::new)); + serverBindings(deployState, context, processingElement, ProcessingChains.defaultBindings).toArray(BindingPattern[]::new)); validateAndAddConfiguredComponents(deployState, cluster, processingElement, "renderer", ContainerModelBuilder::validateRendererElement); } @@ -617,10 +617,11 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { containerSearch.setPageTemplates(PageTemplates.create(applicationPackage)); } - private void addUserHandlers(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { + private void addUserHandlers(DeployState deployState, ApplicationContainerCluster cluster, Element spec, ConfigModelContext context) { + OptionalInt portBindingOverride = isHostedTenantApplication(context) ? OptionalInt.of(HOSTED_VESPA_DATAPLANE_PORT) : OptionalInt.empty(); for (Element component: XML.getChildren(spec, "handler")) { cluster.addComponent( - new DomHandlerBuilder(cluster, OptionalInt.of(HOSTED_VESPA_DATAPLANE_PORT)).build(deployState, cluster, component)); + new DomHandlerBuilder(cluster, portBindingOverride).build(deployState, cluster, component)); } } @@ -879,13 +880,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { container.setPreLoad(nodesElement.getAttribute(VespaDomBuilder.PRELOAD_ATTRIB_NAME)); } - private void addSearchHandler(DeployState deployState, ApplicationContainerCluster cluster, Element searchElement) { + private void addSearchHandler(DeployState deployState, ApplicationContainerCluster cluster, Element searchElement, ConfigModelContext context) { BindingPattern bindingPattern = SearchHandler.DEFAULT_BINDING; - if (deployState.isHosted() && deployState.featureFlags().useRestrictedDataPlaneBindings()) { + if (isHostedTenantApplication(context) && deployState.featureFlags().useRestrictedDataPlaneBindings()) { bindingPattern = SearchHandler.bindingPattern(Optional.of(Integer.toString(HOSTED_VESPA_DATAPLANE_PORT))); } SearchHandler searchHandler = new SearchHandler(cluster, - serverBindings(deployState, searchElement, bindingPattern), + serverBindings(deployState, context, searchElement, bindingPattern), ContainerThreadpool.UserOptions.fromXml(searchElement).orElse(null)); cluster.addComponent(searchHandler); @@ -893,39 +894,39 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { searchHandler.addComponent(Component.fromClassAndBundle(SearchHandler.EXECUTION_FACTORY_CLASS, PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE)); } - private List<BindingPattern> serverBindings(DeployState deployState, Element searchElement, BindingPattern... defaultBindings) { + private List<BindingPattern> serverBindings(DeployState deployState, ConfigModelContext context, Element searchElement, BindingPattern... defaultBindings) { List<Element> bindings = XML.getChildren(searchElement, "binding"); if (bindings.isEmpty()) return List.of(defaultBindings); - return toBindingList(deployState, bindings); + return toBindingList(deployState, context, bindings); } - private List<BindingPattern> toBindingList(DeployState deployState, List<Element> bindingElements) { + private List<BindingPattern> toBindingList(DeployState deployState, ConfigModelContext context, List<Element> bindingElements) { List<BindingPattern> result = new ArrayList<>(); - OptionalInt port = deployState.isHosted() && deployState.featureFlags().useRestrictedDataPlaneBindings() ? OptionalInt.of(HOSTED_VESPA_DATAPLANE_PORT) : OptionalInt.empty(); + OptionalInt portOverride = isHostedTenantApplication(context) && deployState.featureFlags().useRestrictedDataPlaneBindings() ? OptionalInt.of(HOSTED_VESPA_DATAPLANE_PORT) : OptionalInt.empty(); for (Element element: bindingElements) { String text = element.getTextContent().trim(); if (!text.isEmpty()) - result.add(userBindingPattern(text, port)); + result.add(userBindingPattern(text, portOverride)); } return result; } - private static UserBindingPattern userBindingPattern(String path, OptionalInt port) { + private static UserBindingPattern userBindingPattern(String path, OptionalInt portOverride) { UserBindingPattern bindingPattern = UserBindingPattern.fromPattern(path); - return port.isPresent() - ? bindingPattern.withPort(port.getAsInt()) + return portOverride.isPresent() + ? bindingPattern.withPort(portOverride.getAsInt()) : bindingPattern; } - private ContainerDocumentApi buildDocumentApi(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { + private ContainerDocumentApi buildDocumentApi(DeployState deployState, ApplicationContainerCluster cluster, Element spec, ConfigModelContext context) { Element documentApiElement = XML.getChild(spec, "document-api"); if (documentApiElement == null) return null; ContainerDocumentApi.HandlerOptions documentApiOptions = DocumentApiOptionsBuilder.build(documentApiElement); Element ignoreUndefinedFields = XML.getChild(documentApiElement, "ignore-undefined-fields"); - OptionalInt portBindingOverride = deployState.featureFlags().useRestrictedDataPlaneBindings() && deployState.isHosted() + OptionalInt portBindingOverride = deployState.featureFlags().useRestrictedDataPlaneBindings() && isHostedTenantApplication(context) ? OptionalInt.of(HOSTED_VESPA_DATAPLANE_PORT) : OptionalInt.empty(); return new ContainerDocumentApi(cluster, documentApiOptions, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java index b9a81592ae4..39759743723 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.utils; import com.yahoo.config.FileReference; +import com.yahoo.config.ModelReference; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.producer.AbstractConfigProducer; @@ -17,6 +18,7 @@ import java.io.Serializable; import java.util.Collection; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.logging.Level; /** @@ -40,8 +42,7 @@ public class FileSender implements Serializable { * Sends all user configured files for a producer to all given services. */ public <PRODUCER extends AbstractConfigProducer<?>> void sendUserConfiguredFiles(PRODUCER producer) { - if (services.isEmpty()) - return; + if (services.isEmpty()) return; UserConfigRepo userConfigs = producer.getUserConfigs(); Map<Path, FileReference> sentFiles = new HashMap<>(); @@ -64,22 +65,22 @@ public class FileSender implements Serializable { return; } // Inspect fields at this level - sendEntries(builder, sentFiles, configDefinition.getFileDefs()); - sendEntries(builder, sentFiles, configDefinition.getPathDefs()); + sendEntries(builder, sentFiles, configDefinition.getFileDefs(), false); + sendEntries(builder, sentFiles, configDefinition.getPathDefs(), false); + sendEntries(builder, sentFiles, configDefinition.getModelDefs(), true); // Inspect arrays for (Map.Entry<String, ConfigDefinition.ArrayDef> entry : configDefinition.getArrayDefs().entrySet()) { - if (isFileOrPathArray(entry)) { - ConfigPayloadBuilder.Array array = builder.getArray(entry.getKey()); - sendFileEntries(array.getElements(), sentFiles); - } + if ( ! isAnyFileType(entry.getValue().getTypeSpec().getType())) continue; + ConfigPayloadBuilder.Array array = builder.getArray(entry.getKey()); + sendFileEntries(array.getElements(), sentFiles, "model".equals(entry.getValue().getTypeSpec().getType())); } - // Maps + + // Inspect maps for (Map.Entry<String, ConfigDefinition.LeafMapDef> entry : configDefinition.getLeafMapDefs().entrySet()) { - if (isFileOrPathMap(entry)) { - ConfigPayloadBuilder.MapBuilder map = builder.getMap(entry.getKey()); - sendFileEntries(map.getElements(), sentFiles); - } + if ( ! isAnyFileType(entry.getValue().getTypeSpec().getType())) continue; + ConfigPayloadBuilder.MapBuilder map = builder.getMap(entry.getKey()); + sendFileEntries(map.getElements(), sentFiles, "model".equals(entry.getValue().getTypeSpec().getType())); } // Inspect inner fields @@ -98,45 +99,56 @@ public class FileSender implements Serializable { sendUserConfiguredFiles(element, sentFiles, key); } } - - } - - private static boolean isFileOrPathMap(Map.Entry<String, ConfigDefinition.LeafMapDef> entry) { - String mapType = entry.getValue().getTypeSpec().getType(); - return ("file".equals(mapType) || "path".equals(mapType)); } - private static boolean isFileOrPathArray(Map.Entry<String, ConfigDefinition.ArrayDef> entry) { - String arrayType = entry.getValue().getTypeSpec().getType(); - return ("file".equals(arrayType) || "path".equals(arrayType)); + private static boolean isAnyFileType(String type) { + return "file".equals(type) || "path".equals(type) || "model".equals(type); } private void sendEntries(ConfigPayloadBuilder builder, Map<Path, FileReference> sentFiles, - Map<String, ? extends DefaultValued<String>> entries) { + Map<String, ?> entries, + boolean isModelType) { for (String name : entries.keySet()) { ConfigPayloadBuilder fileEntry = builder.getObject(name); if (fileEntry.getValue() == null) throw new IllegalArgumentException("Unable to send file for field '" + name + "': Invalid config value " + fileEntry.getValue()); - sendFileEntry(fileEntry, sentFiles); + sendFileEntry(fileEntry, sentFiles, isModelType); } } - private void sendFileEntries(Collection<ConfigPayloadBuilder> builders, Map<Path, FileReference> sentFiles) { + private void sendFileEntries(Collection<ConfigPayloadBuilder> builders, Map<Path, FileReference> sentFiles, boolean isModelType) { for (ConfigPayloadBuilder builder : builders) { - sendFileEntry(builder, sentFiles); + sendFileEntry(builder, sentFiles, isModelType); } } - private void sendFileEntry(ConfigPayloadBuilder builder, Map<Path, FileReference> sentFiles) { - Path path = Path.fromString(builder.getValue()); + private void sendFileEntry(ConfigPayloadBuilder builder, Map<Path, FileReference> sentFiles, boolean isModelType) { + Path path; + if (isModelType) { + var modelReference = ModelReference.valueOf(builder.getValue()); + if (modelReference.path().isEmpty()) return; + path = Path.fromString(modelReference.path().get().value()); + } + else { + path = Path.fromString(builder.getValue()); + } + FileReference reference = sentFiles.get(path); if (reference == null) { reference = fileRegistry.addFile(path.getRelative()); sentFiles.put(path, reference); } - builder.setValue(reference.value()); + + if (isModelType) { + var model = ModelReference.valueOf(builder.getValue()); + var modelWithReference = ModelReference.unresolved(model.modelId(), model.url(), Optional.of(reference)); + builder.setValue(modelWithReference.toString()); + } + else { + builder.setValue(reference.value()); + } } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/HandlerBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/HandlerBuilderTest.java index 186842ecbf1..6d61610a84f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/HandlerBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/HandlerBuilderTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.config.model.builder.xml.test.DomBuilderTest; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.ComponentsConfig; import com.yahoo.container.jdisc.JdiscBindingsConfig; import com.yahoo.container.usability.BindingsOverviewHandler; @@ -123,6 +124,19 @@ public class HandlerBuilderTest extends ContainerModelBuilderTestBase { } @Test + void does_not_restrict_infrastructure() { + DeployState deployState = new DeployState.Builder() + + .properties( + new TestProperties() + .setApplicationId(ApplicationId.defaultId()) + .setHostedVespa(true) + .setUseRestrictedDataPlaneBindings(false)) + .build(); + verifyDefaultBindings(deployState, "http://*"); + } + + @Test void restricts_custom_bindings_in_hosted_vespa() { DeployState deployState = new DeployState.Builder() .properties(new TestProperties().setHostedVespa(true).setUseRestrictedDataPlaneBindings(true)) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java index c7358ff1d7e..e113e53f541 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/utils/FileSenderTest.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.model.utils; import com.yahoo.config.FileNode; import com.yahoo.config.FileReference; +import com.yahoo.config.ModelReference; +import com.yahoo.config.UrlReference; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.producer.AbstractConfigProducer; @@ -22,6 +24,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -106,6 +109,42 @@ public class FileSenderTest { } @Test + void require_that_simple_model_field_with_just_path_is_modified() { + var originalValue = ModelReference.unresolved(new FileReference("myModel.onnx")); + def.addModelDef("modelVal"); + builder.setField("modelVal", originalValue.toString()); + assertFileSent("myModel.onnx", originalValue); + } + + @Test + void require_that_simple_model_field_with_path_and_url_is_modified() { + var originalValue = ModelReference.unresolved(Optional.empty(), + Optional.of(new UrlReference("myUrl")), + Optional.of(new FileReference("myModel.onnx"))); + def.addModelDef("modelVal"); + builder.setField("modelVal", originalValue.toString()); + assertFileSent("myModel.onnx", originalValue); + } + + @Test + void require_that_simple_model_field_with_just_url_is_not_modified() { + var originalValue = ModelReference.unresolved(new UrlReference("myUrl")); + def.addModelDef("modelVal"); + builder.setField("modelVal",originalValue.toString()); + fileSender().sendUserConfiguredFiles(producer); + assertEquals(originalValue, ModelReference.valueOf(builder.getObject("modelVal").getValue())); + } + + private void assertFileSent(String path, ModelReference originalValue) { + fileRegistry.pathToRef.put(path, new FileNode("myModelHash").value()); + fileSender().sendUserConfiguredFiles(producer); + var expected = ModelReference.unresolved(originalValue.modelId(), + originalValue.url(), + Optional.of(new FileReference("myModelHash"))); + assertEquals(expected, ModelReference.valueOf(builder.getObject("modelVal").getValue())); + } + + @Test void require_that_fields_in_inner_arrays_are_modified() { def.innerArrayDef("inner").addFileDef("fileVal"); def.innerArrayDef("inner").addStringDef("stringVal"); @@ -119,6 +158,25 @@ public class FileSenderTest { } @Test + void require_that_paths_and_model_fields_are_modified() { + def.arrayDef("fileArray").setTypeSpec(new ConfigDefinition.TypeSpec("fileArray", "file", null, null, null, null)); + def.arrayDef("pathArray").setTypeSpec(new ConfigDefinition.TypeSpec("pathArray", "path", null, null, null, null)); + def.arrayDef("stringArray").setTypeSpec(new ConfigDefinition.TypeSpec("stringArray", "string", null, null, null, null)); + builder.getArray("fileArray").append("foo.txt"); + builder.getArray("fileArray").append("bar.txt"); + builder.getArray("pathArray").append("path.txt"); + builder.getArray("stringArray").append("foo.txt"); + fileRegistry.pathToRef.put("foo.txt", new FileNode("foohash").value()); + fileRegistry.pathToRef.put("bar.txt", new FileNode("barhash").value()); + fileRegistry.pathToRef.put("path.txt", new FileNode("pathhash").value()); + fileSender().sendUserConfiguredFiles(producer); + assertEquals("foohash", builder.getArray("fileArray").get(0).getValue()); + assertEquals("barhash", builder.getArray("fileArray").get(1).getValue()); + assertEquals("pathhash", builder.getArray("pathArray").get(0).getValue()); + assertEquals("foo.txt", builder.getArray("stringArray").get(0).getValue()); + } + + @Test void require_that_arrays_are_modified() { def.arrayDef("fileArray").setTypeSpec(new ConfigDefinition.TypeSpec("fileArray", "file", null, null, null, null)); def.arrayDef("pathArray").setTypeSpec(new ConfigDefinition.TypeSpec("pathArray", "path", null, null, null, null)); diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java index 4f078059f46..6beb11cf8fa 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java @@ -249,13 +249,10 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { private ModelReference resolveModel(String modelStringValue) { var model = ModelReference.valueOf(modelStringValue); - // Resolve any of url and path present, in priority order - if (model.url().isPresent() && canResolveUrls()) { - model = new ModelReference(Path.of(resolveUrl(model.url().get().value()).value())); - } - else if (model.path().isPresent()) { - model = new ModelReference(Path.of(resolvePath(model.path().get().value()).value())); - } + if (model.url().isPresent() && canResolveUrls()) // url has priority + model = ModelReference.resolved(Path.of(resolveUrl(model.url().get().value()).value())); + else if (model.path().isPresent()) + model = ModelReference.resolved(Path.of(resolvePath(model.path().get().value()).value())); return model; } diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadApplierTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadApplierTest.java index aa517c943de..a982949e2fc 100644 --- a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadApplierTest.java +++ b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadApplierTest.java @@ -27,9 +27,9 @@ public class ConfigPayloadApplierTest { var inputConfig = new ResolvedTypesConfig.Builder(); inputConfig.myPath(new FileReference("myPath.txt")); inputConfig.myUrl(new UrlReference("myUrl.txt")); - inputConfig.myModel(new ModelReference(Optional.empty(), - Optional.of(new UrlReference("myUrl.txt")), - Optional.of(new FileReference("myPath.txt")))); + inputConfig.myModel(ModelReference.unresolved(Optional.empty(), + Optional.of(new UrlReference("myUrl.txt")), + Optional.of(new FileReference("myPath.txt")))); applier.applyPayload(ConfigPayload.fromInstance(inputConfig.build())); var config = configBuilder.build(); diff --git a/configdefinitions/src/vespa/zookeeper-server.def b/configdefinitions/src/vespa/zookeeper-server.def index 5cff46dd226..6b05e361307 100644 --- a/configdefinitions/src/vespa/zookeeper-server.def +++ b/configdefinitions/src/vespa/zookeeper-server.def @@ -48,3 +48,6 @@ snapshotMethod string default="gz" # Uses default Vespa mTLS config if empty string vespaTlsConfigFile string default="" + +leaderCloseSocketAsync bool default=false +learnerAsyncSending bool default=false diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index 6fae5c97cd2..6401945799b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -68,6 +68,7 @@ public class StatisticsSearcher extends Searcher { private static final String PEAK_QPS_METRIC = "peak_qps"; private static final String DOCS_COVERED_METRIC = "documents_covered"; private static final String DOCS_TOTAL_METRIC = "documents_total"; + private static final String DOCS_TARGET_TOTAL_METRIC = "documents_target_total"; private static final String DEGRADED_QUERIES_METRIC = "degraded_queries"; private static final String RELEVANCE_AT_1_METRIC = "relevance.at_1"; private static final String RELEVANCE_AT_3_METRIC = "relevance.at_3"; @@ -252,6 +253,7 @@ public class StatisticsSearcher extends Searcher { } metric.add(DOCS_COVERED_METRIC, queryCoverage.getDocs(), metricContext); metric.add(DOCS_TOTAL_METRIC, queryCoverage.getActive(), metricContext); + metric.add(DOCS_TARGET_TOTAL_METRIC, queryCoverage.getTargetActive(), metricContext); } int hitCount = result.getConcreteHitCount(); metric.set(HITS_PER_QUERY_METRIC, (double) hitCount, metricContext); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java index e0524091c39..fa28840d7ee 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java @@ -83,8 +83,8 @@ public interface ZoneRegistry { /** Returns all OS upgrade policies */ List<UpgradePolicy> osUpgradePolicies(); - /** Returns the routing methods supported by given zone, with the most preferred method appearing first */ - List<RoutingMethod> routingMethods(ZoneId zone); + /** Returns the routing method used by given zone */ + RoutingMethod routingMethod(ZoneId zone); /** Returns a URL where an informative dashboard can be found. */ URI dashboardUrl(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 92c3198175a..92ebc5b7177 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -695,9 +695,10 @@ public class ApplicationController { deploymentsToRemove.stream() .map(zone -> zone.region().value()) .collect(joining(", ")) + - ", but does not include " + - (deploymentsToRemove.size() > 1 ? "these zones" : "this zone") + - " in deployment.xml. " + + ", but " + (deploymentsToRemove.size() > 1 ? "these " : "this ") + + "instance and region combination" + + (deploymentsToRemove.size() > 1 ? "s are" : " is") + + " removed from deployment.xml. " + ValidationOverrides.toAllowMessage(ValidationId.deploymentRemoval)); // Remove the instance as well, if it is no longer referenced, and contains only production deployments that are removed now. boolean removeInstance = ! deploymentSpec.instanceNames().contains(instance) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java index 402a4bf49a8..d66d1491f73 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java @@ -220,6 +220,6 @@ public class Instance { @Override public String toString() { - return "application '" + id + "'"; + return "application instance '" + id.toFullString() + "'"; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 790f54b5e8c..071d8a4d11f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -110,10 +110,9 @@ public class RoutingController { // To discover the cluster name for a zone-scoped endpoint, we need to read routing policies for (var policy : routingPolicies.read(deployment)) { if (!policy.status().isActive()) continue; - for (var routingMethod : controller.zoneRegistry().routingMethods(policy.id().zone())) { - endpoints.addAll(policy.zoneEndpointsIn(controller.system(), routingMethod, controller.zoneRegistry())); - endpoints.add(policy.regionEndpointIn(controller.system(), routingMethod)); - } + RoutingMethod routingMethod = controller.zoneRegistry().routingMethod(policy.id().zone()); + endpoints.addAll(policy.zoneEndpointsIn(controller.system(), routingMethod, controller.zoneRegistry())); + endpoints.add(policy.regionEndpointIn(controller.system(), routingMethod)); } return EndpointList.copyOf(endpoints); } @@ -364,19 +363,18 @@ public class RoutingController { } private boolean usesSharedRouting(ZoneId zone) { - return controller.zoneRegistry().routingMethods(zone).stream().anyMatch(RoutingMethod::isShared); + return controller.zoneRegistry().routingMethod(zone).isShared(); } /** Returns the routing methods that are available across all given deployments */ private List<RoutingMethod> routingMethodsOfAll(Collection<DeploymentId> deployments) { - var deploymentsByMethod = new HashMap<RoutingMethod, Set<DeploymentId>>(); + Map<RoutingMethod, Set<DeploymentId>> deploymentsByMethod = new HashMap<>(); for (var deployment : deployments) { - for (var method : controller.zoneRegistry().routingMethods(deployment.zoneId())) { - deploymentsByMethod.computeIfAbsent(method, k -> new LinkedHashSet<>()) - .add(deployment); - } + RoutingMethod routingMethod = controller.zoneRegistry().routingMethod(deployment.zoneId()); + deploymentsByMethod.computeIfAbsent(routingMethod, k -> new LinkedHashSet<>()) + .add(deployment); } - var routingMethods = new ArrayList<RoutingMethod>(); + List<RoutingMethod> routingMethods = new ArrayList<>(); deploymentsByMethod.forEach((method, supportedDeployments) -> { if (supportedDeployments.containsAll(deployments)) { routingMethods.add(method); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index beba1cdb358..563c59630d1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -189,8 +189,12 @@ public class TenantController { } private void requireNonExistent(TenantName name) { + var tenant = get(name, true); + if (tenant.isPresent() && tenant.get().type().equals(Tenant.Type.deleted)) { + throw new IllegalArgumentException("Tenant '" + name + "' cannot be created, try a different name"); + } if (SystemApplication.TENANT.equals(name) - || get(name, true).isPresent() + || tenant.isPresent() // Underscores are allowed in existing tenant names, but tenants with - and _ cannot co-exist. E.g. // my-tenant cannot be created if my_tenant exists. || get(name.value().replace('-', '_')).isPresent()) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index 52eeaae1297..6b8ecc95cd9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -171,15 +171,17 @@ public class DeploymentStatus { } /** Returns change potentially with a compatibility platform added, if required for the change to roll out to the given instance. */ - public Change withPermittedPlatform(Change change, InstanceName instance, boolean allowOudatedPlatform) { + public Change withPermittedPlatform(Change change, InstanceName instance, boolean allowOutdatedPlatform) { Change augmented = withCompatibilityPlatform(change, instance); - if (allowOudatedPlatform) + if (allowOutdatedPlatform) return augmented; + // If compatibility platform is present, require that jobs have previously been run on that platform's major. + // If platform is not present, app is already on the (old) platform iff. it has production deployments. boolean alreadyDeployedOnPlatform = augmented.platform().map(platform -> allJobs.production().asList().stream() .anyMatch(job -> job.runs().values().stream() .anyMatch(run -> run.versions().targetPlatform().getMajor() == platform.getMajor()))) - .orElse(false); + .orElse( ! application.productionDeployments().values().stream().allMatch(List::isEmpty)); // Verify target platform is either current, or was previously deployed for this app. if (augmented.platform().isPresent() && ! versionStatus.isOnCurrentMajor(augmented.platform().get()) && ! alreadyDeployedOnPlatform) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java index 8a14dd3a146..b33a43a2031 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java @@ -4,8 +4,10 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.ApplicationList; +import com.yahoo.yolean.Exceptions; import java.time.Duration; +import java.util.logging.Logger; /** * Deploys application changes which have been postponed due to an ongoing upgrade, or a block window. @@ -14,19 +16,29 @@ import java.time.Duration; */ public class OutstandingChangeDeployer extends ControllerMaintainer { + private static final Logger logger = Logger.getLogger(OutstandingChangeDeployer.class.getName()); + public OutstandingChangeDeployer(Controller controller, Duration interval) { super(controller, interval); } @Override protected double maintain() { + double ok = 0, total = 0; for (Application application : ApplicationList.from(controller().applications().readable()) .withProductionDeployment() .withProjectId() .withDeploymentSpec() .asList()) - controller().applications().deploymentTrigger().triggerNewRevision(application.id()); - return 1.0; + try { + ++total; + controller().applications().deploymentTrigger().triggerNewRevision(application.id()); + ++ok; + } + catch (RuntimeException e) { + logger.info("Failed triggering new revision for " + application + ": " + Exceptions.toMessageString(e)); + } + return total > 0 ? ok / total : 1; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index 8e74ef9a983..000acd16155 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.zone.NodeSlice; -import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.Controller; @@ -78,8 +77,7 @@ public class SystemUpgrader extends InfrastructureUpgrader<VespaVersionTarget> { if (application.hasApplicationPackage()) { // For applications with package we do not have a zone-wide version target. This means that we must check // the wanted version of each node. - boolean zoneHasSharedRouting = controller().zoneRegistry().routingMethods(zone.getId()).stream() - .anyMatch(RoutingMethod::isShared); + boolean zoneHasSharedRouting = controller().zoneRegistry().routingMethod(zone.getId()).isShared(); return versionOf(NodeSlice.ALL, zone, application, Node::wantedVersion) .map(wantedVersion -> !wantedVersion.equals(target.version())) .orElse(zoneHasSharedRouting); // Always upgrade if zone uses shared routing, but has no nodes allocated yet diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index cb814a30c22..a1ecc07683e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -213,7 +213,7 @@ public class CuratorDb { } public Mutex lockProvisionState(String provisionStateId) { - return curator.lock(lockRoot.append(provisionStatePath()).append(provisionStateId), Duration.ofSeconds(1)); + return curator.lock(lockRoot.append("provisioning").append("states").append(provisionStateId), Duration.ofSeconds(1)); } public Mutex lockOsVersions() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index a5a09ab6551..ac29f8952a0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; @@ -203,11 +202,10 @@ public class RoutingPolicies { /** Compute region endpoints and their targets from given policies */ private Collection<RegionEndpoint> computeRegionEndpoints(List<RoutingPolicy> policies, Set<ZoneId> inactiveZones) { Map<Endpoint, RegionEndpoint> endpoints = new LinkedHashMap<>(); - RoutingMethod routingMethod = RoutingMethod.exclusive; for (var policy : policies) { if (policy.dnsZone().isEmpty()) continue; - if (!controller.zoneRegistry().routingMethods(policy.id().zone()).contains(routingMethod)) continue; - Endpoint regionEndpoint = policy.regionEndpointIn(controller.system(), routingMethod); + if (controller.zoneRegistry().routingMethod(policy.id().zone()) != RoutingMethod.exclusive) continue; + Endpoint regionEndpoint = policy.regionEndpointIn(controller.system(), RoutingMethod.exclusive); var zonePolicy = db.readZoneRoutingPolicy(policy.id().zone()); long weight = 1; if (isConfiguredOut(zonePolicy, policy, inactiveZones)) { @@ -437,12 +435,12 @@ public class RoutingPolicies { } private static boolean isActive(LoadBalancer loadBalancer) { - switch (loadBalancer.state()) { - case reserved: // Count reserved as active as we want callers (application API) to see the endpoint as early - // as possible - case active: return true; - } - return false; + return switch (loadBalancer.state()) { + // Count reserved as active as we want callers (application API) to see the endpoint as early + // as possible + case reserved, active -> true; + default -> false; + }; } /** Represents records for a region-wide endpoint */ @@ -556,10 +554,10 @@ public class RoutingPolicies { /** Returns the name updater to use for given zone */ private NameServiceForwarder nameServiceForwarderIn(ZoneId zone) { - if (controller.zoneRegistry().routingMethods(zone).contains(RoutingMethod.exclusive)) { - return controller.nameServiceForwarder(); - } - return new NameServiceDiscarder(controller.curator()); + return switch (controller.zoneRegistry().routingMethod(zone)) { + case exclusive -> controller.nameServiceForwarder(); + case sharedLayer4 -> new NameServiceDiscarder(controller.curator()); + }; } /** A {@link NameServiceForwarder} that does nothing. Used in zones where no explicit DNS updates are needed */ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 554bf2a57b4..c4e138c4d18 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -175,12 +175,13 @@ public class ControllerTest { fail("Expected exception due to illegal production deployment removal"); } catch (IllegalArgumentException e) { - assertEquals("deployment-removal: application 'tenant.application' is deployed in us-west-1, but does not include this zone in deployment.xml. " + - ValidationOverrides.toAllowMessage(ValidationId.deploymentRemoval), - e.getMessage()); + assertEquals("deployment-removal: application instance 'tenant.application.default' is deployed in us-west-1, " + + "but this instance and region combination is removed from deployment.xml. " + + ValidationOverrides.toAllowMessage(ValidationId.deploymentRemoval), + e.getMessage()); } assertNotNull(context.instance().deployments().get(productionUsWest1.zone()), - "Zone was not removed"); + "Zone was not removed"); // prod zone removal is allowed with override applicationPackage = new ApplicationPackageBuilder() diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 89b6f6ca606..f7d74373ac7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -2315,6 +2315,7 @@ public class DeploymentTriggerTest { .getMessage()); tester.deploymentTrigger().forceChange(old.instanceId(), Change.of(version0), false); + tester.deploymentTrigger().cancelChange(old.instanceId(), ALL); // Not even version incompatibility tricks the system. tester.controllerTester().flagSource().withListFlag(PermanentFlags.INCOMPATIBLE_VERSIONS.id(), List.of("7"), String.class); @@ -2326,11 +2327,17 @@ public class DeploymentTriggerTest { .build())) .getMessage()); + // Submit new revision on old major + old.submit(new ApplicationPackageBuilder().region("us-central-1").region("us-east-3").region("us-west-1") + .compileVersion(version0) + .build()) + .deploy(); + // Upgrade. old.submit(new ApplicationPackageBuilder().region("us-central-1").region("us-east-3").region("us-west-1") .compileVersion(version1) .build()) - .deploy(); + .deploy(); // And downgrade again. old.submit(new ApplicationPackageBuilder().region("us-central-1").region("us-east-3").region("us-west-1") diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java index 5ec949dfe0c..b27c206e215 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java @@ -25,18 +25,18 @@ import java.util.stream.Collectors; public class ZoneFilterMock implements ZoneList { private final List<ZoneApi> zones; - private final Map<ZoneApi, List<RoutingMethod>> zoneRoutingMethods; + private final Map<ZoneApi, RoutingMethod> zoneRoutingMethods; private final Set<ZoneApi> reprovisionToUpgradeOs; private final boolean negate; - private ZoneFilterMock(List<ZoneApi> zones, Map<ZoneApi, List<RoutingMethod>> zoneRoutingMethods, Set<ZoneApi> reprovisionToUpgradeOs, boolean negate) { + private ZoneFilterMock(List<ZoneApi> zones, Map<ZoneApi, RoutingMethod> zoneRoutingMethods, Set<ZoneApi> reprovisionToUpgradeOs, boolean negate) { this.zones = zones; this.zoneRoutingMethods = zoneRoutingMethods; this.reprovisionToUpgradeOs = reprovisionToUpgradeOs; this.negate = negate; } - public static ZoneFilter from(Collection<? extends ZoneApi> zones, Map<ZoneApi, List<RoutingMethod>> routingMethods, Set<ZoneApi> reprovisionToUpgradeOs) { + public static ZoneFilter from(Collection<? extends ZoneApi> zones, Map<ZoneApi, RoutingMethod> routingMethods, Set<ZoneApi> reprovisionToUpgradeOs) { return new ZoneFilterMock(List.copyOf(zones), Map.copyOf(routingMethods), reprovisionToUpgradeOs, false); } @@ -62,7 +62,7 @@ public class ZoneFilterMock implements ZoneList { @Override public ZoneList routingMethod(RoutingMethod method) { - return filter(zone -> zoneRoutingMethods.getOrDefault(zone, List.of()).contains(method)); + return filter(zone -> zoneRoutingMethods.get(zone) == method); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 0211a052f76..7f40172db2d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -32,6 +32,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -43,7 +44,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry private final Map<ZoneId, Duration> deploymentTimeToLive = new HashMap<>(); private final Map<Environment, RegionName> defaultRegionForEnvironment = new HashMap<>(); private final Map<CloudName, UpgradePolicy> osUpgradePolicies = new HashMap<>(); - private final Map<ZoneApi, List<RoutingMethod>> zoneRoutingMethods = new HashMap<>(); + private final Map<ZoneApi, RoutingMethod> zoneRoutingMethods = new HashMap<>(); private final Map<CloudAccount, Set<ZoneId>> cloudAccountZones = new HashMap<>(); private final Set<ZoneApi> reprovisionToUpgradeOs = new HashSet<>(); private final SystemName system; // Don't even think about making it non-final! ƪ(`▿▿▿▿´ƪ) @@ -125,17 +126,13 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry return setRoutingMethod(zones, RoutingMethod.exclusive); } - public ZoneRegistryMock setRoutingMethod(ZoneApi zone, RoutingMethod... routingMethods) { - return setRoutingMethod(zone, Set.of(routingMethods)); - } - - public ZoneRegistryMock setRoutingMethod(List<? extends ZoneApi> zones, RoutingMethod... routingMethods) { - zones.forEach(zone -> setRoutingMethod(zone, Set.of(routingMethods))); + public ZoneRegistryMock setRoutingMethod(List<? extends ZoneApi> zones, RoutingMethod routingMethod) { + zones.forEach(zone -> setRoutingMethod(zone, routingMethod)); return this; } - private ZoneRegistryMock setRoutingMethod(ZoneApi zone, Set<RoutingMethod> routingMethods) { - this.zoneRoutingMethods.put(zone, List.copyOf(routingMethods)); + public ZoneRegistryMock setRoutingMethod(ZoneApi zone, RoutingMethod routingMethod) { + this.zoneRoutingMethods.put(zone, routingMethod); return this; } @@ -214,8 +211,8 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry } @Override - public List<RoutingMethod> routingMethods(ZoneId zone) { - return List.copyOf(zoneRoutingMethods.getOrDefault(ZoneApiMock.from(zone), List.of())); + public RoutingMethod routingMethod(ZoneId zone) { + return Objects.requireNonNull(zoneRoutingMethods.get(ZoneApiMock.from(zone))); } @Override @@ -272,7 +269,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry @Override public Optional<String> getVipHostname(ZoneId zoneId) { - if (routingMethods(zoneId).stream().anyMatch(RoutingMethod::isShared)) { + if (routingMethod(zoneId).isShared()) { return Optional.of("vip." + zoneId.value()); } return Optional.empty(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index e3292700432..ed4f0597fad 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -925,7 +925,8 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1", POST).userIdentity(USER_ID) .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") .oAuthCredentials(OKTA_CREDENTIALS), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Tenant 'tenant1' already exists\"}", 400); + """ + {"error-code":"BAD_REQUEST","message":"Tenant 'tenant1' cannot be created, try a different name"}""", 400); // Forget a deleted tenant @@ -972,14 +973,14 @@ public class ApplicationApiTest extends ControllerContainerTest { // Invalid deployment fails tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/global-rotation", GET) .userIdentity(USER_ID), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1.instance1' has no deployment in prod.us-central-1\"}", + "{\"error-code\":\"NOT_FOUND\",\"message\":\"application instance 'tenant1.application1.instance1' has no deployment in prod.us-central-1\"}", 404); // Change status of non-existing deployment fails tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1/global-rotation/override", PUT) .userIdentity(USER_ID) .data("{\"reason\":\"unit-test\"}"), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1.instance1' has no deployment in prod.us-central-1\"}", + "{\"error-code\":\"NOT_FOUND\",\"message\":\"application instance 'tenant1.application1.instance1' has no deployment in prod.us-central-1\"}", 404); // GET global rotation status @@ -1037,7 +1038,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // GET global rotation status without specifying endpointId fails tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation", GET) .userIdentity(USER_ID), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"application 'tenant1.application1.instance1' has multiple rotations. Query parameter 'endpointId' must be given\"}", + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"application instance 'tenant1.application1.instance1' has multiple rotations. Query parameter 'endpointId' must be given\"}", 400); // GET global rotation status for us-west-1 in default endpoint diff --git a/default_build_settings.cmake b/default_build_settings.cmake index df55ce727f9..d306e68adff 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -2,19 +2,6 @@ include(VespaExtendedDefaultBuildSettings OPTIONAL) -function(setup_vespa_default_build_settings_rhel_6_10) - message("-- Setting up default build settings for rhel 6.10") - set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) - set(DEFAULT_CMAKE_SHARED_LINKER_FLAGS "-lrt" PARENT_SCOPE) -endfunction() - -function(setup_vespa_default_build_settings_rhel_7) - message("-- Setting up default build settings for rhel 7") - set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib64" "/usr/lib64/llvm7.0/lib" PARENT_SCOPE) - set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/llvm7.0" PARENT_SCOPE) - set(DEFAULT_VESPA_LLVM_VERSION "7" PARENT_SCOPE) -endfunction() - function(setup_vespa_default_build_settings_rhel_8) message("-- Setting up default build settings for rhel 8") set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" PARENT_SCOPE) @@ -78,13 +65,13 @@ endfunction() function(setup_vespa_default_build_settings_fedora_37) message("-- Setting up default build settings for fedora 37") set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) - set(DEFAULT_VESPA_LLVM_VERSION "14" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "15" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_fedora_38) message("-- Setting up default build settings for fedora 38") set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) - set(DEFAULT_VESPA_LLVM_VERSION "14" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "15" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_amzn_2022) @@ -194,13 +181,7 @@ function(vespa_use_default_build_settings) if(COMMAND vespa_use_specific_llvm_version) vespa_use_specific_llvm_version() endif() - if(VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 6.10") - setup_vespa_default_build_settings_rhel_6_10() - elseif(VESPA_OS_DISTRO STREQUAL "rhel" AND - VESPA_OS_DISTRO_VERSION VERSION_GREATER_EQUAL "7" AND - VESPA_OS_DISTRO_VERSION VERSION_LESS "8") - setup_vespa_default_build_settings_rhel_7() - elseif(VESPA_OS_DISTRO STREQUAL "rhel" AND + if(VESPA_OS_DISTRO STREQUAL "rhel" AND VESPA_OS_DISTRO_VERSION VERSION_GREATER_EQUAL "8" AND VESPA_OS_DISTRO_VERSION VERSION_LESS "9") setup_vespa_default_build_settings_rhel_8() diff --git a/dist/vespa.spec b/dist/vespa.spec index e07e5ba83a6..7c13031e0b3 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -158,14 +158,14 @@ BuildRequires: gmock-devel %endif %if 0%{?fc37} BuildRequires: protobuf-devel -BuildRequires: llvm-devel >= 14.0.5 +BuildRequires: llvm-devel >= 15.0.0 BuildRequires: boost-devel >= 1.78 BuildRequires: gtest-devel BuildRequires: gmock-devel %endif %if 0%{?fc38} BuildRequires: protobuf-devel -BuildRequires: llvm-devel >= 14.0.5 +BuildRequires: llvm-devel >= 15.0.0 BuildRequires: boost-devel >= 1.78 BuildRequires: gtest-devel BuildRequires: gmock-devel @@ -207,30 +207,30 @@ Requires: initscripts Requires: libcgroup-tools %endif Requires: numactl -Requires: perl -Requires: perl-Carp -Requires: perl-Data-Dumper -Requires: perl-Digest-MD5 -Requires: perl-Env -Requires: perl-Exporter -Requires: perl-File-Path -Requires: perl-File-Temp -Requires: perl-Getopt-Long -Requires: perl-IO-Socket-IP -Requires: perl-JSON -Requires: perl-libwww-perl -Requires: perl-LWP-Protocol-https +BuildRequires: perl +BuildRequires: perl-Carp +BuildRequires: perl-Data-Dumper +BuildRequires: perl-Digest-MD5 +BuildRequires: perl-Env +BuildRequires: perl-Exporter +BuildRequires: perl-File-Path +BuildRequires: perl-File-Temp +BuildRequires: perl-Getopt-Long +BuildRequires: perl-IO-Socket-IP +BuildRequires: perl-JSON +BuildRequires: perl-libwww-perl +BuildRequires: perl-LWP-Protocol-https %if ! 0%{?amzn2022} && ! 0%{?el9} -Requires: perl-Net-INET6Glue +BuildRequires: perl-Net-INET6Glue %endif -Requires: perl-Pod-Usage -Requires: perl-URI -Requires: valgrind +BuildRequires: perl-Pod-Usage +BuildRequires: perl-URI +BuildRequires: valgrind +BuildRequires: perf Requires: xxhash Requires: xxhash-libs >= 0.8.0 Requires: gdb Requires: hostname -Requires: perf Requires: nc Requires: nghttp2 Requires: net-tools @@ -273,10 +273,10 @@ Requires: gtest %define _vespa_llvm_version 14 %endif %if 0%{?fc37} -%define _vespa_llvm_version 14 +%define _vespa_llvm_version 15 %endif %if 0%{?fc38} -%define _vespa_llvm_version 14 +%define _vespa_llvm_version 15 %endif %define _extra_link_directory %{_vespa_deps_prefix}/lib64 %define _extra_include_directory %{_vespa_deps_prefix}/include;/usr/include/openblas @@ -310,8 +310,8 @@ Requires: java-17-amazon-corretto %else Requires: java-17-openjdk-devel %endif -Requires: perl -Requires: perl-Getopt-Long +BuildRequires: perl +BuildRequires: perl-Getopt-Long Requires(pre): shadow-utils %description base @@ -394,10 +394,10 @@ Requires: llvm-libs >= 13.0.0 Requires: llvm-libs >= 14.0.0 %endif %if 0%{?fc37} -Requires: llvm-libs >= 14.0.5 +Requires: llvm-libs >= 15.0.0 %endif %if 0%{?fc38} -Requires: llvm-libs >= 14.0.5 +Requires: llvm-libs >= 15.0.0 %endif %endif Requires: vespa-onnxruntime = 1.12.1 @@ -460,6 +460,19 @@ Requires: %{name}-base-libs = %{version}-%{release} Vespa - The open big data serving engine - tools +%package systemtest-tools + +Summary: Vespa - The open big data serving engine - tools for system tests + +Requires: %{name} = %{version}-%{release} +Requires: %{name}-base-libs = %{version}-%{release} +Requires: valgrind +Requires: perf + +%description systemtest-tools + +Vespa - The open big data serving engine - tools for system tests + %package ann-benchmark Summary: Vespa - The open big data serving engine - ann-benchmark @@ -644,6 +657,8 @@ fi %exclude %{_prefix}/bin/vespa-stat %exclude %{_prefix}/bin/vespa-security-env %exclude %{_prefix}/bin/vespa-summary-benchmark +%exclude %{_prefix}/bin/vespa-tensor-conformance +%exclude %{_prefix}/bin/vespa-tensor-instructions-benchmark %exclude %{_prefix}/bin/vespa-visit %exclude %{_prefix}/bin/vespa-visit-target %dir %{_prefix}/conf @@ -902,6 +917,15 @@ fi %dir %{_prefix}/lib/jars %{_prefix}/lib/jars/vespaclient-java-jar-with-dependencies.jar +%files systemtest-tools +%if %{_defattr_is_vespa_vespa} +%defattr(-,%{_vespa_user},%{_vespa_group},-) +%endif +%dir %{_prefix} +%dir %{_prefix}/bin +%{_prefix}/bin/vespa-tensor-conformance +%{_prefix}/bin/vespa-tensor-instructions-benchmark + %files ann-benchmark %if %{_defattr_is_vespa_vespa} %defattr(-,%{_vespa_user},%{_vespa_group},-) diff --git a/eval/src/tests/ann/doc_vector_access.h b/eval/src/tests/ann/doc_vector_access.h index 4d1460b8812..81ed436e274 100644 --- a/eval/src/tests/ann/doc_vector_access.h +++ b/eval/src/tests/ann/doc_vector_access.h @@ -1,11 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once + #include <vespa/vespalib/util/arrayref.h> +#include <cstdint> template <typename FltType = float> struct DocVectorAccess { virtual vespalib::ConstArrayRef<FltType> get(uint32_t docid) const = 0; - virtual ~DocVectorAccess() {} + virtual ~DocVectorAccess() = default; }; diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp index a3e4e3df7fa..158bc91dd6a 100644 --- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp +++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp @@ -16,9 +16,7 @@ #include <llvm/IR/DataLayout.h> #include <llvm/Transforms/Scalar.h> #include <llvm/Transforms/IPO/PassManagerBuilder.h> -#if LLVM_VERSION_MAJOR > 9 #include <llvm/Support/ManagedStatic.h> -#endif #include <vespa/eval/eval/check_type.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/util/approx.h> @@ -106,39 +104,35 @@ struct FunctionBuilder : public NodeVisitor, public NodeTraverser { return llvm::FunctionType::get(builder.getDoubleTy(), param_types, false); } - llvm::PointerType *make_eval_forest_funptr_t() { + llvm::FunctionType *make_eval_forest_fun_t() { std::vector<llvm::Type*> param_types; param_types.push_back(builder.getInt8Ty()->getPointerTo()); param_types.push_back(builder.getDoubleTy()->getPointerTo()); - llvm::FunctionType *function_type = llvm::FunctionType::get(builder.getDoubleTy(), param_types, false); - return llvm::PointerType::get(function_type, 0); + return llvm::FunctionType::get(builder.getDoubleTy(), param_types, false); } - llvm::PointerType *make_resolve_param_funptr_t() { + llvm::FunctionType *make_resolve_param_fun_t() { std::vector<llvm::Type*> param_types; param_types.push_back(builder.getInt8Ty()->getPointerTo()); param_types.push_back(builder.getInt64Ty()); - llvm::FunctionType *function_type = llvm::FunctionType::get(builder.getDoubleTy(), param_types, false); - return llvm::PointerType::get(function_type, 0); + return llvm::FunctionType::get(builder.getDoubleTy(), param_types, false); } - llvm::PointerType *make_eval_forest_proxy_funptr_t() { + llvm::FunctionType *make_eval_forest_proxy_fun_t() { std::vector<llvm::Type*> param_types; - param_types.push_back(make_eval_forest_funptr_t()); + param_types.push_back(llvm::PointerType::get(make_eval_forest_fun_t(), 0)); param_types.push_back(builder.getInt8Ty()->getPointerTo()); - param_types.push_back(make_resolve_param_funptr_t()); + param_types.push_back(llvm::PointerType::get(make_resolve_param_fun_t(), 0)); param_types.push_back(builder.getInt8Ty()->getPointerTo()); param_types.push_back(builder.getInt64Ty()); - llvm::FunctionType *function_type = llvm::FunctionType::get(builder.getDoubleTy(), param_types, false); - return llvm::PointerType::get(function_type, 0); + return llvm::FunctionType::get(builder.getDoubleTy(), param_types, false); } - llvm::PointerType *make_check_membership_funptr_t() { + llvm::FunctionType *make_check_membership_fun_t() { std::vector<llvm::Type*> param_types; param_types.push_back(builder.getInt8Ty()->getPointerTo()); param_types.push_back(builder.getDoubleTy()); - llvm::FunctionType *function_type = llvm::FunctionType::get(builder.getInt1Ty(), param_types, false); - return llvm::PointerType::get(function_type, 0); + return llvm::FunctionType::get(builder.getInt1Ty(), param_types, false); } FunctionBuilder(llvm::LLVMContext &context_in, @@ -170,7 +164,7 @@ struct FunctionBuilder : public NodeVisitor, public NodeTraverser { param_types.push_back(builder.getDoubleTy()->getPointerTo()); } else { assert(pass_params == PassParams::LAZY); - param_types.push_back(make_resolve_param_funptr_t()); + param_types.push_back(llvm::PointerType::get(make_resolve_param_fun_t(), 0)); param_types.push_back(builder.getInt8Ty()->getPointerTo()); } llvm::FunctionType *function_type = llvm::FunctionType::get(builder.getDoubleTy(), param_types, false); @@ -194,12 +188,12 @@ struct FunctionBuilder : public NodeVisitor, public NodeTraverser { } else if (pass_params == PassParams::ARRAY) { assert(params.size() == 1); llvm::Value *param_array = params[0]; - llvm::Value *addr = builder.CreateGEP(param_array->getType()->getScalarType()->getPointerElementType(), param_array, builder.getInt64(idx)); - return builder.CreateLoad(addr->getType()->getPointerElementType(), addr); + llvm::Value *addr = builder.CreateGEP(builder.getDoubleTy(), param_array, builder.getInt64(idx)); + return builder.CreateLoad(builder.getDoubleTy(), addr); } assert(pass_params == PassParams::LAZY); assert(params.size() == 2); - return builder.CreateCall(llvm::cast<llvm::FunctionType>(params[0]->getType()->getPointerElementType()), + return builder.CreateCall(make_resolve_param_fun_t(), params[0], {params[1], builder.getInt64(idx)}, "resolve_param"); } @@ -248,17 +242,19 @@ struct FunctionBuilder : public NodeVisitor, public NodeTraverser { forests.push_back(std::move(optimize_result.forest)); void *eval_ptr = (void *) optimize_result.eval; gbdt::Forest *forest = forests.back().get(); - llvm::PointerType *eval_funptr_t = make_eval_forest_funptr_t(); + llvm::FunctionType* eval_fun_t = make_eval_forest_fun_t(); + llvm::PointerType *eval_funptr_t = llvm::PointerType::get(eval_fun_t, 0); llvm::Value *eval_fun = builder.CreateIntToPtr(builder.getInt64((uint64_t)eval_ptr), eval_funptr_t, "inject_eval"); llvm::Value *ctx = builder.CreateIntToPtr(builder.getInt64((uint64_t)forest), builder.getInt8Ty()->getPointerTo(), "inject_ctx"); if (pass_params == PassParams::ARRAY) { - push(builder.CreateCall(llvm::cast<llvm::FunctionType>(eval_fun->getType()->getPointerElementType()), + push(builder.CreateCall(eval_fun_t, eval_fun, {ctx, params[0]}, "call_eval")); } else { assert(pass_params == PassParams::LAZY); - llvm::PointerType *proxy_funptr_t = make_eval_forest_proxy_funptr_t(); + llvm::FunctionType* proxy_fun_t = make_eval_forest_proxy_fun_t(); + llvm::PointerType *proxy_funptr_t = llvm::PointerType::get(proxy_fun_t, 0); llvm::Value *proxy_fun = builder.CreateIntToPtr(builder.getInt64((uint64_t)vespalib_eval_forest_proxy), proxy_funptr_t, "inject_eval_proxy"); - push(builder.CreateCall(llvm::cast<llvm::FunctionType>(proxy_fun->getType()->getPointerElementType()), + push(builder.CreateCall(proxy_fun_t, proxy_fun, {eval_fun, ctx, params[0], params[1], builder.getInt64(stats.num_params)})); } return true; @@ -341,7 +337,6 @@ struct FunctionBuilder : public NodeVisitor, public NodeTraverser { llvm::Value *a = pop_double(); push(builder.CreateCall(fun, a)); } -#if LLVM_VERSION_MAJOR >= 9 void make_call_1(llvm::FunctionCallee fun) { if (!fun || fun.getFunctionType()->getNumParams() != 1) { return make_error(1); @@ -349,16 +344,11 @@ struct FunctionBuilder : public NodeVisitor, public NodeTraverser { llvm::Value *a = pop_double(); push(builder.CreateCall(fun, a)); } -#endif void make_call_1(const llvm::Intrinsic::ID &id) { make_call_1(llvm::Intrinsic::getDeclaration(&module, id, builder.getDoubleTy())); } void make_call_1(const char *name) { -#if LLVM_VERSION_MAJOR >= 9 make_call_1(module.getOrInsertFunction(name, make_call_1_fun_t())); -#else - make_call_1(llvm::dyn_cast<llvm::Function>(module.getOrInsertFunction(name, make_call_1_fun_t()))); -#endif } void make_call_2(llvm::Function *fun) { @@ -369,7 +359,6 @@ struct FunctionBuilder : public NodeVisitor, public NodeTraverser { llvm::Value *a = pop_double(); push(builder.CreateCall(fun, {a, b})); } -#if LLVM_VERSION_MAJOR >= 9 void make_call_2(llvm::FunctionCallee fun) { if (!fun || fun.getFunctionType()->getNumParams() != 2) { return make_error(2); @@ -378,16 +367,11 @@ struct FunctionBuilder : public NodeVisitor, public NodeTraverser { llvm::Value *a = pop_double(); push(builder.CreateCall(fun, {a, b})); } -#endif void make_call_2(const llvm::Intrinsic::ID &id) { make_call_2(llvm::Intrinsic::getDeclaration(&module, id, builder.getDoubleTy())); } void make_call_2(const char *name) { -#if LLVM_VERSION_MAJOR >= 9 make_call_2(module.getOrInsertFunction(name, make_call_2_fun_t())); -#else - make_call_2(llvm::dyn_cast<llvm::Function>(module.getOrInsertFunction(name, make_call_2_fun_t()))); -#endif } //------------------------------------------------------------------------- @@ -410,10 +394,11 @@ struct FunctionBuilder : public NodeVisitor, public NodeTraverser { plugin_state.emplace_back(new SetMemberHash(item)); void *call_ptr = (void *) SetMemberHash::check_membership; PluginState *state = plugin_state.back().get(); - llvm::PointerType *funptr_t = make_check_membership_funptr_t(); + llvm::FunctionType *fun_t = make_check_membership_fun_t(); + llvm::PointerType *funptr_t = llvm::PointerType::get(fun_t, 0); llvm::Value *call_fun = builder.CreateIntToPtr(builder.getInt64((uint64_t)call_ptr), funptr_t, "inject_call_addr"); llvm::Value *ctx = builder.CreateIntToPtr(builder.getInt64((uint64_t)state), builder.getInt8Ty()->getPointerTo(), "inject_ctx"); - push(builder.CreateCall(llvm::cast<llvm::FunctionType>(call_fun->getType()->getPointerElementType()), + push(builder.CreateCall(fun_t, call_fun, {ctx, lhs}, "call_check_membership")); } else { // build explicit code to check all set members diff --git a/eval/src/vespa/eval/instruction/dense_lambda_peek_optimizer.cpp b/eval/src/vespa/eval/instruction/dense_lambda_peek_optimizer.cpp index b51d882d406..b2fe30d83e0 100644 --- a/eval/src/vespa/eval/instruction/dense_lambda_peek_optimizer.cpp +++ b/eval/src/vespa/eval/instruction/dense_lambda_peek_optimizer.cpp @@ -11,6 +11,7 @@ #include <vespa/eval/eval/call_nodes.h> #include <vespa/eval/eval/tensor_nodes.h> #include <vespa/eval/eval/llvm/compile_cache.h> +#include <vespa/vespalib/util/small_vector.h> #include <optional> using namespace vespalib::eval::nodes; diff --git a/eval/src/vespa/eval/instruction/dense_tensor_peek_function.h b/eval/src/vespa/eval/instruction/dense_tensor_peek_function.h index 517d8ffc6e1..496f4ceaed7 100644 --- a/eval/src/vespa/eval/instruction/dense_tensor_peek_function.h +++ b/eval/src/vespa/eval/instruction/dense_tensor_peek_function.h @@ -3,6 +3,7 @@ #pragma once #include <vespa/eval/eval/tensor_function.h> +#include <vespa/vespalib/util/small_vector.h> namespace vespalib::eval { @@ -23,7 +24,7 @@ private: SmallVector<std::pair<int64_t,size_t>> _spec; public: DenseTensorPeekFunction(std::vector<Child> children, SmallVector<std::pair<int64_t,size_t>> spec); - ~DenseTensorPeekFunction(); + ~DenseTensorPeekFunction() override; const ValueType &result_type() const override { return DoubleValue::shared_type(); } void push_children(std::vector<Child::CREF> &children) const override; InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; diff --git a/eval/src/vespa/eval/instruction/generic_concat.h b/eval/src/vespa/eval/instruction/generic_concat.h index 2fb5af88181..5b27988c5bd 100644 --- a/eval/src/vespa/eval/instruction/generic_concat.h +++ b/eval/src/vespa/eval/instruction/generic_concat.h @@ -6,7 +6,7 @@ #include <vespa/eval/eval/value_type.h> #include <vespa/eval/eval/interpreted_function.h> #include <vespa/vespalib/stllike/string.h> -#include <vector> +#include <vespa/vespalib/util/small_vector.h> namespace vespalib::eval { struct ValueBuilderFactory; } diff --git a/eval/src/vespa/eval/instruction/generic_join.cpp b/eval/src/vespa/eval/instruction/generic_join.cpp index 60dc3e55143..03c759c225f 100644 --- a/eval/src/vespa/eval/instruction/generic_join.cpp +++ b/eval/src/vespa/eval/instruction/generic_join.cpp @@ -51,7 +51,7 @@ generic_mixed_join(const Value &lhs, const Value &rhs, const JoinParam ¶m) } } return builder->build(std::move(builder)); -}; +} namespace { @@ -64,7 +64,7 @@ void my_mixed_join_op(State &state, uint64_t param_in) { auto &result = state.stash.create<std::unique_ptr<Value>>(std::move(up)); const Value &result_ref = *(result.get()); state.pop_pop_push(result_ref); -}; +} //----------------------------------------------------------------------------- @@ -95,7 +95,7 @@ void my_mixed_dense_join_op(State &state, uint64_t param_in) { assert(rhs == rhs_cells.end()); } state.pop_pop_push(state.stash.create<ValueView>(param.res_type, index, TypedCells(out_cells))); -}; +} //----------------------------------------------------------------------------- @@ -110,7 +110,7 @@ void my_dense_join_op(State &state, uint64_t param_in) { auto join_cells = [&](size_t lhs_idx, size_t rhs_idx) { *dst++ = fun(lhs_cells[lhs_idx], rhs_cells[rhs_idx]); }; param.dense_plan.execute(0, 0, join_cells); state.pop_pop_push(state.stash.create<DenseValueView>(param.res_type, TypedCells(out_cells))); -}; +} //----------------------------------------------------------------------------- @@ -119,7 +119,7 @@ void my_double_join_op(State &state, uint64_t param_in) { Fun fun(unwrap_param<JoinParam>(param_in).function); state.pop_pop_push(state.stash.create<DoubleValue>(fun(state.peek(1).as_double(), state.peek(0).as_double()))); -}; +} //----------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/instruction/generic_join.h b/eval/src/vespa/eval/instruction/generic_join.h index 72ae0b89d2c..7d9c6352868 100644 --- a/eval/src/vespa/eval/instruction/generic_join.h +++ b/eval/src/vespa/eval/instruction/generic_join.h @@ -6,6 +6,7 @@ #include <vespa/eval/eval/value_type.h> #include <vespa/eval/eval/operation.h> #include <vespa/eval/eval/interpreted_function.h> +#include <vespa/vespalib/util/small_vector.h> namespace vespalib { class Stash; } namespace vespalib::eval { struct ValueBuilderFactory; } @@ -64,7 +65,7 @@ struct SparseJoinPlan { bool should_forward_lhs_index() const; bool should_forward_rhs_index() const; SparseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type); - SparseJoinPlan(size_t num_mapped_dims); // full overlap plan + explicit SparseJoinPlan(size_t num_mapped_dims); // full overlap plan ~SparseJoinPlan(); }; diff --git a/eval/src/vespa/eval/instruction/generic_lambda.cpp b/eval/src/vespa/eval/instruction/generic_lambda.cpp index 2fe508fc310..1558d99b960 100644 --- a/eval/src/vespa/eval/instruction/generic_lambda.cpp +++ b/eval/src/vespa/eval/instruction/generic_lambda.cpp @@ -3,7 +3,7 @@ #include "generic_lambda.h" #include <vespa/eval/eval/llvm/compiled_function.h> #include <vespa/eval/eval/llvm/compile_cache.h> -#include <assert.h> +#include <vespa/vespalib/util/small_vector.h> using namespace vespalib::eval::tensor_function; diff --git a/eval/src/vespa/eval/instruction/generic_peek.cpp b/eval/src/vespa/eval/instruction/generic_peek.cpp index b1952cfefb7..a9900ce523e 100644 --- a/eval/src/vespa/eval/instruction/generic_peek.cpp +++ b/eval/src/vespa/eval/instruction/generic_peek.cpp @@ -8,6 +8,7 @@ #include <vespa/vespalib/util/typify.h> #include <vespa/vespalib/util/visit_ranges.h> #include <vespa/vespalib/util/shared_string_repo.h> +#include <vespa/vespalib/util/small_vector.h> #include <cassert> #include <map> diff --git a/eval/src/vespa/eval/instruction/generic_reduce.h b/eval/src/vespa/eval/instruction/generic_reduce.h index b9b6e7c5167..0e2a82ba6cc 100644 --- a/eval/src/vespa/eval/instruction/generic_reduce.h +++ b/eval/src/vespa/eval/instruction/generic_reduce.h @@ -6,6 +6,7 @@ #include <vespa/eval/eval/aggr.h> #include <vespa/eval/eval/interpreted_function.h> #include <vespa/eval/eval/nested_loop.h> +#include <vespa/vespalib/util/small_vector.h> namespace vespalib { class Stash; } namespace vespalib::eval { struct ValueBuilderFactory; } diff --git a/eval/src/vespa/eval/instruction/sparse_no_overlap_join_function.cpp b/eval/src/vespa/eval/instruction/sparse_no_overlap_join_function.cpp index f4808faadcc..8922e0da362 100644 --- a/eval/src/vespa/eval/instruction/sparse_no_overlap_join_function.cpp +++ b/eval/src/vespa/eval/instruction/sparse_no_overlap_join_function.cpp @@ -52,7 +52,7 @@ const Value &my_fast_no_overlap_sparse_join(const FastAddrMap &lhs_map, const Fa size_t addr_idx = store_rhs_idx[i]; output_addr[addr_idx] = r_addr[i]; } - result.add_mapping(ConstArrayRef(output_addr)); + result.add_mapping(output_addr); CT cell_value = fun(lhs_cells[lhs_subspace], rhs_cells[rhs_subspace]); result.my_cells.push_back_fast(cell_value); } @@ -109,7 +109,7 @@ SparseNoOverlapJoinFunction::compile_self(const ValueBuilderFactory &factory, St lhs().result_type(), rhs().result_type(), function(), factory); auto op = typify_invoke<2,MyTypify,SelectSparseNoOverlapJoinOp>(result_type().cell_meta().limit(), function()); - return InterpretedFunction::Instruction(op, wrap_param<JoinParam>(param)); + return {op, wrap_param<JoinParam>(param)}; } bool diff --git a/fbench/CMakeLists.txt b/fbench/CMakeLists.txt index 851d6706247..ff287d221ec 100644 --- a/fbench/CMakeLists.txt +++ b/fbench/CMakeLists.txt @@ -16,5 +16,3 @@ vespa_define_module( src/test src/test/authority ) - -vespa_install_script(util/resultfilter.pl vespa-fbench-result-filter.pl bin) diff --git a/fbench/README b/fbench/README index 8d217ad0dc5..17807cb3931 100644 --- a/fbench/README +++ b/fbench/README @@ -20,16 +20,8 @@ The above installation provides the follwing vespa-fbench executables: /opt/vespa/bin/vespa-fbench /opt/vespa/bin/vespa-fbench-filter-file /opt/vespa/bin/vespa-fbench-geturl - /opt/vespa/bin/vespa-fbench-result-filter.pl /opt/vespa/bin/vespa-fbench-split-file -Additional utilities referenced in this document can be fetched from -https://github.com/vespa-engine/vespa/tree/master/fbench/util: - plot.pl - pretest.sh - runtests.sh - separate.pl - It is also possible to use Docker to directly execute vespa-fbench by using the pre-built Vespa docker image: docker run --entrypoint /opt/vespa/bin/vespa-fbench \ @@ -251,90 +243,3 @@ results. This section will explain what each of these numbers mean. utilization will drop since the client has 'wasted' the time spent on the failed request. - - -6 Running test series ---------------------- - -For more complete benchmarking you will want to combine the results -from several test runs and present them together in a graph or maybe a -spreadsheet. The perl script vespa-fbench-result-filter.pl may be used to convert -the output from vespa-fbench into a single line of numbers. Lines of numbers -produced from several test runs may then be concatenated into the same -text file and used to plot a graph with gnuplot or imported into an -application accepting structured text files (like Excel). - -The task described above is performed by the runtests.sh script. It -runs vespa-fbench several times with varying client count and cycle -time. Between each test run, the script pretest.sh (located in the bin -directory) is run. The pretest.sh script should make sure that the -server you want to benchmark is in the same state before each of the -test runs. This typically means that the caches should be cleared. The -supplied pretest.sh file does nothing, and should therefore be -modified to fit your needs before you start benchmarking with the -runtests.sh script. NOTE: 'runtests.sh' must be run from the vespa-fbench -install directory in order to find the scripts and programs it depends -on. (vespa-fbench is run as 'bin/vespa-fbench' etc.). - -| usage: runtests.sh [-o] [-l] <minClients> <maxClients> <deltaClients> -| <minCycle> <maxCycle> <deltaCycle> [vespa-fbench options] <hostname> <port> -| -| The number of clients varies from <minClients> to <maxClients> with -| <deltaClients> increments. For each client count, the cycle time will -| vary in the same way according to <minCycle>, <maxCycle> and <deltaCycle>. -| vespa-fbench is run with each combination of client count and cycle time, and -| the result output is filtered with the 'vespa-fbench-result-filter.pl' script. -| If you want to save the results you should redirect stdout to a file. -| -| -o : change the order in which the tests are performed so that client -| count varies for each cycle time. -| -l : output a blank line between test subseries. If -o is not specified this -| will output a blank line between test series using different client count. -| If -o was specified this will output blank lines between test series -| using different cycle time. -| -| [vespa-fbench options] <hostname> <port>: These arguments are passed to vespa-fbench. -| There are 2 things to remember: first; do not specify either of the -n -| or -c options since they will override the values for client count and -| cycle time generated by this script. secondly; make sure you specify -| the correct host and port number. See the vespa-fbench usage (run vespa-fbench -| without parameters) for more info on how to invoke vespa-fbench. - -Example: You want to see how well fastserver performs with varying -client count and cycle time. Assume that you have already prepared 200 -query files. To test with client count -from 10 to 200 with intervals of 10 clients and cycle time from 0 to -5000 milliseconds with 500 ms intervals you may do the following: - -$ bin/runtests.sh 10 200 10 0 5000 500 <host> <port> - -The duration of each test run will be 60 seconds (the default). This -may be a little short. You will also get all results written directly -to your console. Say you want to run each test run for 5 minutes and -you want to collect the results in the file 'results.txt'. You may -then do the following: - -$ bin/runtests.sh 10 200 10 0 5000 500 -s 300 <host> <port> > result.txt - -The '-s 300' option will be given to vespa-fbench causing each test run to -have a duration of 300 seconds = 5 minutes. The standard output is -simply redirected to a file to collect the results for future use. - -The perl utility scripts separate.pl and plot.pl may be used to create -graphs using gnuplot. - -| usage: separate.pl <sepcol> -| Separate a tabular numeric file into chunks using a blank -| line whenever the value in column 'sepcol' changes. - -| usage: plot.pl [-h] [-x] <plotno> -| Plot the contents of 'result.txt'. -| -h This help -| -x Output to X11 window (default PS-file 'graph.ps') -| plotno: 1: Response Time Percentiles by NumCli -| 2: Rate by NumCli -| 3: Response Time Percentiles by Rate - -Note that the separate.pl script does the same thing as the -l option -of runtests.sh; it inserts blank lines into the result to let gnuplot -interpret each chunk as a separate dataseries. diff --git a/fbench/util/plot.pl b/fbench/util/plot.pl deleted file mode 100755 index 45311d9f97c..00000000000 --- a/fbench/util/plot.pl +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/perl -s -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -# TODO -# - parameter for input and output file name -# - more graphs - -sub usage { - die qq{usage: plot.pl [-h] [-x] <plotno> <format> -Plot the contents of 'result.txt' to 'graph.<format>'. - -h This help - -x Output to X11 window - plotno: 1: Response Time Percentiles by NumCli - 2: Rate by NumCli - 3: Response Time Percentiles by Rate - format: png (default), ps -}; -} - -$plotno = shift || die usage; -$term = shift || "png"; - -if ($h) { - usage; -} - -# setup the output -if ($x) { - # X11 output - open(PLOTSCRIPT, "| gnuplot -persist"); - print PLOTSCRIPT "set term X11\n"; - -} else { - open(PLOTSCRIPT, "| gnuplot"); - if ("$term" eq "ps") { - print PLOTSCRIPT "set term postscript\n"; - print PLOTSCRIPT "set output \"graph.ps\"\n"; - } - else { - print PLOTSCRIPT "set term png transparent small medium enhanced\n"; - print PLOTSCRIPT "set output \"graph.png\"\n"; - } -} -select(PLOTSCRIPT); - - - -# choose the graph -if ($plotno == 1) { - # Cli Percentile - print qq{ -set data style lines -set title "Response Time Percentiles by NumCli" -set xlabel "Number of clients" -set ylabel "Response time (msec)" -set key left top -plot 'result.txt' using 1:10 title "max", 'result.txt' using 1:17 title "99 %", 'result.txt' using 1:16 title "95 %", 'result.txt' using 1:15 title "90 %", 'result.txt' using 1:14 title "75 %", 'result.txt' using 1:13 title "50 %", 'result.txt' using 1:12 title "25 %", 'result.txt' using 1:9 title "min" - }; - -} elsif ($plotno == 2) { - # Cli Rate - print qq{ -set data style lines -set title "Rate by NumCli" -set xlabel "Number of clients" -set ylabel "Rate (queries/sec)" -set nokey -plot 'result.txt' using 1:18 - }; -} elsif ($plotno == 3) { - # Rate Percentile - print qq{ -set data style lines -set title "Response Time Percentiles by Rate" -set xlabel "Rate (queries/sec)" -set ylabel "Response time (msec)" -set key left top -plot 'result.txt' using 18:17 title "99 %", 'result.txt' using 18:16 title "95 %", 'result.txt' using 18:15 title "90 %", 'result.txt' using 18:14 title "75 %", 'result.txt' using 18:13 title "50 %", 'result.txt' using 18:12 title "25 %" - }; -} - -close(PLOTSCRIPT); diff --git a/fbench/util/pretest.sh b/fbench/util/pretest.sh deleted file mode 100755 index 1bf2a0372f8..00000000000 --- a/fbench/util/pretest.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -# -# This script will be run by the 'runtests.sh' script before -# each individual test run. - -# do not produce any output, log error messages to 'pretest.err' -exec > /dev/null 2>>pretest.err - diff --git a/fbench/util/resultfilter.pl b/fbench/util/resultfilter.pl deleted file mode 100755 index f66aed42dc3..00000000000 --- a/fbench/util/resultfilter.pl +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/perl -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -# This script converts an fbench summary report read from stdin to a -# single line containing only the numerical values written to -# stdout. - -while(<>) { - chomp(); - if(/:\s*([-+]?[\d.]+)/) { - print $1, " "; - } -} -print "\n"; diff --git a/fbench/util/runtests.sh b/fbench/util/runtests.sh deleted file mode 100755 index 1c5c7583e6e..00000000000 --- a/fbench/util/runtests.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/bin/sh -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -opt_o=false -opt_l=false - -opt_error=false - -while getopts "ol" option; do - case $option in - "o") opt_o=true;; - "l") opt_l=true;; - "*") opt_error=true;; - esac -done - -shift $(($OPTIND - 1)) -if [ $# -lt 8 ] || [ "$opt_error" = "true" ]; then - echo "usage: runtests.sh [-o] [-l] <minClients> <maxClients> <deltaClients>" - echo " <minCycle> <maxCycle> <deltaCycle> [vespa-fbench options] <hostname> <port>" - echo "" - echo "The number of clients varies from <minClients> to <maxClients> with" - echo "<deltaClients> increments. For each client count, the cycle time will" - echo "vary in the same way according to <minCycle>, <maxCycle> and <deltaCycle>." - echo "vespa-fbench is run with each combination of client count and cycle time, and" - echo "the result output is filtered with the 'vespa-fbench-result-filter.pl' script." - echo "If you want to save the results you should redirect stdout to a file." - echo "" - echo " -o : change the order in which the tests are performed so that client" - echo " count varies for each cycle time." - echo " -l : output a blank line between test subseries. If -o is not specified this" - echo " will output a blank line between test series using different client count." - echo " If -o was specified this will output blank lines between test series" - echo " using different cycle time." - echo "" - echo "[vespa-fbench options] <hostname> <port>: These arguments are passed to vespa-fbench." - echo " There are 2 things to remenber: first; do not specify either of the -n" - echo " or -c options since they will override the values for client count and" - echo " cycle time generated by this script. secondly; make sure you specify" - echo " the correct host and port number. See the vespa-fbench usage (run vespa-fbench" - echo " without parameters) for more info on how to invoke vespa-fbench." - exit 1 -fi - -minClients=$1; shift -maxClients=$1; shift -deltaClients=$1; shift -minCycle=$1; shift -maxCycle=$1; shift -deltaCycle=$1; shift - -if [ ! $deltaClients -gt 0 ]; then - echo "error: deltaClients must be greater than 0 !" - exit 1 -fi - -if [ ! $deltaCycle -gt 0 ]; then - echo "error: deltaCycle must be greater than 0 !" - exit 1 -fi - -echo "# vespa-fbench results collected by 'runtests.sh'." -echo "#" -echo "#1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20" -echo "#clients duration cycle lowlimit skip fail ok overtime min max avg 25% 50% 75% 90% 95% 99% rate util zerohit" -echo "#--------------------------------------------------------------------------------------------------" - -if [ "$opt_o" = "true" ]; then - cycle=$minCycle - while [ ! $cycle -gt $maxCycle ]; do - clients=$minClients - while [ ! $clients -gt $maxClients ]; do - test -f pretest.sh && ./pretest.sh > /dev/null 2>&1 - vespa-fbench -n $clients -c $cycle $@ | vespa-fbench-result-filter.pl - clients=$(($clients + $deltaClients)) - done - [ "$opt_l" = "true" ] && echo "" - cycle=$(($cycle + $deltaCycle)) - done -else - clients=$minClients - while [ ! $clients -gt $maxClients ]; do - cycle=$minCycle - while [ ! $cycle -gt $maxCycle ]; do - test -f pretest.sh && ./pretest.sh > /dev/null 2>&1 - vespa-fbench -n $clients -c $cycle $@ | vespa-fbench-result-filter.pl - cycle=$(($cycle + $deltaCycle)) - done - [ "$opt_l" = "true" ] && echo "" - clients=$(($clients + $deltaClients)) - done -fi diff --git a/fbench/util/separate.pl b/fbench/util/separate.pl deleted file mode 100755 index 418733bde7c..00000000000 --- a/fbench/util/separate.pl +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/perl -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -$sepcol = shift; - -if ($sepcol eq "") { - die qq{usage: separate.pl <sepcol> - Separate a tabular numeric file into chunks using a blank - line whenever the value in column 'sepcol' changes. -}; -} - -$oldval = -2; -$newval = -2; - -while (<>) { - if (/^#/) { - print; - } else { - chomp; - @vals = split; - $newval = $vals[$sepcol]; - if ($newval != $oldval) { - print "\n"; - $oldval = $newval; - } - print "@vals\n"; - } -} diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 12bf24eb737..092fb16d0cb 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -523,6 +523,20 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag ZOOKEEPER_LEADER_CLOSE_SOCKET_ASYNC = defineFeatureFlag( + "zookeeper-leader-close-socket-async", false, + List.of("hmusum"), "2022-09-19", "2022-11-01", + "Sets ZooKeeper config leader.closeSocketAsync", + "Takes effect when restarting zookeeper server", + ZONE_ID, APPLICATION_ID); + + public static final UnboundBooleanFlag ZOOKEEPER_LEARNER_ASYNC_SENDING = defineFeatureFlag( + "zookeeper-learner-async-sending", false, + List.of("hmusum"), "2022-09-19", "2022-11-01", + "Sets ZooKeeper config leader.closeSocketAsync", + "Takes effect when restarting zookeeper server", + ZONE_ID, APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/fsa/queryproc/permute_query.cpp b/fsa/queryproc/permute_query.cpp index 0e305ef7b75..7586252210e 100644 --- a/fsa/queryproc/permute_query.cpp +++ b/fsa/queryproc/permute_query.cpp @@ -7,12 +7,6 @@ #include "ngram.h" #include "base64.h" -#if (__GNUG__ <3 || (__GNUG__ == 3 && __GNUC_MINOR__ < 1)) -namespace std { -const char *fixed = ""; -} -#endif - using namespace fsa; unsigned int gram_count(unsigned int mg, unsigned int q) diff --git a/fsa/src/alltest/lookup_test.cpp b/fsa/src/alltest/lookup_test.cpp index d9ae17658d1..19880d7a0a3 100644 --- a/fsa/src/alltest/lookup_test.cpp +++ b/fsa/src/alltest/lookup_test.cpp @@ -7,12 +7,6 @@ #include <vespa/fsa/fsa.h> -#if (__GNUG__ <3 || (__GNUG__ == 3 && __GNUC_MINOR__ < 1)) -namespace std { -const char *left = ""; -} -#endif - using namespace fsa; int main(int argc, char** argv) diff --git a/fsa/src/vespa/fsa/automaton.cpp b/fsa/src/vespa/fsa/automaton.cpp index 57bcfb180a8..75aead62ba5 100644 --- a/fsa/src/vespa/fsa/automaton.cpp +++ b/fsa/src/vespa/fsa/automaton.cpp @@ -638,11 +638,7 @@ void Automaton::cleanUp() _register.clear(); delete _q0; _q0 = NULL; -#if ((__GNUG__ == 3 && __GNUC_MINOR__ >= 1) || __GNUG__ > 3) _previous_input.clear(); -#else - _previous_input = ""; -#endif } } diff --git a/fsa/src/vespa/fsa/fsa.h b/fsa/src/vespa/fsa/fsa.h index f3703b398ba..b6e5789e043 100644 --- a/fsa/src/vespa/fsa/fsa.h +++ b/fsa/src/vespa/fsa/fsa.h @@ -776,35 +776,8 @@ public: */ virtual ~HashedState() {} -#if ((__GNUG__ == 3 && __GNUC_MINOR__ >= 1) || __GNUG__ > 3) using State::start; using State::delta; -#else - virtual bool start(symbol_t in) { start(); return delta(in); } - virtual bool start(const symbol_t *in) { start(); return delta(in); } - virtual bool start(const char *in) { start(); return delta(in); } - virtual bool start(const std::string &in) { start(); return delta(in); } - virtual bool delta(const symbol_t *in) - { - const symbol_t *p=in; - while(*p && _state>0){ - delta(*p); - p++; - } - return _state!=0; - } - virtual bool delta(const char *in) { return delta((const symbol_t *)in); } - virtual bool delta(const std::string &in) - { - unsigned int idx=0; - - while(idx<in.length() && _state>0){ - delta(in[idx]); - idx++; - } - return _state!=0; - } -#endif /** * @brief Set the state to the starting state of the automaton. @@ -923,35 +896,8 @@ public: */ virtual ~CounterState() {} -#if ((__GNUG__ == 3 && __GNUC_MINOR__ >= 1) || __GNUG__ > 3) using State::start; using State::delta; -#else - virtual bool start(symbol_t in) { start(); return delta(in); } - virtual bool start(const symbol_t *in) { start(); return delta(in); } - virtual bool start(const char *in) { start(); return delta(in); } - virtual bool start(const std::string &in) { start(); return delta(in); } - virtual bool delta(const symbol_t *in) - { - const symbol_t *p=in; - while(*p && _state>0){ - delta(*p); - p++; - } - return _state!=0; - } - virtual bool delta(const char *in) { return delta((const symbol_t *)in); } - virtual bool delta(const std::string &in) - { - unsigned int idx=0; - - while(idx<in.length() && _state>0){ - delta(in[idx]); - idx++; - } - return _state!=0; - } -#endif /** * @brief Set the state to the starting state of the automaton. @@ -1240,35 +1186,8 @@ public: */ virtual ~MemoryState() {} -#if ((__GNUG__ == 3 && __GNUC_MINOR__ >= 1) || __GNUG__ > 3) using State::start; using State::delta; -#else - virtual bool start(symbol_t in) { start(); return delta(in); } - virtual bool start(const symbol_t *in) { start(); return delta(in); } - virtual bool start(const char *in) { start(); return delta(in); } - virtual bool start(const std::string &in) { start(); return delta(in); } - virtual bool delta(const symbol_t *in) - { - const symbol_t *p=in; - while(*p && _state>0){ - delta(*p); - p++; - } - return _state!=0; - } - virtual bool delta(const char *in) { return delta((const symbol_t *)in); } - virtual bool delta(const std::string &in) - { - unsigned int idx=0; - - while(idx<in.length() && _state>0){ - delta(in[idx]); - idx++; - } - return _state!=0; - } -#endif /** * @brief Set the state to the starting state of the automaton. @@ -1280,11 +1199,7 @@ public: */ bool start() override { -#if ((__GNUG__ == 3 && __GNUC_MINOR__ >= 1) || __GNUG__ > 3) _memory.clear(); -#else - _memory = ""; -#endif return State::start(); } @@ -1425,35 +1340,8 @@ public: */ virtual ~HashedMemoryState() {} -#if ((__GNUG__ == 3 && __GNUC_MINOR__ >= 1) || __GNUG__ > 3) using State::start; using State::delta; -#else - virtual bool start(symbol_t in) { start(); return delta(in); } - virtual bool start(const symbol_t *in) { start(); return delta(in); } - virtual bool start(const char *in) { start(); return delta(in); } - virtual bool start(const std::string &in) { start(); return delta(in); } - virtual bool delta(const symbol_t *in) - { - const symbol_t *p=in; - while(*p && _state>0){ - delta(*p); - p++; - } - return _state!=0; - } - virtual bool delta(const char *in) { return delta((const symbol_t *)in); } - virtual bool delta(const std::string &in) - { - unsigned int idx=0; - - while(idx<in.length() && _state>0){ - delta(in[idx]); - idx++; - } - return _state!=0; - } -#endif /** * @brief Set the state to the starting state of the automaton. @@ -1466,11 +1354,7 @@ public: bool start() override { _hash = 0; -#if ((__GNUG__ == 3 && __GNUC_MINOR__ >= 1) || __GNUG__ > 3) _memory.clear(); -#else - _memory = ""; -#endif return State::start(); } @@ -1606,35 +1490,8 @@ public: */ virtual ~HashedCounterState() {} -#if ((__GNUG__ == 3 && __GNUC_MINOR__ >= 1) || __GNUG__ > 3) using State::start; using State::delta; -#else - virtual bool start(symbol_t in) { start(); return delta(in); } - virtual bool start(const symbol_t *in) { start(); return delta(in); } - virtual bool start(const char *in) { start(); return delta(in); } - virtual bool start(const std::string &in) { start(); return delta(in); } - virtual bool delta(const symbol_t *in) - { - const symbol_t *p=in; - while(*p && _state>0){ - delta(*p); - p++; - } - return _state!=0; - } - virtual bool delta(const char *in) { return delta((const symbol_t *)in); } - virtual bool delta(const std::string &in) - { - unsigned int idx=0; - - while(idx<in.length() && _state>0){ - delta(in[idx]); - idx++; - } - return _state!=0; - } -#endif /** * @brief Set the state to the starting state of the automaton. @@ -1749,30 +1606,7 @@ public: uint32_t _counter; /**< Counter value. */ -#if ((__GNUG__ == 3 && __GNUC_MINOR__ >= 1) || __GNUG__ > 3) using State::delta; -#else - virtual bool delta(const symbol_t *in) - { - const symbol_t *p=in; - while(*p && _state>0){ - delta(*p); - p++; - } - return _state!=0; - } - virtual bool delta(const char *in) { return delta((const symbol_t *)in); } - virtual bool delta(const std::string &in) - { - unsigned int idx=0; - - while(idx<in.length() && _state>0){ - delta(in[idx]); - idx++; - } - return _state!=0; - } -#endif /** * @brief Delta transition for hashed word counter states. @@ -1933,17 +1767,6 @@ public: // }}} -#if (__GNUG__ < 3 || (__GNUG__ == 3 && __GNUC_MINOR__ < 1)) - friend class State; - friend class HashedState; - friend class MemoryState; - friend class HashedMemoryState; - friend class CounterState; - friend class HashedCounterState; - friend class WordCounterState; - friend class HashedWordCounterState; -#endif - public: /** * @brief Magic number for identifying fsa files. diff --git a/fsa/src/vespa/fsa/vectorizer.h b/fsa/src/vespa/fsa/vectorizer.h index 6ce94e92569..7e61c4f6d88 100644 --- a/fsa/src/vespa/fsa/vectorizer.h +++ b/fsa/src/vespa/fsa/vectorizer.h @@ -448,10 +448,6 @@ private: // }}} -#if (__GNUG__<3 || (__GNUG__ == 3 && __GNUC_MINOR__ < 1)) - friend RawVector::iterator; -#endif - private: /** diff --git a/fsa/src/vespa/fsamanagers/fsamanager.cpp b/fsa/src/vespa/fsamanagers/fsamanager.cpp index c26e49adedd..037e0f53f79 100644 --- a/fsa/src/vespa/fsamanagers/fsamanager.cpp +++ b/fsa/src/vespa/fsamanagers/fsamanager.cpp @@ -42,11 +42,7 @@ bool FSAManager::load(const std::string &id, const std::string &url) { std::string file=url; -#if ((__GNUG__ == 3 && __GNUC_MINOR__ >= 1) || __GNUG__ > 3) if(!url.compare(0,7,"http://")) -#else - if(!url.compare("http://",0,7)) -#endif { unsigned int pos=url.find_last_of('/'); if(pos==url.size()-1) return false; diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java index fd995c3fb78..1d96d8a0cdf 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java @@ -5,9 +5,7 @@ import com.yahoo.component.annotation.Inject; import com.yahoo.language.Linguistics; import com.yahoo.language.detect.Detector; import com.yahoo.language.process.Tokenizer; -import com.yahoo.language.simple.SimpleDetector; import com.yahoo.language.simple.SimpleLinguistics; -import opennlp.tools.langdetect.LanguageDetectorModel; /** * Returns a linguistics implementation based on OpenNlp. diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 062a7ce018d..49d00c03d23 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -226,7 +226,7 @@ public class StorageMaintainer { String output = uncheck(() -> Files.readAllLines(Paths.get("/proc/cpuinfo")).stream() .filter(line -> line.startsWith("microcode")) .findFirst() - .orElseThrow(() -> ConvergenceException.ofError("No microcode information found in /proc/cpuinfo"))); + .orElse("microcode : UNKNOWN")); String[] results = output.split(":"); if (results.length != 2) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ServiceDumpReport.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ServiceDumpReport.java index 452f786301b..7ce287a83f1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ServiceDumpReport.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/ServiceDumpReport.java @@ -89,10 +89,15 @@ class ServiceDumpReport extends BaseReport { } public static ServiceDumpReport createErrorReport( - ServiceDumpReport request, Instant startedAt, Instant failedAt, String message) { + ServiceDumpReport reqOrNull, Instant startedAt, Instant failedAt, String message) { + Long createdAt = reqOrNull != null ? reqOrNull.getCreatedMillisOrNull() : Long.valueOf(startedAt.toEpochMilli()); + String configId = reqOrNull != null ? reqOrNull.configId() : "unknown"; + Long expireAt = reqOrNull != null ? reqOrNull.expireAt() : null; + List<String> artifacts = reqOrNull != null ? reqOrNull.artifacts() : List.of(); + DumpOptions dumpOptions = reqOrNull != null ? reqOrNull.dumpOptions() : null; return new ServiceDumpReport( - request.getCreatedMillisOrNull(), startedAt.toEpochMilli(), null, failedAt.toEpochMilli(), null, - request.configId(), request.expireAt(), message, request.artifacts(), request.dumpOptions()); + createdAt, startedAt.toEpochMilli(), null, failedAt.toEpochMilli(), null, + configId, expireAt, message, artifacts, dumpOptions); } @JsonGetter(STARTED_AT_FIELD) public Long startedAt() { return startedAt; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java index d942e5c9e80..3a0cd412a2e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; import com.yahoo.yolean.concurrent.Sleeper; +import java.io.UncheckedIOException; import java.net.URI; import java.time.Clock; import java.time.Instant; @@ -66,8 +67,14 @@ public class VespaServiceDumperImpl implements VespaServiceDumper { Instant startedAt = clock.instant(); NodeSpec nodeSpec = context.node(); - ServiceDumpReport request = nodeSpec.reports().getReport(ServiceDumpReport.REPORT_ID, ServiceDumpReport.class) - .orElse(null); + ServiceDumpReport request; + try { + request = nodeSpec.reports().getReport(ServiceDumpReport.REPORT_ID, ServiceDumpReport.class) + .orElse(null); + } catch (IllegalArgumentException | UncheckedIOException e) { + handleFailure(context, null, startedAt, e, "Invalid JSON in service dump request"); + return; + } if (request == null || request.isCompletedOrFailed()) { context.log(log, Level.FINE, "No service dump requested or dump already completed/failed"); return; @@ -114,7 +121,7 @@ public class VespaServiceDumperImpl implements VespaServiceDumper { uploadArtifacts(context, destination, producedArtifacts); storeReport(context, ServiceDumpReport.createSuccessReport(request, startedAt, clock.instant(), destination)); } catch (Exception e) { - handleFailure(context, request, startedAt, e); + handleFailure(context, request, startedAt, e, e.getMessage()); } finally { if (unixPathDirectory.exists()) { context.log(log, Level.INFO, "Deleting directory '" + unixPathDirectory +"'."); @@ -147,15 +154,16 @@ public class VespaServiceDumperImpl implements VespaServiceDumper { : Instant.ofEpochMilli(request.expireAt()); } - private void handleFailure(NodeAgentContext context, ServiceDumpReport request, Instant startedAt, Exception failure) { + private void handleFailure(NodeAgentContext context, ServiceDumpReport requestOrNull, Instant startedAt, + Exception failure, String message) { context.log(log, Level.WARNING, failure.toString(), failure); - ServiceDumpReport report = ServiceDumpReport.createErrorReport(request, startedAt, clock.instant(), failure.toString()); + ServiceDumpReport report = ServiceDumpReport.createErrorReport(requestOrNull, startedAt, clock.instant(), message); storeReport(context, report); } - private void handleFailure(NodeAgentContext context, ServiceDumpReport request, Instant startedAt, String message) { + private void handleFailure(NodeAgentContext context, ServiceDumpReport requestOrNull, Instant startedAt, String message) { context.log(log, Level.WARNING, message); - ServiceDumpReport report = ServiceDumpReport.createErrorReport(request, startedAt, clock.instant(), message); + ServiceDumpReport report = ServiceDumpReport.createErrorReport(requestOrNull, startedAt, clock.instant(), message); storeReport(context, report); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java index fcdef83e06f..081f0038e06 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java @@ -1,6 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; @@ -170,6 +173,32 @@ class VespaServiceDumperImplTest { assertSyncedFiles(context, syncClient, expectedUris); } + @Test + void fails_gracefully_on_invalid_request_json() { + // Setup mocks + ContainerOperations operations = mock(ContainerOperations.class); + SyncClient syncClient = createSyncClientMock(); + NodeRepoMock nodeRepository = new NodeRepoMock(); + ManualClock clock = new ManualClock(Instant.ofEpochMilli(1600001000000L)); + JsonNodeFactory fac = new ObjectMapper().getNodeFactory(); + ObjectNode invalidRequest = new ObjectNode(fac) + .set("dumpOptions", new ObjectNode(fac).put("duration", "invalidDurationDataType")); + NodeSpec spec = NodeSpec.Builder + .testSpec(HOSTNAME, NodeState.active) + .report(ServiceDumpReport.REPORT_ID, invalidRequest) + .build(); + nodeRepository.updateNodeSpec(spec); + VespaServiceDumper reporter = new VespaServiceDumperImpl( + ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, clock); + NodeAgentContextImpl context = NodeAgentContextImpl.builder(spec) + .fileSystem(fileSystem) + .build(); + reporter.processServiceDumpRequest(context); + String expectedJson = "{\"createdMillis\":1600001000000,\"startedAt\":1600001000000,\"failedAt\":1600001000000," + + "\"configId\":\"unknown\",\"error\":\"Invalid JSON in service dump request\",\"artifacts\":[]}"; + assertReportEquals(nodeRepository, expectedJson); + } + private static NodeSpec createNodeSpecWithDumpRequest(NodeRepoMock repository, List<String> artifacts, ServiceDumpReport.DumpOptions options) { ServiceDumpReport request = ServiceDumpReport.createRequestReport( diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index 947a000eecf..75616d8d380 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -139,12 +139,9 @@ public class CapacityPolicies { : new NodeResources(0.5, 2, 50, 0.3); } - /** - * Returns whether the nodes requested can share physical host with other applications. - * A security feature which only makes sense for prod. - */ + /** Returns whether the nodes requested can share physical host with other applications */ public boolean decideExclusivity(Capacity capacity, boolean requestedExclusivity) { - if (zone.environment() == Environment.prod && capacity.cloudAccount().isPresent()) return true; // Implicit exclusive when using custom cloud account + if (capacity.cloudAccount().isPresent()) return true; // Implicit exclusive when using custom cloud account return requestedExclusivity && (capacity.isRequired() || zone.environment() == Environment.prod); } diff --git a/screwdriver/release-rpms.sh b/screwdriver/release-rpms.sh index 5de292eb9c5..09ea568ddd8 100755 --- a/screwdriver/release-rpms.sh +++ b/screwdriver/release-rpms.sh @@ -12,11 +12,14 @@ fi readonly VESPA_RELEASE="$1" readonly VESPA_REF="$2" -VESPA_RPM=$(dnf repoquery --repofrompath=vespa,https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/centos-stream-8-x86_64 --repoid=vespa -q vespa | cut -d: -f2 | cut -d- -f1 | sort -V | tail -1) -echo "Latest RPM on Copr: $VESPA_RPM" +VESPA_RPM_X86_64=$(dnf repoquery --repofrompath=vespa,https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/centos-stream-8-x86_64 --repoid=vespa -q vespa | cut -d: -f2 | cut -d- -f1 | sort -V | tail -1) +echo "Latest x86_64 RPM on Copr: $VESPA_RPM_X86_64" -if [ "$VESPA_RELEASE" == "$VESPA_RPM" ]; then - echo "Vespa rpm for version $VESPA_RELEASE already exists. Exiting." +VESPA_RPM_AARCH64=$(dnf repoquery --repofrompath=vespa,https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/centos-stream-8-aarch64 --repoid=vespa -q vespa | cut -d: -f2 | cut -d- -f1 | sort -V | tail -1) +echo "Latest aarch64 RPM on Copr: $VESPA_RPM_AARCH64" + +if [[ "$VESPA_RELEASE" == "$VESPA_RPM_X86_64" ]] && [[ "$VESPA_RELEASE" == "$VESPA_RPM_AARCH64" ]]; then + echo "Vespa RPMs for version $VESPA_RELEASE already exists. Exiting." exit 0 fi @@ -31,9 +34,12 @@ git clone git@github.com:vespa-engine/vespa cd vespa dist/release-vespa-rpm.sh $VESPA_RELEASE $VESPA_REF -while [ "$VESPA_RELEASE" != "$VESPA_RPM" ]; do +while [[ "$VESPA_RELEASE" != "$VESPA_RPM_X86_64" ]] || [[ "$VESPA_RELEASE" != "$VESPA_RPM_AARCH64" ]] ; do dnf clean --repofrompath=vespa,https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/centos-stream-8-x86_64 --repoid=vespa metadata - VESPA_RPM=$(dnf repoquery --repofrompath=vespa,https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/centos-stream-8-x86_64 --repoid=vespa -q vespa | cut -d: -f2 | cut -d- -f1 | sort -V | tail -1) - echo "RPM: $VESPA_RPM" + VESPA_RPM_X86_64=$(dnf repoquery --repofrompath=vespa,https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/centos-stream-8-x86_64 --repoid=vespa -q vespa | cut -d: -f2 | cut -d- -f1 | sort -V | tail -1) + echo "RPM x86_64: $VESPA_RPM_X86_64" + dnf clean --repofrompath=vespa,https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/centos-stream-8-aarch64 --repoid=vespa metadata + VESPA_RPM_AARCH64=$(dnf repoquery --repofrompath=vespa,https://copr-be.cloud.fedoraproject.org/results/@vespa/vespa/centos-stream-8-aarch64 --repoid=vespa -q vespa | cut -d: -f2 | cut -d- -f1 | sort -V | tail -1) + echo "RPM aarch64: $VESPA_RPM_AARCH64" sleep 150 done diff --git a/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp b/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp index f2433b2adf6..d7904881e4d 100644 --- a/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp @@ -7,6 +7,7 @@ #include <vespa/searchcore/proton/common/i_document_type_inspector.h> #include <vespa/searchcore/proton/common/indexschema_inspector.h> #include <vespa/searchcore/proton/test/attribute_utils.h> +#include <vespa/searchsummary/docsummary/docsum_field_writer_commands.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/test/insertion_operators.h> @@ -14,13 +15,15 @@ #include <vespa/log/log.h> LOG_SETUP("attibute_aspect_delayer_test"); +using search::attribute::Config; using vespa::config::search::AttributesConfig; using vespa::config::search::AttributesConfigBuilder; using vespa::config::search::IndexschemaConfig; using vespa::config::search::IndexschemaConfigBuilder; using vespa::config::search::SummaryConfig; using vespa::config::search::SummaryConfigBuilder; -using search::attribute::Config; + +using namespace search::docsummary; namespace vespa::config::search::internal { @@ -192,15 +195,17 @@ TEST_F(DelayerTest, require_that_empty_config_is_ok) TEST_F(DelayerTest, require_that_simple_attribute_config_is_ok) { - setup(attrCfg({make_int32_sv_cfg()}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer", "attribute", "a")})); + setup(attrCfg({make_int32_sv_cfg()}), attrCfg({make_int32_sv_cfg()}), + sCfg({make_summary_field("a", "integer", command::attribute, "a")})); assertAttributeConfig({make_int32_sv_cfg()}); - assertSummaryConfig({make_summary_field("a", "integer", "attribute", "a")}); + assertSummaryConfig({make_summary_field("a", "integer", command::attribute, "a")}); } TEST_F(DelayerTest, require_that_adding_attribute_aspect_is_delayed_if_field_type_is_unchanged) { addFields({"a"}); - setup(attrCfg({}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer", "attribute", "a")})); + setup(attrCfg({}), attrCfg({make_int32_sv_cfg()}), + sCfg({make_summary_field("a", "integer", command::attribute, "a")})); assertAttributeConfig({}); assertSummaryConfig({make_summary_field("a", "integer")}); } @@ -208,24 +213,27 @@ TEST_F(DelayerTest, require_that_adding_attribute_aspect_is_delayed_if_field_typ TEST_F(DelayerTest, require_that_adding_attribute_aspect_is_delayed_if_field_type_is_unchanged_geopos_override) { addFields({"a"}); - setup(attrCfg({}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer", "geopos", "a")})); + setup(attrCfg({}), attrCfg({make_int32_sv_cfg()}), + sCfg({make_summary_field("a", "integer", command::geo_position, "a")})); assertAttributeConfig({}); - assertSummaryConfig({make_summary_field("a", "integer", "geopos", "a")}); + assertSummaryConfig({make_summary_field("a", "integer", command::geo_position, "a")}); } TEST_F(DelayerTest, require_that_adding_attribute_aspect_is_delayed_if_field_type_is_unchanged_mapped_summary) { addFields({"a"}); - setup(attrCfg({}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a_mapped", "integer", "attribute", "a")})); + setup(attrCfg({}), attrCfg({make_int32_sv_cfg()}), + sCfg({make_summary_field("a_mapped", "integer", command::attribute, "a")})); assertAttributeConfig({}); - assertSummaryConfig({make_summary_field("a_mapped", "integer", "copy", "a")}); + assertSummaryConfig({make_summary_field("a_mapped", "integer", command::copy, "a")}); } TEST_F(DelayerTest, require_that_adding_attribute_is_not_delayed_if_field_type_changed) { - setup(attrCfg({}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer", "attribute", "a")})); + setup(attrCfg({}), attrCfg({make_int32_sv_cfg()}), + sCfg({make_summary_field("a", "integer", command::attribute, "a")})); assertAttributeConfig({make_int32_sv_cfg()}); - assertSummaryConfig({make_summary_field("a", "integer", "attribute", "a")}); + assertSummaryConfig({make_summary_field("a", "integer", command::attribute, "a")}); } TEST_F(DelayerTest, require_that_removing_attribute_aspect_is_delayed_if_field_type_is_unchanged) @@ -233,7 +241,7 @@ TEST_F(DelayerTest, require_that_removing_attribute_aspect_is_delayed_if_field_t addFields({"a"}); setup(attrCfg({make_int32_sv_cfg()}), attrCfg({}), sCfg({make_summary_field("a", "integer")})); assertAttributeConfig({make_int32_sv_cfg()}); - assertSummaryConfig({make_summary_field("a", "integer", "attribute", "a")}); + assertSummaryConfig({make_summary_field("a", "integer", command::attribute, "a")}); } TEST_F(DelayerTest, require_that_summary_map_override_is_removed_when_summary_aspect_is_removed_even_if_removing_attribute_aspect_is_delayed) @@ -272,7 +280,8 @@ TEST_F(DelayerTest, require_that_adding_attribute_aspect_is_delayed_for_tensor_f { addFields({"a"}); setup(attrCfg({}), - attrCfg({make_tensor_cfg("tensor(x[10])")}), sCfg({make_summary_field("a", "tensor", "attribute", "a")})); + attrCfg({make_tensor_cfg("tensor(x[10])")}), + sCfg({make_summary_field("a", "tensor", command::attribute, "a")})); assertAttributeConfig({}); assertSummaryConfig({make_summary_field("a", "tensor")}); } @@ -283,7 +292,7 @@ TEST_F(DelayerTest, require_that_removing_attribute_aspect_is_delayed_for_tensor setup(attrCfg({make_tensor_cfg("tensor(x[10])")}), attrCfg({}), sCfg({make_summary_field("a", "tensor")})); assertAttributeConfig({make_tensor_cfg("tensor(x[10])")}); - assertSummaryConfig({make_summary_field("a", "tensor", "attribute", "a")}); + assertSummaryConfig({make_summary_field("a", "tensor", command::attribute, "a")}); } TEST_F(DelayerTest, require_that_removing_attribute_aspect_is_not_delayed_for_predicate) @@ -305,56 +314,63 @@ TEST_F(DelayerTest, require_that_removing_attribute_aspect_is_not_delayed_for_re TEST_F(DelayerTest, require_that_fast_access_flag_change_is_delayed_false_true_edge) { addFields({"a"}); - setup(attrCfg({make_int32_sv_cfg()}), attrCfg({make_fa(make_int32_sv_cfg())}), sCfg({make_summary_field("a", "integer", "attribute", "a")})); + setup(attrCfg({make_int32_sv_cfg()}), attrCfg({make_fa(make_int32_sv_cfg())}), + sCfg({make_summary_field("a", "integer", command::attribute, "a")})); assertAttributeConfig({make_int32_sv_cfg()}); - assertSummaryConfig({make_summary_field("a", "integer", "attribute", "a")}); + assertSummaryConfig({make_summary_field("a", "integer", command::attribute, "a")}); } TEST_F(DelayerTest, require_that_fast_access_flag_change_is_delayed_true_false_edge) { addFields({"a"}); - setup(attrCfg({make_fa(make_int32_sv_cfg())}), attrCfg({make_int32_sv_cfg()}), sCfg({make_summary_field("a", "integer", "attribute", "a")})); + setup(attrCfg({make_fa(make_int32_sv_cfg())}), attrCfg({make_int32_sv_cfg()}), + sCfg({make_summary_field("a", "integer", command::attribute, "a")})); assertAttributeConfig({make_fa(make_int32_sv_cfg())}); - assertSummaryConfig({make_summary_field("a", "integer", "attribute", "a")}); + assertSummaryConfig({make_summary_field("a", "integer", command::attribute, "a")}); } TEST_F(DelayerTest, require_that_fast_access_flag_change_is_delayed_false_true_edge_on_tensor_attribute) { addFields({"a"}); - setup(attrCfg({make_tensor_cfg("tensor(x[10])")}), attrCfg({make_fa(make_tensor_cfg("tensor(x[10])"))}), sCfg({make_summary_field("a", "tensor", "attribute", "a")})); + setup(attrCfg({make_tensor_cfg("tensor(x[10])")}), attrCfg({make_fa(make_tensor_cfg("tensor(x[10])"))}), + sCfg({make_summary_field("a", "tensor", command::attribute, "a")})); assertAttributeConfig({make_tensor_cfg("tensor(x[10])")}); - assertSummaryConfig({make_summary_field("a", "tensor", "attribute", "a")}); + assertSummaryConfig({make_summary_field("a", "tensor", command::attribute, "a")}); } TEST_F(DelayerTest, require_that_fast_access_flag_change_is_delayed_true_false_edge_on_tensor_attribute) { addFields({"a"}); setup(attrCfg({make_fa(make_tensor_cfg("tensor(x[10])"))}), - attrCfg({make_tensor_cfg("tensor(x[10])")}), sCfg({make_summary_field("a", "tensor", "attribute", "a")})); + attrCfg({make_tensor_cfg("tensor(x[10])")}), + sCfg({make_summary_field("a", "tensor", command::attribute, "a")})); assertAttributeConfig({make_fa(make_tensor_cfg("tensor(x[10])"))}); - assertSummaryConfig({make_summary_field("a", "tensor", "attribute", "a")}); + assertSummaryConfig({make_summary_field("a", "tensor", command::attribute, "a")}); } TEST_F(DelayerTest, require_that_fast_access_flag_change_is_not_delayed_true_false_edge_on_string_attribute_indexed_field) { addFields({"a"}); addOldIndexField("a"); - setup(attrCfg({make_fa(make_string_sv_cfg())}), attrCfg({make_string_sv_cfg()}), sCfg({make_summary_field("a", "string", "attribute", "a")})); + setup(attrCfg({make_fa(make_string_sv_cfg())}), attrCfg({make_string_sv_cfg()}), + sCfg({make_summary_field("a", "string", command::attribute, "a")})); assertAttributeConfig({make_string_sv_cfg()}); - assertSummaryConfig({make_summary_field("a", "string", "attribute", "a")}); + assertSummaryConfig({make_summary_field("a", "string", command::attribute, "a")}); } TEST_F(DelayerTest, require_that_adding_attribute_aspect_to_struct_field_is_not_delayed_if_field_type_is_changed) { - setup(attrCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), sCfg({make_summary_field("array", "jsonstring", "attributecombiner", "array")})); + setup(attrCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), + sCfg({make_summary_field("array", "jsonstring", command::attribute_combiner, "array")})); assertAttributeConfig({make_int32_sv_cfg("array.a")}); - assertSummaryConfig({make_summary_field("array", "jsonstring", "attributecombiner", "array")}); + assertSummaryConfig({make_summary_field("array", "jsonstring", command::attribute_combiner, "array")}); } TEST_F(DelayerTest, require_that_adding_attribute_aspect_to_struct_field_is_delayed_if_field_type_is_unchanged) { addFields({"array.a"}); - setup(attrCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), sCfg({make_summary_field("array", "jsonstring", "attributecombiner", "array")})); + setup(attrCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), + sCfg({make_summary_field("array", "jsonstring", command::attribute_combiner, "array")})); assertAttributeConfig({}); assertSummaryConfig({make_summary_field("array", "jsonstring")}); } @@ -370,9 +386,12 @@ TEST_F(DelayerTest, require_that_removing_attribute_aspect_from_struct_field_is_ TEST_F(DelayerTest, require_that_adding_attribute_aspect_to_struct_field_is_delayed_if_field_type_is_unchanged_with_filtering_docsum) { addFields({"array.a"}); - setup(attrCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), sCfg({make_summary_field("array", "jsonstring", "attributecombiner", "array"), make_summary_field("array_filtered", "jsonstring", "matchedattributeelementsfilter", "array")})); + setup(attrCfg({}), attrCfg({make_int32_sv_cfg("array.a")}), + sCfg({make_summary_field("array", "jsonstring", command::attribute_combiner, "array"), + make_summary_field("array_filtered", "jsonstring", command::matched_attribute_elements_filter, "array")})); assertAttributeConfig({}); - assertSummaryConfig({make_summary_field("array", "jsonstring"), make_summary_field("array_filtered", "jsonstring", "matchedelementsfilter", "array")}); + assertSummaryConfig({make_summary_field("array", "jsonstring"), + make_summary_field("array_filtered", "jsonstring", command::matched_elements_filter, "array")}); } } diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index 17fb15b39cd..e8710a00789 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -82,11 +82,7 @@ namespace proton { class MockDocsumFieldWriterFactory : public search::docsummary::IDocsumFieldWriterFactory { public: - std::unique_ptr<DocsumFieldWriter> create_docsum_field_writer(const vespalib::string& fieldName, const vespalib::string& overrideName, const vespalib::string& argument, bool& rc) override { - (void) fieldName; - (void) overrideName; - (void) argument; - (void) rc; + std::unique_ptr<DocsumFieldWriter> create_docsum_field_writer(const vespalib::string&, const vespalib::string&, const vespalib::string&) override { return {}; } @@ -311,12 +307,9 @@ class MockJuniperConverter : public IJuniperConverter { vespalib::string _result; public: - void insert_juniper_field(vespalib::stringref input, vespalib::slime::Inserter&) override { + void convert(vespalib::stringref input, vespalib::slime::Inserter&) override { _result = input; } - void insert_juniper_field(const document::StringFieldValue& input, vespalib::slime::Inserter&) override { - _result = input.getValueRef(); - } const vespalib::string& get_result() const noexcept { return _result; } }; diff --git a/searchcore/src/vespa/searchcore/bmcluster/bm_cluster.cpp b/searchcore/src/vespa/searchcore/bmcluster/bm_cluster.cpp index 123e2b7577d..47c1ec709fd 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bm_cluster.cpp +++ b/searchcore/src/vespa/searchcore/bmcluster/bm_cluster.cpp @@ -23,6 +23,7 @@ #include <vespa/vespalib/util/stringfmt.h> #include <filesystem> #include <thread> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".bmcluster.bm_cluster"); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp index e25bc4c71d5..1b13e80563a 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp @@ -4,31 +4,28 @@ #include <vespa/config-attributes.h> #include <vespa/config-summary.h> #include <vespa/searchcommon/attribute/attribute_utils.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/searchcore/proton/common/config_hash.hpp> #include <vespa/searchcore/proton/common/i_document_type_inspector.h> #include <vespa/searchcore/proton/common/i_indexschema_inspector.h> #include <vespa/searchlib/attribute/configconverter.h> -#include <vespa/searchcommon/attribute/config.h> +#include <vespa/searchsummary/docsummary/docsum_field_writer_commands.h> #include <vespa/vespalib/stllike/hash_set.hpp> -using search::attribute::isUpdateableInMemoryOnly; using search::attribute::BasicType; using search::attribute::ConfigConverter; +using search::attribute::isUpdateableInMemoryOnly; using vespa::config::search::AttributesConfig; using vespa::config::search::AttributesConfigBuilder; using vespa::config::search::SummaryConfig; using vespa::config::search::SummaryConfigBuilder; +using namespace search::docsummary; + namespace proton { namespace { -vespalib::string attribute_combiner_dfw_string("attributecombiner"); -vespalib::string matched_attribute_elements_filter_dfw_string("matchedattributeelementsfilter"); -vespalib::string matched_elements_filter_dfw_string("matchedelementsfilter"); -vespalib::string copy_dfw_string("copy"); -vespalib::string attribute_dfw_string("attribute"); - using AttributesConfigHash = ConfigHash<AttributesConfig::Attribute>; bool willTriggerReprocessOnAttributeAspectRemoval(const search::attribute::Config &cfg, @@ -53,7 +50,7 @@ void remove_docsum_field_rewriter(SummaryConfig::Classes::Fields& summary_field) { if (source_field(summary_field) != summary_field.name) { - summary_field.command = copy_dfw_string; + summary_field.command = command::copy; } else { summary_field.command = ""; summary_field.source = ""; @@ -225,31 +222,31 @@ AttributeAspectConfigRewriter::build_summary_config(const SummaryConfig& new_sum summary_config_builder = new_summary_config; for (auto &summary_class : summary_config_builder.classes) { for (auto &summary_field : summary_class.fields) { - if (summary_field.command == attribute_dfw_string) { + if (summary_field.command == command::attribute) { if (is_delayed_add_attribute_aspect(source_field(summary_field))) { remove_docsum_field_rewriter(summary_field); } - } else if (summary_field.command == attribute_combiner_dfw_string) { + } else if (summary_field.command == command::attribute_combiner) { if (is_delayed_add_attribute_aspect_struct(source_field(summary_field))) { remove_docsum_field_rewriter(summary_field); } - } else if (summary_field.command == matched_attribute_elements_filter_dfw_string) { + } else if (summary_field.command == command::matched_attribute_elements_filter) { if (is_delayed_add_attribute_aspect_struct(source_field(summary_field)) || is_delayed_add_attribute_aspect(source_field(summary_field))) { - summary_field.command = matched_elements_filter_dfw_string; + summary_field.command = command::matched_elements_filter; } - } else if (summary_field.command == matched_elements_filter_dfw_string) { + } else if (summary_field.command == command::matched_elements_filter) { if (is_delayed_remove_attribute_aspect(source_field(summary_field))) { - summary_field.command = matched_attribute_elements_filter_dfw_string; + summary_field.command = command::matched_attribute_elements_filter; } } else if (summary_field.command == "") { if (is_delayed_remove_attribute_aspect(summary_field.name)) { - summary_field.command = attribute_dfw_string; + summary_field.command = command::attribute; summary_field.source = summary_field.name; } - } else if (summary_field.command == copy_dfw_string) { + } else if (summary_field.command == command::copy) { if (is_delayed_remove_attribute_aspect(source_field(summary_field))) { - summary_field.command = attribute_dfw_string; + summary_field.command = command::attribute; summary_field.source = source_field(summary_field); } } diff --git a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt index 25e9a469f93..fffbd12764b 100644 --- a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt @@ -32,7 +32,3 @@ vespa_add_library(searchcore_pcommon STATIC EXTERNAL_DEPENDS ${VESPA_STDCXX_FS_LIB} ) - -if(VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 8.2" OR VESPA_OS_DISTRO_COMBINED STREQUAL "centos 8") - set_source_files_properties(hw_info_sampler.cpp PROPERTIES COMPILE_FLAGS -DRHEL_8_2_KLUDGE) -endif() diff --git a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp index ea2d2de8b41..2f054843dd3 100644 --- a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp @@ -165,16 +165,3 @@ HwInfoSampler::sampleDiskWriteSpeed(const vespalib::string &path, const Config & } } - -#ifdef RHEL_8_2_KLUDGE - -// Kludge to avoid unresolved symbols with gcc-toolset-9 on RHEL 8.2 -#include <codecvt> - -namespace std { - -template class codecvt_utf8<wchar_t>; - -} - -#endif diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp index 9a4ec121d3b..576b8f5bbd4 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/docsumcontext.cpp @@ -85,7 +85,7 @@ DocsumContext::createSlimeReply() Cursor &docSumC = array.addObject(); ObjectSymbolInserter inserter(docSumC, docsumSym); if ((docId != search::endDocId) && rci.outputClass != nullptr) { - _docsumWriter.insertDocsum(rci, docId, &_docsumState, &_docsumStore, inserter); + _docsumWriter.insertDocsum(rci, docId, _docsumState, &_docsumStore, inserter); } num_ok++; } diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index fbb17a43ee6..032307c1157 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -242,7 +242,7 @@ StoreOnlyDocSubDB::setupSummaryManager(SummaryManager::SP summaryManager) _rSummaryMgr = std::move(summaryManager); _iSummaryMgr = _rSummaryMgr; // Upcast allowed with std::shared_ptr _flushedDocumentStoreSerialNum = _iSummaryMgr->getBackingStore().lastSyncToken(); - _summaryAdapter.reset(new SummaryAdapter(_rSummaryMgr)); + _summaryAdapter = std::make_shared<SummaryAdapter>(_rSummaryMgr); } diff --git a/searchcore/src/vespa/searchcore/proton/server/summaryadapter.cpp b/searchcore/src/vespa/searchcore/proton/server/summaryadapter.cpp index 57786a5f788..038af801b80 100644 --- a/searchcore/src/vespa/searchcore/proton/server/summaryadapter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/summaryadapter.cpp @@ -3,6 +3,7 @@ #include "summaryadapter.h" #include <vespa/searchcore/proton/docsummary/summarymanager.h> #include <vespa/vespalib/objects/nbostream.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".proton.server.summaryadapter"); @@ -11,12 +12,12 @@ using namespace document; namespace proton { -SummaryAdapter::SummaryAdapter(const SummaryManager::SP &mgr) - : _mgr(mgr), +SummaryAdapter::SummaryAdapter(SummaryManager::SP mgr) + : _mgr(std::move(mgr)), _lastSerial(_mgr->getBackingStore().lastSyncToken()) {} -SummaryAdapter::~SummaryAdapter() {} +SummaryAdapter::~SummaryAdapter() = default; bool SummaryAdapter::ignore(SerialNum serialNum) const { diff --git a/searchcore/src/vespa/searchcore/proton/server/summaryadapter.h b/searchcore/src/vespa/searchcore/proton/server/summaryadapter.h index 89b3827628c..0402d860577 100644 --- a/searchcore/src/vespa/searchcore/proton/server/summaryadapter.h +++ b/searchcore/src/vespa/searchcore/proton/server/summaryadapter.h @@ -17,8 +17,8 @@ private: ISummaryManager & imgr() const; public: - SummaryAdapter(const std::shared_ptr<SummaryManager> &mgr); - ~SummaryAdapter(); + explicit SummaryAdapter(std::shared_ptr<SummaryManager> mgr); + ~SummaryAdapter() override; void put(SerialNum serialNum, const DocumentIdT lid, const Document &doc) override; void put(SerialNum serialNum, const DocumentIdT lid, const vespalib::nbostream &doc) override; diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp index 8531ea66d38..4165d13d3fd 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp @@ -11,6 +11,7 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/size_literals.h> #include <thread> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docstore.logdatastore"); diff --git a/searchlib/src/vespa/searchlib/queryeval/global_filter.h b/searchlib/src/vespa/searchlib/queryeval/global_filter.h index e162bccdbc5..acae2bf6297 100644 --- a/searchlib/src/vespa/searchlib/queryeval/global_filter.h +++ b/searchlib/src/vespa/searchlib/queryeval/global_filter.h @@ -5,7 +5,7 @@ #include <memory> #include <vector> -namespace vespalib { class ThreadBundle; } +namespace vespalib { struct ThreadBundle; } namespace search { class BitVector; } namespace search::queryeval { diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.hpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.hpp index 11500c0b7f6..4fd75bb2fec 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.hpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.hpp @@ -5,6 +5,7 @@ #include "hnsw_index_loader.h" #include "hnsw_graph.h" #include <vespa/searchlib/util/fileutil.h> +#include <cassert> namespace search::tensor { @@ -18,7 +19,7 @@ HnswIndexLoader<ReaderType>::init() } template <typename ReaderType> -HnswIndexLoader<ReaderType>::~HnswIndexLoader() {} +HnswIndexLoader<ReaderType>::~HnswIndexLoader() = default; template <typename ReaderType> HnswIndexLoader<ReaderType>::HnswIndexLoader(HnswGraph& graph, std::unique_ptr<ReaderType> reader) diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp index 8ac24d08dcf..29218b47f53 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp @@ -4,6 +4,7 @@ #include "hnsw_graph.h" #include <vespa/searchlib/util/bufferwriter.h> #include <limits> +#include <cassert> namespace search::tensor { diff --git a/searchsummary/CMakeLists.txt b/searchsummary/CMakeLists.txt index 5fcd0f7e19a..a5dc62da5c0 100644 --- a/searchsummary/CMakeLists.txt +++ b/searchsummary/CMakeLists.txt @@ -16,12 +16,12 @@ vespa_define_module( TESTS src/tests/docsummary + src/tests/docsummary/annotation_converter src/tests/docsummary/attribute_combiner src/tests/docsummary/attributedfw src/tests/docsummary/document_id_dfw src/tests/docsummary/matched_elements_filter src/tests/docsummary/slime_filler src/tests/docsummary/slime_summary - src/tests/docsummary/summary_field_converter src/tests/juniper ) diff --git a/searchsummary/src/tests/docsummary/CMakeLists.txt b/searchsummary/src/tests/docsummary/CMakeLists.txt index 26a2963809a..4cd12eb4db6 100644 --- a/searchsummary/src/tests/docsummary/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(searchsummary_positionsdfw_test_app TEST positionsdfw_test.cpp DEPENDS searchsummary + GTest::GTest ) vespa_add_test(NAME searchsummary_positionsdfw_test_app COMMAND searchsummary_positionsdfw_test_app) diff --git a/searchsummary/src/tests/docsummary/annotation_converter/CMakeLists.txt b/searchsummary/src/tests/docsummary/annotation_converter/CMakeLists.txt new file mode 100644 index 00000000000..22e0d3e6477 --- /dev/null +++ b/searchsummary/src/tests/docsummary/annotation_converter/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchsummary_annotation_converter_test_app TEST + SOURCES + annotation_converter_test.cpp + DEPENDS + searchsummary + GTest::GTest +) +vespa_add_test(NAME searchsummary_annotation_converter_test_app COMMAND searchsummary_annotation_converter_test_app) diff --git a/searchsummary/src/tests/docsummary/annotation_converter/annotation_converter_test.cpp b/searchsummary/src/tests/docsummary/annotation_converter/annotation_converter_test.cpp new file mode 100644 index 00000000000..753ae8d9044 --- /dev/null +++ b/searchsummary/src/tests/docsummary/annotation_converter/annotation_converter_test.cpp @@ -0,0 +1,176 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/document/annotation/annotation.h> +#include <vespa/document/annotation/span.h> +#include <vespa/document/annotation/spanlist.h> +#include <vespa/document/annotation/spantree.h> +#include <vespa/document/fieldvalue/stringfieldvalue.h> +#include <vespa/document/repo/configbuilder.h> +#include <vespa/document/repo/fixedtyperepo.h> +#include <vespa/juniper/juniper_separators.h> +#include <vespa/searchsummary/docsummary/annotation_converter.h> +#include <vespa/searchsummary/docsummary/i_juniper_converter.h> +#include <vespa/searchsummary/docsummary/linguisticsannotation.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/stllike/asciistream.h> + +using document::Annotation; +using document::DocumentType; +using document::DocumentTypeRepo; +using document::Span; +using document::SpanList; +using document::SpanTree; +using document::StringFieldValue; +using search::docsummary::AnnotationConverter; +using search::docsummary::IJuniperConverter; +using search::linguistics::SPANTREE_NAME; +using search::linguistics::TERM; +using vespalib::Slime; +using vespalib::slime::SlimeInserter; + +namespace { + +DocumenttypesConfig +get_document_types_config() +{ + using namespace document::config_builder; + DocumenttypesConfigBuilderHelper builder; + builder.document(42, "indexingdocument", + Struct("indexingdocument.header"), + Struct("indexingdocument.body")); + return builder.config(); +} + +class MockJuniperConverter : public IJuniperConverter +{ + vespalib::string _result; +public: + void convert(vespalib::stringref input, vespalib::slime::Inserter&) override { + _result = input; + } + const vespalib::string& get_result() const noexcept { return _result; } +}; + +} + +class AnnotationConverterTest : public testing::Test +{ +protected: + std::shared_ptr<const DocumentTypeRepo> _repo; + const DocumentType* _document_type; + document::FixedTypeRepo _fixed_repo; + + AnnotationConverterTest(); + ~AnnotationConverterTest() override; + void set_span_tree(StringFieldValue& value, std::unique_ptr<SpanTree> tree); + StringFieldValue make_annotated_string(); + StringFieldValue make_annotated_chinese_string(); + vespalib::string make_exp_il_annotated_string(); + vespalib::string make_exp_il_annotated_chinese_string(); + void expect_annotated(const vespalib::string& exp, const StringFieldValue& fv); +}; + +AnnotationConverterTest::AnnotationConverterTest() + : testing::Test(), + _repo(std::make_unique<DocumentTypeRepo>(get_document_types_config())), + _document_type(_repo->getDocumentType("indexingdocument")), + _fixed_repo(*_repo, *_document_type) +{ +} + +AnnotationConverterTest::~AnnotationConverterTest() = default; + +void +AnnotationConverterTest::set_span_tree(StringFieldValue & value, std::unique_ptr<SpanTree> tree) +{ + StringFieldValue::SpanTrees trees; + trees.push_back(std::move(tree)); + value.setSpanTrees(trees, _fixed_repo); +} + +StringFieldValue +AnnotationConverterTest::make_annotated_string() +{ + auto span_list_up = std::make_unique<SpanList>(); + auto span_list = span_list_up.get(); + auto tree = std::make_unique<SpanTree>(SPANTREE_NAME, std::move(span_list_up)); + tree->annotate(span_list->add(std::make_unique<Span>(0, 3)), *TERM); + tree->annotate(span_list->add(std::make_unique<Span>(4, 3)), + Annotation(*TERM, std::make_unique<StringFieldValue>("baz"))); + StringFieldValue value("foo bar"); + set_span_tree(value, std::move(tree)); + return value; +} + +StringFieldValue +AnnotationConverterTest::make_annotated_chinese_string() +{ + auto span_list_up = std::make_unique<SpanList>(); + auto span_list = span_list_up.get(); + auto tree = std::make_unique<SpanTree>(SPANTREE_NAME, std::move(span_list_up)); + // These chinese characters each use 3 bytes in their UTF8 encoding. + tree->annotate(span_list->add(std::make_unique<Span>(0, 15)), *TERM); + tree->annotate(span_list->add(std::make_unique<Span>(15, 9)), *TERM); + StringFieldValue value("我就是那个大灰狼"); + set_span_tree(value, std::move(tree)); + return value; +} + +vespalib::string +AnnotationConverterTest::make_exp_il_annotated_string() +{ + using namespace juniper::separators; + vespalib::asciistream exp; + exp << "foo" << unit_separator_string << + " " << unit_separator_string << interlinear_annotation_anchor_string << + "bar" << interlinear_annotation_separator_string << + "baz" << interlinear_annotation_terminator_string << unit_separator_string; + return exp.str(); +} + +vespalib::string +AnnotationConverterTest::make_exp_il_annotated_chinese_string() +{ + using namespace juniper::separators; + vespalib::asciistream exp; + exp << "我就是那个" << unit_separator_string << + "大灰狼" << unit_separator_string; + return exp.str(); +} + +void +AnnotationConverterTest::expect_annotated(const vespalib::string& exp, const StringFieldValue& fv) +{ + MockJuniperConverter juniper_converter; + AnnotationConverter annotation_converter(juniper_converter); + Slime slime; + SlimeInserter inserter(slime); + annotation_converter.convert(fv, inserter); + EXPECT_EQ(exp, juniper_converter.get_result()); +} + + +TEST_F(AnnotationConverterTest, convert_plain_string) +{ + using namespace juniper::separators; + vespalib::string exp("Foo Bar Baz"); + StringFieldValue plain_string("Foo Bar Baz"); + expect_annotated(exp + unit_separator_string, plain_string); +} + +TEST_F(AnnotationConverterTest, convert_annotated_string) +{ + auto exp = make_exp_il_annotated_string(); + auto annotated_string = make_annotated_string(); + expect_annotated(exp, annotated_string); +} + +TEST_F(AnnotationConverterTest, convert_annotated_chinese_string) +{ + auto exp = make_exp_il_annotated_chinese_string(); + auto annotated_chinese_string = make_annotated_chinese_string(); + expect_annotated(exp, annotated_chinese_string); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp index 8bf3db4d112..005fed41838 100644 --- a/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp +++ b/searchsummary/src/tests/docsummary/attribute_combiner/attribute_combiner_test.cpp @@ -91,7 +91,7 @@ AttributeCombinerTest::assertWritten(const vespalib::string &exp_slime_as_json, { vespalib::Slime act; vespalib::slime::SlimeInserter inserter(act); - writer->insertField(docId, nullptr, &state, search::docsummary::RES_JSONSTRING, inserter); + writer->insertField(docId, nullptr, state, inserter); SlimeValue exp(exp_slime_as_json); EXPECT_EQ(exp.slime, act); diff --git a/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp b/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp index e9d00629d6f..bba3a5ab506 100644 --- a/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp +++ b/searchsummary/src/tests/docsummary/attributedfw/attributedfw_test.cpp @@ -69,7 +69,7 @@ public: void expect_field(const vespalib::string& exp_slime_as_json, uint32_t docid) { vespalib::Slime act; vespalib::slime::SlimeInserter inserter(act); - _writer->insertField(docid, nullptr, &_state, search::docsummary::RES_JSONSTRING, inserter); + _writer->insertField(docid, nullptr, _state, inserter); SlimeValue exp(exp_slime_as_json); EXPECT_EQ(exp.slime, act); diff --git a/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp b/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp index 1c4e201e745..4819c34272c 100644 --- a/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp +++ b/searchsummary/src/tests/docsummary/document_id_dfw/document_id_dfw_test.cpp @@ -5,6 +5,8 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/searchlib/common/matching_elements.h> +#include <vespa/searchsummary/docsummary/docsumstate.h> #include <vespa/searchsummary/docsummary/docsum_store_document.h> #include <vespa/searchsummary/docsummary/document_id_dfw.h> #include <vespa/searchsummary/docsummary/resultclass.h> @@ -20,8 +22,12 @@ using document::DocumentType; using document::DocumentTypeRepo; using document::config_builder::DocumenttypesConfigBuilderHelper; using document::config_builder::Struct; +using search::MatchingElements; +using search::MatchingElementsFields; using search::docsummary::DocsumStoreDocument; using search::docsummary::DocumentIdDFW; +using search::docsummary::GetDocsumsState; +using search::docsummary::GetDocsumsStateCallback; using search::docsummary::IDocsumStoreDocument; using search::docsummary::ResultClass; using search::docsummary::ResultConfig; @@ -47,6 +53,12 @@ make_doc_type_repo() return std::make_unique<const DocumentTypeRepo>(builder.config()); } +struct MyGetDocsumsStateCallback : GetDocsumsStateCallback { + virtual void FillSummaryFeatures(GetDocsumsState&) override {} + virtual void FillRankFeatures(GetDocsumsState&) override {} + std::unique_ptr<MatchingElements> fill_matching_elements(const MatchingElementsFields &) override { abort(); } +}; + class DocumentIdDFWTest : public ::testing::Test { vespalib::string _field_name; @@ -96,7 +108,9 @@ DocumentIdDFWTest::write(const IDocsumStoreDocument* doc) Cursor & docsum = top_inserter.insertObject(); ObjectInserter field_inserter(docsum, _field_name_view); DocumentIdDFW writer; - writer.insertField(0, doc, nullptr, search::docsummary::RES_LONG_STRING, field_inserter); + MyGetDocsumsStateCallback callback; + GetDocsumsState state(callback); + writer.insertField(0, doc, state, field_inserter); return slime; } diff --git a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp index 519961dedb6..8ac37ae76eb 100644 --- a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp +++ b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp @@ -218,7 +218,7 @@ private: Slime slime; SlimeInserter inserter(slime); - writer->insertField(doc_id, doc.get(), &state, ResType::RES_JSONSTRING, inserter); + writer->insertField(doc_id, doc.get(), state, inserter); return slime; } diff --git a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp index f2e949cbddf..f23bd2f0437 100644 --- a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp +++ b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp @@ -1,17 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Unit tests for positionsdfw. +#include <vespa/juniper/rpinterface.h> #include <vespa/searchlib/attribute/extendableattributes.h> #include <vespa/searchlib/attribute/iattributemanager.h> #include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchsummary/docsummary/docsum_field_writer.h> -#include <vespa/searchsummary/docsummary/positionsdfw.h> -#include <vespa/searchsummary/docsummary/idocsumenvironment.h> #include <vespa/searchsummary/docsummary/docsumstate.h> +#include <vespa/searchsummary/docsummary/idocsumenvironment.h> +#include <vespa/searchsummary/docsummary/positionsdfw.h> #include <vespa/searchsummary/test/slime_value.h> -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/data/slime/slime.h> -#include <vespa/juniper/rpinterface.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP("positionsdfw_test"); @@ -29,33 +28,6 @@ namespace search::docsummary { namespace { -class Test : public vespalib::TestApp { - void requireThat2DPositionFieldIsWritten(); - -public: - int Main() override; -}; - -int -Test::Main() -{ - TEST_INIT("positionsdfw_test"); - - TEST_DO(requireThat2DPositionFieldIsWritten()); - - TEST_DONE(); -} - -struct MyEnvironment : IDocsumEnvironment { - IAttributeManager *attribute_man; - - MyEnvironment() : attribute_man(0) {} - - const IAttributeManager *getAttributeManager() const override { return attribute_man; } - string lookupIndex(const string &s) const override { return s; } - const juniper::Juniper *getJuniper() const override { return nullptr; } -}; - class MyAttributeContext : public IAttributeContext { const IAttributeVector &_attr; public: @@ -132,20 +104,22 @@ void checkWritePositionField(AttrType &attr, MyAttributeManager attribute_man(attr); PositionsDFW::UP writer = PositionsDFW::create(attr.getName().c_str(), &attribute_man, false); ASSERT_TRUE(writer.get()); - ResType res_type = RES_JSONSTRING; MyGetDocsumsStateCallback callback; GetDocsumsState state(callback); state._attributes.push_back(&attr); vespalib::Slime target; vespalib::slime::SlimeInserter inserter(target); - writer->insertField(doc_id, &state, res_type, inserter); + writer->insertField(doc_id, state, inserter); test::SlimeValue expected(expect_json); - EXPECT_EQUAL(expected.slime, target); + EXPECT_EQ(expected.slime, target); } -void Test::requireThat2DPositionFieldIsWritten() { +} // namespace + +TEST(PositionsDFWTest, require_that_2D_position_field_is_written) +{ SingleInt64ExtAttribute attr("foo"); checkWritePositionField(attr, 0x3e, "{x:6,y:7,latlong:'N0.000007;E0.000006'}"); checkWritePositionField(attr, 007, "{x:-1,y:-1,latlong:'S0.000001;W0.000001'}"); @@ -154,7 +128,6 @@ void Test::requireThat2DPositionFieldIsWritten() { checkWritePositionField(attr, 42, "null"); } -} // namespace } -TEST_APPHOOK(search::docsummary::Test); +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchsummary/src/tests/docsummary/slime_filler/slime_filler_test.cpp b/searchsummary/src/tests/docsummary/slime_filler/slime_filler_test.cpp index cf4006c5e67..505386f5b91 100644 --- a/searchsummary/src/tests/docsummary/slime_filler/slime_filler_test.cpp +++ b/searchsummary/src/tests/docsummary/slime_filler/slime_filler_test.cpp @@ -1,12 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/document/annotation/annotation.h> -#include <vespa/document/annotation/span.h> -#include <vespa/document/annotation/spanlist.h> -#include <vespa/document/annotation/spantree.h> #include <vespa/document/base/documentid.h> #include <vespa/document/datatype/documenttype.h> -#include <vespa/document/datatype/urldatatype.h> #include <vespa/document/datatype/referencedatatype.h> #include <vespa/document/datatype/tensor_data_type.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> @@ -31,23 +26,18 @@ #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/value_codec.h> -#include <vespa/juniper/juniper_separators.h> -#include <vespa/searchsummary/docsummary/docsum_field_writer.h> -#include <vespa/searchsummary/docsummary/i_docsum_field_writer_factory.h> -#include <vespa/searchsummary/docsummary/i_juniper_converter.h> +#include <vespa/searchsummary/docsummary/i_string_field_converter.h> #include <vespa/searchsummary/docsummary/linguisticsannotation.h> #include <vespa/searchsummary/docsummary/resultconfig.h> #include <vespa/searchsummary/docsummary/slime_filler.h> +#include <vespa/searchsummary/docsummary/slime_filler_filter.h> #include <vespa/vespalib/data/slime/binary_format.h> #include <vespa/vespalib/data/slime/json_format.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/simple_buffer.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/util/size_literals.h> -#include <vespa/config-summary.h> -using document::Annotation; using document::ArrayFieldValue; using document::BoolFieldValue; using document::ByteFieldValue; @@ -69,21 +59,16 @@ using document::RawFieldValue; using document::ReferenceDataType; using document::ReferenceFieldValue; using document::ShortFieldValue; -using document::Span; -using document::SpanList; -using document::SpanTree; using document::StringFieldValue; using document::StructDataType; using document::StructFieldValue; using document::TensorDataType; using document::TensorFieldValue; -using document::UrlDataType; using document::WeightedSetFieldValue; -using search::docsummary::IDocsumFieldWriterFactory; -using search::docsummary::IJuniperConverter; -using search::docsummary::DocsumFieldWriter; +using search::docsummary::IStringFieldConverter; using search::docsummary::ResultConfig; using search::docsummary::SlimeFiller; +using search::docsummary::SlimeFillerFilter; using search::linguistics::SPANTREE_NAME; using search::linguistics::TERM; using vespalib::SimpleBuffer; @@ -95,7 +80,6 @@ using vespalib::eval::ValueType; using vespalib::slime::Cursor; using vespalib::slime::JsonFormat; using vespalib::slime::SlimeInserter; -using vespa::config::search::SummaryConfigBuilder; namespace { @@ -114,15 +98,6 @@ slime_to_string(const Slime& slime) } vespalib::string -make_slime_string(vespalib::stringref value) -{ - Slime slime; - SlimeInserter inserter(slime); - inserter.insertString({value}); - return slime_to_string(slime); -} - -vespalib::string make_slime_data_string(vespalib::stringref data) { Slime slime; @@ -139,15 +114,6 @@ make_slime_tensor_string(const Value& value) return make_slime_data_string({s.peek(), s.size()}); } -class MockDocsumFieldWriterFactory : public IDocsumFieldWriterFactory -{ -public: - std::unique_ptr<DocsumFieldWriter> create_docsum_field_writer(const vespalib::string&, const vespalib::string&, const vespalib::string&, bool&) override { - return {}; - } - -}; - DocumenttypesConfig get_document_types_config() { @@ -172,20 +138,25 @@ get_document_types_config() .addField("d", nested_type_id) .addField("e", nested_type_id) .addField("f", nested_type_id)) + .addField("nested_array", Array(nested_type_id)) + .addField("nested_map", Map(DataType::T_STRING, nested_type_id)) .addField("ref", ref_type_id), Struct("indexingdocument.body")) .referenceType(ref_type_id, ref_target_doctype_id); return builder.config(); } -class MockJuniperConverter : public IJuniperConverter +class MockStringFieldConverter : public IStringFieldConverter { vespalib::string _result; public: - void insert_juniper_field(vespalib::stringref input, vespalib::slime::Inserter&) override { - _result = input; + MockStringFieldConverter() + : IStringFieldConverter(), + _result() + { } - void insert_juniper_field(const document::StringFieldValue& input, vespalib::slime::Inserter&) override { + ~MockStringFieldConverter() override = default; + void convert(const document::StringFieldValue& input, vespalib::slime::Inserter&) override { _result = input.getValueRef(); } const vespalib::string& get_result() const noexcept { return _result; } @@ -198,32 +169,26 @@ class SlimeFillerTest : public testing::Test protected: std::shared_ptr<const DocumentTypeRepo> _repo; const DocumentType* _document_type; - document::FixedTypeRepo _fixed_repo; SlimeFillerTest(); ~SlimeFillerTest() override; const DataType& get_data_type(const vespalib::string& name) const; const ReferenceDataType& get_as_ref_type(const vespalib::string& name) const; - void set_span_tree(StringFieldValue& value, std::unique_ptr<SpanTree> tree); - StringFieldValue make_annotated_string(); - StringFieldValue make_annotated_chinese_string(); - vespalib::string make_exp_il_annotated_string(); - vespalib::string make_exp_il_annotated_chinese_string(); ArrayFieldValue make_array(); WeightedSetFieldValue make_weighted_set(); MapFieldValue make_map(); - void expect_insert(const vespalib::string& exp, const FieldValue& fv, bool tokenize, const std::vector<uint32_t>* matching_elems); - void expect_insert(const vespalib::string& exp, const FieldValue& fv, bool tokenize); + StructFieldValue make_nested_value(int i); + void expect_insert(const vespalib::string& exp, const FieldValue& fv, const std::vector<uint32_t>* matching_elems); void expect_insert(const vespalib::string& exp, const FieldValue& fv); void expect_insert_filtered(const vespalib::string& exp, const FieldValue& fv, const std::vector<uint32_t>& matching_elems); - void expect_insert_callback(const vespalib::string& exp, const FieldValue& fv, bool tokenize); + void expect_insert(const vespalib::string& exp, const FieldValue& fv, SlimeFillerFilter& filter); + void expect_insert_callback(const vespalib::string& exp, const FieldValue& fv); }; SlimeFillerTest::SlimeFillerTest() : testing::Test(), _repo(std::make_unique<DocumentTypeRepo>(get_document_types_config())), - _document_type(_repo->getDocumentType("indexingdocument")), - _fixed_repo(*_repo, *_document_type) + _document_type(_repo->getDocumentType("indexingdocument")) { } @@ -242,64 +207,6 @@ SlimeFillerTest::get_as_ref_type(const vespalib::string& name) const { return dynamic_cast<const ReferenceDataType&>(get_data_type(name)); } -void -SlimeFillerTest::set_span_tree(StringFieldValue & value, std::unique_ptr<SpanTree> tree) -{ - StringFieldValue::SpanTrees trees; - trees.push_back(std::move(tree)); - value.setSpanTrees(trees, _fixed_repo); -} - -StringFieldValue -SlimeFillerTest::make_annotated_string() -{ - auto span_list_up = std::make_unique<SpanList>(); - auto span_list = span_list_up.get(); - auto tree = std::make_unique<SpanTree>(SPANTREE_NAME, std::move(span_list_up)); - tree->annotate(span_list->add(std::make_unique<Span>(0, 3)), *TERM); - tree->annotate(span_list->add(std::make_unique<Span>(4, 3)), - Annotation(*TERM, std::make_unique<StringFieldValue>("baz"))); - StringFieldValue value("foo bar"); - set_span_tree(value, std::move(tree)); - return value; -} - -StringFieldValue -SlimeFillerTest::make_annotated_chinese_string() -{ - auto span_list_up = std::make_unique<SpanList>(); - auto span_list = span_list_up.get(); - auto tree = std::make_unique<SpanTree>(SPANTREE_NAME, std::move(span_list_up)); - // These chinese characters each use 3 bytes in their UTF8 encoding. - tree->annotate(span_list->add(std::make_unique<Span>(0, 15)), *TERM); - tree->annotate(span_list->add(std::make_unique<Span>(15, 9)), *TERM); - StringFieldValue value("我就是那个大灰狼"); - set_span_tree(value, std::move(tree)); - return value; -} - -vespalib::string -SlimeFillerTest::make_exp_il_annotated_string() -{ - using namespace juniper::separators; - vespalib::asciistream exp; - exp << "foo" << unit_separator_string << - " " << unit_separator_string << interlinear_annotation_anchor_string << - "bar" << interlinear_annotation_separator_string << - "baz" << interlinear_annotation_terminator_string << unit_separator_string; - return exp.str(); -} - -vespalib::string -SlimeFillerTest::make_exp_il_annotated_chinese_string() -{ - using namespace juniper::separators; - vespalib::asciistream exp; - exp << "我就是那个" << unit_separator_string << - "大灰狼" << unit_separator_string; - return exp.str(); -} - ArrayFieldValue SlimeFillerTest::make_array() { @@ -330,15 +237,28 @@ SlimeFillerTest::make_map() return map; } +StructFieldValue +SlimeFillerTest::make_nested_value(int i) +{ + StructFieldValue nested(get_data_type("nested")); + StructFieldValue nested2(get_data_type("nested")); + nested.setValue("a", IntFieldValue(42 + 100 * i)); + nested.setValue("b", IntFieldValue(44 + 100 * i)); + nested.setValue("c", IntFieldValue(46 + 100 * i)); + nested2.setValue("a", IntFieldValue(62 + 100 * i)); + nested2.setValue("c", IntFieldValue(66 + 100 * i)); + nested.setValue("d", nested2); + nested.setValue("f", nested2); + return nested; +} + void -SlimeFillerTest::expect_insert(const vespalib::string& exp, const FieldValue& fv, bool tokenize, const std::vector<uint32_t>* matching_elems) +SlimeFillerTest::expect_insert(const vespalib::string& exp, const FieldValue& fv, const std::vector<uint32_t>* matching_elems) { Slime slime; SlimeInserter inserter(slime); - SlimeFiller filler(inserter, tokenize, matching_elems); + SlimeFiller filler(inserter, matching_elems); fv.accept(filler); - SimpleBuffer buf; - JsonFormat::encode(slime, buf, true); auto act = slime_to_string(slime); EXPECT_EQ(exp, act); } @@ -346,28 +266,33 @@ SlimeFillerTest::expect_insert(const vespalib::string& exp, const FieldValue& fv void SlimeFillerTest::expect_insert_filtered(const vespalib::string& exp, const FieldValue& fv, const std::vector<uint32_t>& matching_elems) { - expect_insert(exp, fv, false, &matching_elems); + expect_insert(exp, fv, &matching_elems); } void -SlimeFillerTest::expect_insert(const vespalib::string& exp, const FieldValue& fv, bool tokenize) +SlimeFillerTest::expect_insert(const vespalib::string& exp, const FieldValue& fv) { - expect_insert(exp, fv, tokenize, nullptr); + expect_insert(exp, fv, nullptr); } void -SlimeFillerTest::expect_insert(const vespalib::string& exp, const FieldValue& fv) +SlimeFillerTest::expect_insert(const vespalib::string& exp, const FieldValue& fv, SlimeFillerFilter& filter) { - expect_insert(exp, fv, false); + Slime slime; + SlimeInserter inserter(slime); + SlimeFiller filler(inserter, nullptr, &filter); + fv.accept(filler); + auto act = slime_to_string(slime); + EXPECT_EQ(exp, act); } void -SlimeFillerTest::expect_insert_callback(const vespalib::string& exp, const FieldValue& fv, bool tokenize) +SlimeFillerTest::expect_insert_callback(const vespalib::string& exp, const FieldValue& fv) { Slime slime; SlimeInserter inserter(slime); - MockJuniperConverter converter; - SlimeFiller filler(inserter, tokenize, &converter); + MockStringFieldConverter converter; + SlimeFiller filler(inserter, &converter, nullptr); fv.accept(filler); auto act_null = slime_to_string(slime); EXPECT_EQ("null", act_null); @@ -415,14 +340,8 @@ TEST_F(SlimeFillerTest, insert_string) expect_insert(R"("Foo Bar Baz")", StringFieldValue("Foo Bar Baz")); } { - SCOPED_TRACE("annotated string"); - auto exp = make_exp_il_annotated_string(); - expect_insert(make_slime_string(exp), make_annotated_string(), true); - } - { - SCOPED_TRACE("annotated chinese string"); - auto exp = make_exp_il_annotated_chinese_string(); - expect_insert(make_slime_string(exp), make_annotated_chinese_string(), true); + SCOPED_TRACE("empty string"); + expect_insert(R"("")", StringFieldValue()); } } @@ -569,43 +488,45 @@ TEST_F(SlimeFillerTest, insert_map_filtered) TEST_F(SlimeFillerTest, insert_struct) { - StructFieldValue nested(get_data_type("nested")); - StructFieldValue nested2(get_data_type("nested")); - nested.setValue("a", IntFieldValue(42)); - nested.setValue("b", IntFieldValue(44)); - nested.setValue("c", IntFieldValue(46)); - nested2.setValue("a", IntFieldValue(62)); - nested2.setValue("c", IntFieldValue(66)); - nested.setValue("d", nested2); - nested.setValue("f", nested2); - // Field order depends on assigned field ids, cf. document::Field::calculateIdV7() + auto nested = make_nested_value(0); + // Field order depends on assigned field ids, cf. document::Field::calculateIdV7(), and symbol insertion order in slime expect_insert(R"({"f":{"c":66,"a":62},"c":46,"a":42,"b":44,"d":{"c":66,"a":62}})", nested); + SlimeFillerFilter filter; + filter.add("a").add("c").add("f.a").add("d"); + expect_insert(R"({"f":{"a":62},"a":42,"c":46,"d":{"a":62,"c":66}})", nested, filter); } -TEST_F(SlimeFillerTest, insert_string_with_callback) +TEST_F(SlimeFillerTest, insert_struct_array) { - { - SCOPED_TRACE("plain string"); - using namespace juniper::separators; - vespalib::string exp("Foo Bar Baz"); - StringFieldValue plain_string("Foo Bar Baz"); - expect_insert_callback(exp + unit_separator_string, plain_string, true); - expect_insert_callback(exp, plain_string, false); - } - { - SCOPED_TRACE("annotated string"); - auto exp = make_exp_il_annotated_string(); - auto annotated_string = make_annotated_string(); - expect_insert_callback(exp, annotated_string, true); - expect_insert_callback("foo bar", annotated_string, false); + ArrayFieldValue array(get_data_type("Array<nested>")); + for (int i = 0; i < 3; ++i) { + array.add(make_nested_value(i)); } - { - SCOPED_TRACE("annotated chinese string"); - auto exp = make_exp_il_annotated_chinese_string(); - auto annotated_chinese_string = make_annotated_chinese_string(); - expect_insert_callback(exp, annotated_chinese_string, true); - expect_insert_callback(annotated_chinese_string.getValueRef(), annotated_chinese_string, false); + expect_insert(R"([{"f":{"c":66,"a":62},"c":46,"a":42,"b":44,"d":{"c":66,"a":62}},{"f":{"c":166,"a":162},"c":146,"a":142,"b":144,"d":{"c":166,"a":162}},{"f":{"c":266,"a":262},"c":246,"a":242,"b":244,"d":{"c":266,"a":262}}])", array); + SlimeFillerFilter filter; + filter.add("a").add("c").add("f.a").add("d"); + expect_insert(R"([{"f":{"a":62},"a":42,"c":46,"d":{"a":62,"c":66}},{"f":{"a":162},"a":142,"c":146,"d":{"a":162,"c":166}},{"f":{"a":262},"a":242,"c":246,"d":{"a":262,"c":266}}])", array, filter); +} + +TEST_F(SlimeFillerTest, insert_struct_map) +{ + MapFieldValue map(get_data_type("Map<String,nested>")); + for (int i = 0; i < 3; ++i) { + vespalib::asciistream key; + key << "key" << (i + 1); + map.put(StringFieldValue(key.str()), make_nested_value(i)); } + expect_insert(R"([{"key":"key1","value":{"f":{"c":66,"a":62},"c":46,"a":42,"b":44,"d":{"c":66,"a":62}}},{"key":"key2","value":{"f":{"c":166,"a":162},"c":146,"a":142,"b":144,"d":{"c":166,"a":162}}},{"key":"key3","value":{"f":{"c":266,"a":262},"c":246,"a":242,"b":244,"d":{"c":266,"a":262}}}])", map); + SlimeFillerFilter filter; + filter.add("value.a").add("value.c").add("value.f.a").add("value.d"); + expect_insert(R"([{"key":"key1","value":{"f":{"a":62},"a":42,"c":46,"d":{"a":62,"c":66}}},{"key":"key2","value":{"f":{"a":162},"a":142,"c":146,"d":{"a":162,"c":166}}},{"key":"key3","value":{"f":{"a":262},"a":242,"c":246,"d":{"a":262,"c":266}}}])", map, filter); +} + +TEST_F(SlimeFillerTest, insert_string_with_callback) +{ + vespalib::string exp("Foo Bar Baz"); + StringFieldValue plain_string(exp); + expect_insert_callback(exp, plain_string); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchsummary/src/tests/docsummary/slime_summary/CMakeLists.txt b/searchsummary/src/tests/docsummary/slime_summary/CMakeLists.txt index 344a33952d6..26456dae395 100644 --- a/searchsummary/src/tests/docsummary/slime_summary/CMakeLists.txt +++ b/searchsummary/src/tests/docsummary/slime_summary/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(searchsummary_slime_summary_test_app TEST slime_summary_test.cpp DEPENDS searchsummary + GTest::GTest ) vespa_add_test(NAME searchsummary_slime_summary_test_app COMMAND searchsummary_slime_summary_test_app) diff --git a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp index cbde3d77b4a..fa53cf202ff 100644 --- a/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp +++ b/searchsummary/src/tests/docsummary/slime_summary/slime_summary_test.cpp @@ -1,8 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #include <vespa/document/base/documentid.h> #include <vespa/document/datatype/documenttype.h> -#include <vespa/document/fieldvalue/document.h> #include <vespa/document/fieldvalue/bytefieldvalue.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/document/fieldvalue/doublefieldvalue.h> #include <vespa/document/fieldvalue/floatfieldvalue.h> #include <vespa/document/fieldvalue/intfieldvalue.h> @@ -10,14 +11,14 @@ #include <vespa/document/fieldvalue/rawfieldvalue.h> #include <vespa/document/fieldvalue/shortfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> -#include <vespa/vespalib/testkit/test_kit.h> #include <vespa/searchlib/common/matching_elements.h> -#include <vespa/searchsummary/docsummary/docsumwriter.h> +#include <vespa/searchsummary/docsummary/docsum_store_document.h> #include <vespa/searchsummary/docsummary/docsumstate.h> +#include <vespa/searchsummary/docsummary/docsumwriter.h> #include <vespa/searchsummary/docsummary/keywordextractor.h> -#include <vespa/searchsummary/docsummary/docsum_store_document.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/smart_buffer.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/size_literals.h> using namespace vespalib::slime::convenience; @@ -42,27 +43,27 @@ using document::StructFieldValue; namespace { -struct DocsumFixture : IDocsumStore, GetDocsumsStateCallback { +struct SlimeSummaryTest : testing::Test, IDocsumStore, GetDocsumsStateCallback { std::unique_ptr<DynamicDocsumWriter> writer; StructDataType int_pair_type; DocumentType doc_type; GetDocsumsState state; bool fail_get_mapped_docsum; bool empty_get_mapped_docsum; - DocsumFixture(); - ~DocsumFixture() override; + SlimeSummaryTest(); + ~SlimeSummaryTest() override; void getDocsum(Slime &slime) { Slime slimeOut; SlimeInserter inserter(slimeOut); auto rci = writer->resolveClassInfo(state._args.getResultClassName()); - writer->insertDocsum(rci, 1u, &state, this, inserter); + writer->insertDocsum(rci, 1u, state, this, inserter); vespalib::SmartBuffer buf(4_Ki); BinaryFormat::encode(slimeOut, buf); - EXPECT_GREATER(BinaryFormat::decode(buf.obtain(), slime), 0u); + EXPECT_GT(BinaryFormat::decode(buf.obtain(), slime), 0u); } uint32_t getNumDocs() const override { return 2; } std::unique_ptr<const IDocsumStoreDocument> getMappedDocsum(uint32_t docid) override { - EXPECT_EQUAL(1u, docid); + EXPECT_EQ(1u, docid); if (fail_get_mapped_docsum) { return {}; } @@ -94,7 +95,7 @@ struct DocsumFixture : IDocsumStore, GetDocsumsStateCallback { }; -DocsumFixture::DocsumFixture() +SlimeSummaryTest::SlimeSummaryTest() : writer(), int_pair_type("int_pair"), doc_type("test"), @@ -132,49 +133,54 @@ DocsumFixture::DocsumFixture() doc_type.addField(Field("longdata_field", *DataType::RAW)); doc_type.addField(Field("int_pair_field", int_pair_type)); } -DocsumFixture::~DocsumFixture() = default; +SlimeSummaryTest::~SlimeSummaryTest() = default; } // namespace <unnamed> -TEST_FF("require that docsum can be written as slime", DocsumFixture(), Slime()) { - f1.getDocsum(f2); - EXPECT_EQUAL(f2.get()["int_field"].asLong(), 4u); - EXPECT_EQUAL(f2.get()["short_field"].asLong(), 2u); - EXPECT_EQUAL(f2.get()["byte_field"].asLong(), 1u); - EXPECT_EQUAL(f2.get()["float_field"].asDouble(), 4.5); - EXPECT_EQUAL(f2.get()["double_field"].asDouble(), 8.75); - EXPECT_EQUAL(f2.get()["int64_field"].asLong(), 8u); - EXPECT_EQUAL(f2.get()["string_field"].asString().make_string(), std::string("string")); - EXPECT_EQUAL(f2.get()["data_field"].asData().make_string(), std::string("data")); - EXPECT_EQUAL(f2.get()["longstring_field"].asString().make_string(), std::string("long_string")); - EXPECT_EQUAL(f2.get()["longdata_field"].asData().make_string(), std::string("long_data")); - EXPECT_EQUAL(f2.get()["int_pair_field"]["foo"].asLong(), 1u); - EXPECT_EQUAL(f2.get()["int_pair_field"]["bar"].asLong(), 2u); +TEST_F(SlimeSummaryTest, docsum_can_be_written_as_slime) +{ + Slime s; + getDocsum(s); + EXPECT_EQ(s.get()["int_field"].asLong(), 4u); + EXPECT_EQ(s.get()["short_field"].asLong(), 2u); + EXPECT_EQ(s.get()["byte_field"].asLong(), 1u); + EXPECT_EQ(s.get()["float_field"].asDouble(), 4.5); + EXPECT_EQ(s.get()["double_field"].asDouble(), 8.75); + EXPECT_EQ(s.get()["int64_field"].asLong(), 8u); + EXPECT_EQ(s.get()["string_field"].asString().make_string(), std::string("string")); + EXPECT_EQ(s.get()["data_field"].asData().make_string(), std::string("data")); + EXPECT_EQ(s.get()["longstring_field"].asString().make_string(), std::string("long_string")); + EXPECT_EQ(s.get()["longdata_field"].asData().make_string(), std::string("long_data")); + EXPECT_EQ(s.get()["int_pair_field"]["foo"].asLong(), 1u); + EXPECT_EQ(s.get()["int_pair_field"]["bar"].asLong(), 2u); } -TEST_FF("require that unknown summary class gives empty slime", DocsumFixture(), Slime()) +TEST_F(SlimeSummaryTest, unknown_summary_class_gives_empty_slime) { - f1.state._args.setResultClassName("unknown"); - f1.getDocsum(f2); - EXPECT_TRUE(f2.get().valid()); - EXPECT_EQUAL(vespalib::slime::NIX::ID, f2.get().type().getId()); + state._args.setResultClassName("unknown"); + Slime s; + getDocsum(s); + EXPECT_TRUE(s.get().valid()); + EXPECT_EQ(vespalib::slime::NIX::ID, s.get().type().getId()); } -TEST_FF("require that failure to retrieve docsum store document gives empty slime", DocsumFixture(), Slime()) +TEST_F(SlimeSummaryTest, failure_to_retrieve_docsum_store_document_gives_empty_slime) { - f1.fail_get_mapped_docsum = true; - f1.getDocsum(f2); - EXPECT_TRUE(f2.get().valid()); - EXPECT_EQUAL(vespalib::slime::NIX::ID, f2.get().type().getId()); + fail_get_mapped_docsum = true; + Slime s; + getDocsum(s); + EXPECT_TRUE(s.get().valid()); + EXPECT_EQ(vespalib::slime::NIX::ID, s.get().type().getId()); } -TEST_FF("require that empty docsum store document gives empty object", DocsumFixture(), Slime()) +TEST_F(SlimeSummaryTest, empty_docsum_store_document_gives_empty_object) { - f1.empty_get_mapped_docsum = true; - f1.getDocsum(f2); - EXPECT_TRUE(f2.get().valid()); - EXPECT_EQUAL(vespalib::slime::OBJECT::ID, f2.get().type().getId()); - EXPECT_EQUAL(0u, f2.get().fields()); + empty_get_mapped_docsum = true; + Slime s; + getDocsum(s); + EXPECT_TRUE(s.get().valid()); + EXPECT_EQ(vespalib::slime::OBJECT::ID, s.get().type().getId()); + EXPECT_EQ(0u, s.get().fields()); } -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchsummary/src/tests/docsummary/summary_field_converter/CMakeLists.txt b/searchsummary/src/tests/docsummary/summary_field_converter/CMakeLists.txt deleted file mode 100644 index cfda566ee6c..00000000000 --- a/searchsummary/src/tests/docsummary/summary_field_converter/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchsummary_summary_field_converter_test_app - SOURCES - summary_field_converter_test.cpp - DEPENDS - searchsummary -) -vespa_add_test(NAME searchsummary_summary_field_converter_test_app COMMAND searchsummary_summary_field_converter_test_app) diff --git a/searchsummary/src/tests/docsummary/summary_field_converter/summary_field_converter_test.cpp b/searchsummary/src/tests/docsummary/summary_field_converter/summary_field_converter_test.cpp deleted file mode 100644 index 0eff397bc10..00000000000 --- a/searchsummary/src/tests/docsummary/summary_field_converter/summary_field_converter_test.cpp +++ /dev/null @@ -1,741 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Unit tests for summaryfieldconverter. - -#include <vespa/document/annotation/annotation.h> -#include <vespa/document/annotation/span.h> -#include <vespa/document/annotation/spanlist.h> -#include <vespa/document/annotation/spantree.h> -#include <vespa/document/base/documentid.h> -#include <vespa/document/base/exceptions.h> -#include <vespa/document/base/field.h> -#include <vespa/document/config/documenttypes_config_fwd.h> -#include <vespa/document/datatype/annotationtype.h> -#include <vespa/document/datatype/arraydatatype.h> -#include <vespa/document/datatype/datatype.h> -#include <vespa/document/datatype/documenttype.h> -#include <vespa/document/datatype/structdatatype.h> -#include <vespa/document/datatype/urldatatype.h> -#include <vespa/document/datatype/weightedsetdatatype.h> -#include <vespa/document/datatype/referencedatatype.h> -#include <vespa/document/datatype/tensor_data_type.h> -#include <vespa/document/fieldvalue/arrayfieldvalue.h> -#include <vespa/document/fieldvalue/bytefieldvalue.h> -#include <vespa/document/fieldvalue/document.h> -#include <vespa/document/fieldvalue/doublefieldvalue.h> -#include <vespa/document/fieldvalue/floatfieldvalue.h> -#include <vespa/document/fieldvalue/intfieldvalue.h> -#include <vespa/document/fieldvalue/longfieldvalue.h> -#include <vespa/document/fieldvalue/predicatefieldvalue.h> -#include <vespa/document/fieldvalue/rawfieldvalue.h> -#include <vespa/document/fieldvalue/shortfieldvalue.h> -#include <vespa/document/fieldvalue/stringfieldvalue.h> -#include <vespa/document/fieldvalue/structfieldvalue.h> -#include <vespa/document/fieldvalue/weightedsetfieldvalue.h> -#include <vespa/document/fieldvalue/tensorfieldvalue.h> -#include <vespa/document/fieldvalue/referencefieldvalue.h> -#include <vespa/document/predicate/predicate.h> -#include <vespa/document/repo/configbuilder.h> -#include <vespa/document/repo/fixedtyperepo.h> -#include <vespa/searchsummary/docsummary/summaryfieldconverter.h> -#include <vespa/searchsummary/docsummary/linguisticsannotation.h> -#include <vespa/searchsummary/docsummary/searchdatatype.h> -#include <vespa/searchcommon/common/schema.h> -#include <vespa/vespalib/geo/zcurve.h> -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/data/slime/json_format.h> -#include <vespa/vespalib/data/slime/binary_format.h> -#include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> -#include <vespa/eval/eval/simple_value.h> -#include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/eval/value.h> -#include <vespa/eval/eval/test/value_compare.h> -#include <vespa/vespalib/data/slime/slime.h> - -using document::Annotation; -using document::AnnotationType; -using document::ArrayDataType; -using document::ArrayFieldValue; -using document::ByteFieldValue; -using document::DataType; -using document::Document; -using document::DocumentId; -using document::DocumentType; -using document::DocumentTypeRepo; -using document::DoubleFieldValue; -using document::FeatureSet; -using document::Field; -using document::FieldNotFoundException; -using document::FieldValue; -using document::FloatFieldValue; -using document::IntFieldValue; -using document::LongFieldValue; -using document::Predicate; -using document::PredicateFieldValue; -using document::RawFieldValue; -using document::ReferenceDataType; -using document::ReferenceFieldValue; -using document::ShortFieldValue; -using document::Span; -using document::SpanList; -using document::SpanTree; -using document::StringFieldValue; -using document::StructDataType; -using document::StructFieldValue; -using document::TensorDataType; -using document::TensorFieldValue; -using document::UrlDataType; -using document::WeightedSetDataType; -using document::WeightedSetFieldValue; -using search::index::Schema; -using search::linguistics::SPANTREE_NAME; -using search::linguistics::TERM; -using vespalib::Slime; -using vespalib::eval::SimpleValue; -using vespalib::eval::TensorSpec; -using vespalib::eval::Value; -using vespalib::eval::ValueType; -using vespalib::geo::ZCurve; -using vespalib::slime::Cursor; -using vespalib::string; - -using namespace search::docsummary; - -typedef SummaryFieldConverter SFC; - -namespace { - -struct FieldBlock { - vespalib::string input; - Slime slime; - search::RawBuf binary; - vespalib::string json; - - explicit FieldBlock(const vespalib::string &jsonInput); - ~FieldBlock(); -}; - -FieldBlock::FieldBlock(const vespalib::string &jsonInput) - : input(jsonInput), slime(), binary(1024), json() -{ - size_t used = vespalib::slime::JsonFormat::decode(jsonInput, slime); - EXPECT_TRUE(used > 0); - { - search::SlimeOutputRawBufAdapter adapter(binary); - vespalib::slime::JsonFormat::encode(slime, adapter, true); - json.assign(binary.GetDrainPos(), binary.GetUsedLen()); - binary.reset(); - } - search::SlimeOutputRawBufAdapter adapter(binary); - vespalib::slime::BinaryFormat::encode(slime, adapter); -} - -FieldBlock::~FieldBlock() = default; - -class Test : public vespalib::TestApp { - std::unique_ptr<Schema> _schema; - std::shared_ptr<const DocumentTypeRepo> _documentRepo; - const DocumentType *_documentType; - document::FixedTypeRepo _fixedRepo; - - void setUp(); - void tearDown(); - - const DataType &getDataType(const string &name) const; - const ReferenceDataType& getAsRefType(const string& name) const; - - template <typename T> - T getValueAs(const string &field_name, const Document &doc); - - template <typename T> - T - cvtValueAs(const FieldValue::UP &fv); - - template <typename T> - T - cvtAttributeAs(const FieldValue::UP &fv); - - template <typename T> - T - cvtSummaryAs(bool markup, const FieldValue::UP &fv); - - void checkString(const string &str, const FieldValue *value); - void checkStringForAllConversions(const string& expected, const FieldValue* fv); - void checkData(const search::RawBuf &data, const FieldValue *value); - void checkTensor(const Value::UP &tensor, const FieldValue *value); - template <unsigned int N> - void checkArray(const char *(&str)[N], const FieldValue *value); - void setSummaryField(const string &name); - void setAttributeField(const string &name); - - void requireThatSummaryIsAnUnmodifiedString(); - void requireThatAttributeIsAnUnmodifiedString(); - void requireThatArrayIsFlattenedInSummaryField(); - void requireThatWeightedSetIsFlattenedInSummaryField(); - void requireThatPositionsAreTransformedInSummary(); - void requireThatArrayIsPreservedInAttributeField(); - void requireThatPositionsAreTransformedInAttributeField(); - void requireThatPositionArrayIsTransformedInAttributeField(); - void requireThatPositionWeightedSetIsTransformedInAttributeField(); - void requireThatAttributeCanBePrimitiveTypes(); - void requireThatSummaryCanBePrimitiveTypes(); - void requireThatSummaryHandlesCjk(); - void requireThatSearchDataTypeUsesDefaultDataTypes(); - void requireThatLinguisticsAnnotationUsesDefaultDataTypes(); - void requireThatPredicateIsPrinted(); - void requireThatTensorIsNotConverted(); - void requireThatNonEmptyReferenceIsConvertedToStringWithId(); - void requireThatEmptyReferenceIsConvertedToEmptyString(); - void requireThatReferenceInCompositeTypeEmitsSlimeData(); - const DocumentType &getDocType() const { return *_documentType; } - Document makeDocument(); - StringFieldValue annotateTerm(const string &term); - StringFieldValue makeAnnotatedChineseString(); - StringFieldValue makeAnnotatedString(); - void setSpanTree(StringFieldValue & value, SpanTree::UP tree); -public: - Test(); - ~Test(); - int Main() override; -}; - -DocumenttypesConfig getDocumenttypesConfig() { - using namespace document::config_builder; - DocumenttypesConfigBuilderHelper builder; - const int ref_target_doctype_id = 1234; - const int ref_type_id = 5678; - builder.document(ref_target_doctype_id, "target_dummy_document", - Struct("target_dummy_document.header"), - Struct("target_dummy_document.body")); - builder.document(42, "indexingdocument", - Struct("indexingdocument.header") - .addField("empty", DataType::T_STRING) - .addField("string", DataType::T_STRING) - .addField("plain_string", DataType::T_STRING) - .addField("string_array", Array(DataType::T_STRING)) - .addField("string_wset", Wset(DataType::T_STRING)) - .addField("position1", DataType::T_INT) - .addField("position2", DataType::T_LONG) - .addField("position2_array", Array(DataType::T_LONG)) - .addField("position2_wset", Wset(DataType::T_LONG)) - .addField("uri", UrlDataType::getInstance().getId()) - .addField("uri_array", - Array(UrlDataType::getInstance().getId())) - .addField("int", DataType::T_INT) - .addField("long", DataType::T_LONG) - .addField("short", DataType::T_SHORT) - .addField("byte", DataType::T_BYTE) - .addField("double", DataType::T_DOUBLE) - .addField("float", DataType::T_FLOAT) - .addField("chinese", DataType::T_STRING) - .addField("predicate", DataType::T_PREDICATE) - .addTensorField("tensor", "tensor(x{},y{})") - .addField("ref", ref_type_id) - .addField("nested", Struct("indexingdocument.header.nested") - .addField("inner_ref", ref_type_id)), - Struct("indexingdocument.body")) - .referenceType(ref_type_id, ref_target_doctype_id); - return builder.config(); -} - -Test::Test() : - _documentRepo(std::make_unique<DocumentTypeRepo>(getDocumenttypesConfig())), - _documentType(_documentRepo->getDocumentType("indexingdocument")), - _fixedRepo(*_documentRepo, *_documentType) -{ - ASSERT_TRUE(_documentType); -} - -Test::~Test() {} - -#define TEST_CALL(func) \ - TEST_DO(setUp()); \ - TEST_DO(func); \ - TEST_DO(tearDown()) - -int -Test::Main() -{ - TEST_INIT("summaryfieldconverter_test"); - - TEST_CALL(requireThatSummaryIsAnUnmodifiedString()); - TEST_CALL(requireThatAttributeIsAnUnmodifiedString()); - TEST_CALL(requireThatArrayIsFlattenedInSummaryField()); - TEST_CALL(requireThatWeightedSetIsFlattenedInSummaryField()); - TEST_CALL(requireThatPositionsAreTransformedInSummary()); - TEST_CALL(requireThatArrayIsPreservedInAttributeField()); - TEST_CALL(requireThatPositionsAreTransformedInAttributeField()); - TEST_CALL(requireThatPositionArrayIsTransformedInAttributeField()); - TEST_CALL(requireThatPositionWeightedSetIsTransformedInAttributeField()); - TEST_CALL(requireThatAttributeCanBePrimitiveTypes()); - TEST_CALL(requireThatSummaryCanBePrimitiveTypes()); - TEST_CALL(requireThatSummaryHandlesCjk()); - TEST_CALL(requireThatSearchDataTypeUsesDefaultDataTypes()); - TEST_CALL(requireThatLinguisticsAnnotationUsesDefaultDataTypes()); - TEST_CALL(requireThatPredicateIsPrinted()); - TEST_CALL(requireThatTensorIsNotConverted()); - TEST_CALL(requireThatNonEmptyReferenceIsConvertedToStringWithId()); - TEST_CALL(requireThatEmptyReferenceIsConvertedToEmptyString()); - TEST_CALL(requireThatReferenceInCompositeTypeEmitsSlimeData()); - - TEST_DONE(); -} - -void Test::setUp() { - _schema = std::make_unique<Schema>(); -} - -void Test::tearDown() { -} - -const DataType &Test::getDataType(const string &name) const { - const DataType *type = _documentRepo->getDataType(*_documentType, name); - ASSERT_TRUE(type); - return *type; -} - -StringFieldValue Test::makeAnnotatedString() { - auto span_list_up = std::make_unique<SpanList>(); - auto span_list = span_list_up.get(); - auto tree = std::make_unique<SpanTree>(SPANTREE_NAME, std::move(span_list_up)); - // Annotations don't have to be added sequentially. - tree->annotate(span_list->add(std::make_unique<Span>(8, 3)), - Annotation(*TERM, std::make_unique<StringFieldValue>("Annotation"))); - tree->annotate(span_list->add(std::make_unique<Span>(0, 3)), *TERM); - tree->annotate(span_list->add(std::make_unique<Span>(4, 3)), *TERM); - tree->annotate(span_list->add(std::make_unique<Span>(4, 3)), - Annotation(*TERM, std::make_unique<StringFieldValue>("Multiple"))); - tree->annotate(span_list->add(std::make_unique<Span>(1, 2)), - Annotation(*TERM, std::make_unique<StringFieldValue>("Overlap"))); - StringFieldValue value("Foo Bar Baz"); - setSpanTree(value, std::move(tree)); - return value; -} - -StringFieldValue Test::annotateTerm(const string &term) { - auto tree = std::make_unique<SpanTree>(SPANTREE_NAME, std::make_unique<Span>(0, term.size())); - tree->annotate(tree->getRoot(), *TERM); - StringFieldValue value(term); - setSpanTree(value, std::move(tree)); - return value; -} - -void Test::setSpanTree(StringFieldValue & value, SpanTree::UP tree) { - StringFieldValue::SpanTrees trees; - trees.push_back(std::move(tree)); - value.setSpanTrees(trees, _fixedRepo); -} - -StringFieldValue Test::makeAnnotatedChineseString() { - auto span_list_up = std::make_unique<SpanList>(); - auto span_list = span_list_up.get(); - auto tree = std::make_unique<SpanTree>(SPANTREE_NAME, std::move(span_list_up)); - // These chinese characters each use 3 bytes in their UTF8 encoding. - tree->annotate(span_list->add(std::make_unique<Span>(0, 15)), *TERM); - tree->annotate(span_list->add(std::make_unique<Span>(15, 9)), *TERM); - StringFieldValue value("我就是那个大灰狼"); - setSpanTree(value, std::move(tree)); - return value; -} - -Document Test::makeDocument() { - Document doc(getDocType(), DocumentId("id:ns:indexingdocument::")); - doc.setRepo(*_documentRepo); - doc.setValue("string", makeAnnotatedString()); - - doc.setValue("plain_string", StringFieldValue("Plain")); - - ArrayFieldValue array(getDataType("Array<String>")); - array.add(annotateTerm("\"foO\"")); - array.add(annotateTerm("ba\\R")); - doc.setValue("string_array", array); - - WeightedSetFieldValue wset(getDataType("WeightedSet<String>")); - wset.add(annotateTerm("\"foo\""), 2); - wset.add(annotateTerm("ba\\r"), 4); - doc.setValue("string_wset", wset); - - doc.setValue("position1", IntFieldValue(5)); - - doc.setValue("position2", LongFieldValue(ZCurve::encode(4, 2))); - - StructFieldValue uri(getDataType("url")); - uri.setValue("all", annotateTerm("http://www.example.com:42/foobar?q#frag")); - uri.setValue("scheme", annotateTerm("http")); - uri.setValue("host", annotateTerm("www.example.com")); - uri.setValue("port", annotateTerm("42")); - uri.setValue("path", annotateTerm("foobar")); - uri.setValue("query", annotateTerm("q")); - uri.setValue("fragment", annotateTerm("frag")); - doc.setValue("uri", uri); - - ArrayFieldValue uri_array(getDataType("Array<url>")); - uri.setValue("all", annotateTerm("http://www.example.com:80/foobar?q#frag")); - uri.setValue("port", annotateTerm("80")); - uri_array.add(uri); - uri.setValue("all", annotateTerm("https://www.example.com:443/foo?q#frag")); - uri.setValue("scheme", annotateTerm("https")); - uri.setValue("path", annotateTerm("foo")); - uri.setValue("port", annotateTerm("443")); - uri_array.add(uri); - doc.setValue("uri_array", uri_array); - - ArrayFieldValue position2_array(getDataType("Array<Long>")); - position2_array.add(LongFieldValue(ZCurve::encode(4, 2))); - position2_array.add(LongFieldValue(ZCurve::encode(4, 4))); - doc.setValue("position2_array", position2_array); - - WeightedSetFieldValue position2_wset(getDataType("WeightedSet<Long>")); - position2_wset.add(LongFieldValue(ZCurve::encode(4, 2)), 4); - position2_wset.add(LongFieldValue(ZCurve::encode(4, 4)), 2); - doc.setValue("position2_wset", position2_wset); - - doc.setValue("int", IntFieldValue(42)); - doc.setValue("long", LongFieldValue(84)); - doc.setValue("short", ShortFieldValue(21)); - doc.setValue("byte", ByteFieldValue(11)); - doc.setValue("double", DoubleFieldValue(0.4)); - doc.setValue("float", FloatFieldValue(0.2f)); - - doc.setValue("chinese", makeAnnotatedChineseString()); - return doc; -} - -template <typename T> -T Test::getValueAs(const string &field_name, const Document &doc) { - FieldValue::UP fv(doc.getValue(field_name)); - const T *value = dynamic_cast<const T *>(fv.get()); - ASSERT_TRUE(value); - return *value; -} - -template <typename T> -T -Test::cvtValueAs(const FieldValue::UP &fv) -{ - ASSERT_TRUE(fv.get() != NULL); - const T *value = dynamic_cast<const T *>(fv.get()); - ASSERT_TRUE(value); - return *value; -} - -template <typename T> -T -Test::cvtAttributeAs(const FieldValue::UP &fv) -{ - ASSERT_TRUE(fv.get() != NULL); - return cvtValueAs<T>(fv); -} - -template <typename T> -T -Test::cvtSummaryAs(bool markup, const FieldValue::UP &fv) -{ - ASSERT_TRUE(fv.get() != NULL); - FieldValue::UP r = SFC::convertSummaryField(markup, *fv); - return cvtValueAs<T>(r); -} - -void Test::checkString(const string &str, const FieldValue *value) { - ASSERT_TRUE(value); - const StringFieldValue *s = dynamic_cast<const StringFieldValue *>(value); - ASSERT_TRUE(s); - // fprintf(stderr, ">>>%s<<< >>>%s<<<\n", str.c_str(), s->getValue().c_str()); - EXPECT_EQUAL(str, s->getValue()); -} - -void Test::checkData(const search::RawBuf &buf, const FieldValue *value) { - ASSERT_TRUE(value); - const RawFieldValue *s = dynamic_cast<const RawFieldValue *>(value); - ASSERT_TRUE(s); - auto got = s->getAsRaw(); - ASSERT_EQUAL(buf.GetUsedLen(), got.second); - EXPECT_TRUE(memcmp(buf.GetDrainPos(), got.first, got.second) == 0); -} - -void Test::checkTensor(const Value::UP &tensor, const FieldValue *value) { - ASSERT_TRUE(value); - const TensorFieldValue *s = dynamic_cast<const TensorFieldValue *>(value); - ASSERT_TRUE(s); - auto tvalue = s->getAsTensorPtr(); - EXPECT_EQUAL(tensor.get() != nullptr, tvalue != nullptr); - if (tensor) { - EXPECT_EQUAL(*tensor, *tvalue); - } -} - -template <unsigned int N> -void Test::checkArray(const char *(&str)[N], const FieldValue *value) { - ASSERT_TRUE(value); - const ArrayFieldValue *a = dynamic_cast<const ArrayFieldValue *>(value); - ASSERT_TRUE(a); - EXPECT_EQUAL(N, a->size()); - for (size_t i = 0; i < a->size() && i < N; ++i) { - checkString(str[i], &(*a)[i]); - } -} - -void Test::setSummaryField(const string &field) { - _schema->addSummaryField(Schema::Field(field, search::index::schema::DataType::STRING)); -} - -void Test::setAttributeField(const string &field) { - _schema->addAttributeField(Schema::Field(field, search::index::schema::DataType::STRING)); -} - -void Test::requireThatSummaryIsAnUnmodifiedString() { - setSummaryField("string"); - Document summary = makeDocument(); - checkString("Foo Bar Baz", SFC::convertSummaryField(false, - *summary.getValue("string")).get()); -} - -void Test::requireThatAttributeIsAnUnmodifiedString() { - setAttributeField("string"); - Document attribute = makeDocument(); - checkString("Foo Bar Baz", - attribute.getValue("string").get()); -} - -void Test::requireThatArrayIsFlattenedInSummaryField() { - setSummaryField("string_array"); - Document summary = makeDocument(); - FieldBlock expect("[\"\\\"foO\\\"\",\"ba\\\\R\"]"); - checkData(expect.binary, - SFC::convertSummaryField(false, - *summary.getValue("string_array")).get()); -} - -void Test::requireThatWeightedSetIsFlattenedInSummaryField() { - setSummaryField("string_wset"); - Document summary = makeDocument(); - FieldBlock expect("[{\"item\":\"\\\"foo\\\"\",\"weight\":2},{\"item\":\"ba\\\\r\",\"weight\":4}]"); - checkData(expect.binary, - SFC::convertSummaryField(false, - *summary.getValue("string_wset")).get()); -} - -void Test::requireThatPositionsAreTransformedInSummary() { - setSummaryField("position1"); - setSummaryField("position2"); - Document summary = makeDocument(); - FieldValue::UP fv = summary.getValue("position1"); - EXPECT_EQUAL(5, cvtSummaryAs<IntFieldValue>(false, fv).getValue()); - FieldValue::UP fv2 = summary.getValue("position2"); - EXPECT_EQUAL(24, cvtSummaryAs<LongFieldValue>(false, fv2).getValue()); -} - -void Test::requireThatArrayIsPreservedInAttributeField() { - setAttributeField("string_array"); - Document attribute = makeDocument(); - const char *array[] = { "\"foO\"", "ba\\R" }; - checkArray(array, - attribute.getValue("string_array").get()); -} - -void Test::requireThatPositionsAreTransformedInAttributeField() { - setAttributeField("position1"); - setAttributeField("position2"); - Document attr = makeDocument(); - FieldValue::UP fv = attr.getValue("position1"); - EXPECT_EQUAL(5, cvtAttributeAs<IntFieldValue>(fv).getValue()); - fv = attr.getValue("position2"); - EXPECT_EQUAL(24, cvtAttributeAs<LongFieldValue>(fv).getValue()); -} - -void Test::requireThatPositionArrayIsTransformedInAttributeField() { - setAttributeField("position2_array"); - Document attr = makeDocument(); - FieldValue::UP fv = attr.getValue("position2_array"); - ArrayFieldValue a = cvtAttributeAs<ArrayFieldValue>(fv); - EXPECT_EQUAL(2u, a.size()); - EXPECT_EQUAL(24, dynamic_cast<LongFieldValue &>(a[0]).getValue()); - EXPECT_EQUAL(48, dynamic_cast<LongFieldValue &>(a[1]).getValue()); -} - -void Test::requireThatPositionWeightedSetIsTransformedInAttributeField() { - setAttributeField("position2_wset"); - Document attr = makeDocument(); - FieldValue::UP fv = attr.getValue("position2_wset"); - WeightedSetFieldValue w = cvtAttributeAs<WeightedSetFieldValue>(fv); - EXPECT_EQUAL(2u, w.size()); - WeightedSetFieldValue::iterator it = w.begin(); - EXPECT_EQUAL(24, dynamic_cast<const LongFieldValue&>(*it->first).getValue()); - EXPECT_EQUAL(4, dynamic_cast<IntFieldValue &>(*it->second).getValue()); - ++it; - EXPECT_EQUAL(48, dynamic_cast<const LongFieldValue&>(*it->first).getValue()); - EXPECT_EQUAL(2, dynamic_cast<IntFieldValue &>(*it->second).getValue()); -} - -void Test::requireThatAttributeCanBePrimitiveTypes() { - setAttributeField("int"); - setAttributeField("long"); - setAttributeField("short"); - setAttributeField("byte"); - setAttributeField("double"); - setAttributeField("float"); - Document attribute = makeDocument(); - FieldValue::UP fv = attribute.getValue("int"); - EXPECT_EQUAL(42, cvtAttributeAs<IntFieldValue>(fv).getValue()); - fv = attribute.getValue("long"); - EXPECT_EQUAL(84, cvtAttributeAs<LongFieldValue>(fv).getValue()); - fv = attribute.getValue("short"); - EXPECT_EQUAL(21, cvtAttributeAs<ShortFieldValue>(fv).getValue()); - fv = attribute.getValue("byte"); - EXPECT_EQUAL(11, cvtAttributeAs<ByteFieldValue>(fv).getValue()); - fv = attribute.getValue("double"); - EXPECT_EQUAL(0.4, cvtAttributeAs<DoubleFieldValue>(fv).getValue()); - fv = attribute.getValue("float"); - EXPECT_EQUAL(0.2f, cvtAttributeAs<FloatFieldValue>(fv).getValue()); -} - -void Test::requireThatSummaryCanBePrimitiveTypes() { - setSummaryField("int"); - setSummaryField("long"); - setSummaryField("short"); - setSummaryField("byte"); - setSummaryField("double"); - setSummaryField("float"); - Document summary = makeDocument(); - FieldValue::UP fv = summary.getValue("int"); - EXPECT_EQUAL(42, cvtSummaryAs<IntFieldValue>(false, fv).getValue()); - fv = summary.getValue("long"); - EXPECT_EQUAL(84, cvtSummaryAs<LongFieldValue>(false, fv).getValue()); - fv = summary.getValue("short"); - EXPECT_EQUAL(21, cvtSummaryAs<ShortFieldValue>(false, fv).getValue()); - fv = summary.getValue("byte"); - EXPECT_EQUAL(11, cvtSummaryAs<ShortFieldValue>(false, fv).getValue()); - fv = summary.getValue("double"); - EXPECT_EQUAL(0.4, cvtSummaryAs<DoubleFieldValue>(false, fv).getValue()); - fv = summary.getValue("float"); - EXPECT_EQUAL(0.2f, cvtSummaryAs<FloatFieldValue>(false, fv).getValue()); -} - -void Test::requireThatSummaryHandlesCjk() { - Document summary = makeDocument(); - FieldValue::UP fv = summary.getValue("chinese"); - EXPECT_EQUAL("我就是那个\037大灰狼\037", - cvtSummaryAs<StringFieldValue>(true, fv).getValue()); -} - -void Test::requireThatSearchDataTypeUsesDefaultDataTypes() { - const StructDataType *uri = - dynamic_cast<const StructDataType *>(SearchDataType::URI); - ASSERT_TRUE(uri); - ASSERT_TRUE(uri->hasField("all")); - ASSERT_TRUE(uri->hasField("scheme")); - ASSERT_TRUE(uri->hasField("host")); - ASSERT_TRUE(uri->hasField("port")); - ASSERT_TRUE(uri->hasField("path")); - ASSERT_TRUE(uri->hasField("query")); - ASSERT_TRUE(uri->hasField("fragment")); - EXPECT_EQUAL(*DataType::STRING, uri->getField("all").getDataType()); - EXPECT_EQUAL(*DataType::STRING, uri->getField("scheme").getDataType()); - EXPECT_EQUAL(*DataType::STRING, uri->getField("host").getDataType()); - EXPECT_EQUAL(*DataType::STRING, uri->getField("port").getDataType()); - EXPECT_EQUAL(*DataType::STRING, uri->getField("path").getDataType()); - EXPECT_EQUAL(*DataType::STRING, uri->getField("query").getDataType()); - EXPECT_EQUAL(*DataType::STRING, uri->getField("fragment").getDataType()); -} - -void Test::requireThatLinguisticsAnnotationUsesDefaultDataTypes() { - EXPECT_EQUAL(*AnnotationType::TERM, *search::linguistics::TERM); - ASSERT_TRUE(AnnotationType::TERM->getDataType()); - ASSERT_TRUE(search::linguistics::TERM->getDataType()); - EXPECT_EQUAL(*AnnotationType::TERM->getDataType(), - *search::linguistics::TERM->getDataType()); -} - -void -Test::requireThatPredicateIsPrinted() -{ - auto input = std::make_unique<Slime>(); - Cursor &obj = input->setObject(); - obj.setLong(Predicate::NODE_TYPE, Predicate::TYPE_FEATURE_SET); - obj.setString(Predicate::KEY, "foo"); - Cursor &arr = obj.setArray(Predicate::SET); - arr.addString("bar"); - - Document doc(getDocType(), DocumentId("id:ns:indexingdocument::")); - doc.setRepo(*_documentRepo); - doc.setValue("predicate", PredicateFieldValue(std::move(input))); - - checkString("'foo' in ['bar']\n", - SFC::convertSummaryField(false, *doc.getValue("predicate")).get()); -} - -Value::UP make_tensor(const TensorSpec &spec) { - return SimpleValue::from_spec(spec); -} - -void -Test::requireThatTensorIsNotConverted() -{ - TensorDataType tensorDataType(ValueType::from_spec("tensor(x{},y{})")); - TensorFieldValue tensorFieldValue(tensorDataType); - tensorFieldValue = make_tensor(TensorSpec("tensor(x{},y{})") - .add({{"x", "4"}, {"y", "5"}}, 7)); - Document doc(getDocType(), DocumentId("id:ns:indexingdocument::")); - doc.setRepo(*_documentRepo); - doc.setValue("tensor", tensorFieldValue); - - TEST_CALL(checkTensor(make_tensor(TensorSpec("tensor(x{},y{})") - .add({{"x", "4"}, {"y", "5"}}, 7)), - SFC::convertSummaryField(false, - *doc.getValue("tensor")).get())); - doc.setValue("tensor", TensorFieldValue()); - - TEST_CALL(checkTensor(Value::UP(), - SFC::convertSummaryField(false, - *doc.getValue("tensor")).get())); -} - -void Test::checkStringForAllConversions(const string& expected, const FieldValue* fv) { - ASSERT_TRUE(fv != nullptr); - checkString(expected, SFC::convertSummaryField(false, *fv).get()); -} - -const ReferenceDataType& Test::getAsRefType(const string& name) const { - return dynamic_cast<const ReferenceDataType&>(getDataType(name)); -} - -void Test::requireThatNonEmptyReferenceIsConvertedToStringWithId() { - Document doc(getDocType(), DocumentId("id:ns:indexingdocument::")); - doc.setRepo(*_documentRepo); - doc.setValue("ref", ReferenceFieldValue( - getAsRefType("Reference<target_dummy_document>"), - DocumentId("id:ns:target_dummy_document::foo"))); - - checkStringForAllConversions("id:ns:target_dummy_document::foo", - doc.getValue("ref").get()); -} - -void Test::requireThatEmptyReferenceIsConvertedToEmptyString() { - Document doc(getDocType(), DocumentId("id:ns:indexingdocument::")); - doc.setRepo(*_documentRepo); - doc.setValue("ref", ReferenceFieldValue( - getAsRefType("Reference<target_dummy_document>"))); - - checkStringForAllConversions("", doc.getValue("ref").get()); - -} - -// Own test for this to ensure that SlimeFiller code path is executed, -// as this only triggers for composite field types. -void Test::requireThatReferenceInCompositeTypeEmitsSlimeData() { - Document doc(getDocType(), DocumentId("id:ns:indexingdocument::")); - doc.setRepo(*_documentRepo); - - StructFieldValue sfv(getDataType("indexingdocument.header.nested")); - sfv.setValue("inner_ref", ReferenceFieldValue( - getAsRefType("Reference<target_dummy_document>"), - DocumentId("id:ns:target_dummy_document::foo"))); - doc.setValue("nested", sfv); - - FieldBlock expect(R"({"inner_ref":"id:ns:target_dummy_document::foo"})"); - checkData(expect.binary, - SFC::convertSummaryField(false, *doc.getValue("nested")).get()); -} - -} // namespace - -TEST_APPHOOK(Test); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt index 6aba9614e73..37ee0697149 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt +++ b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt @@ -9,6 +9,7 @@ vespa_add_library(searchsummary_docsummary OBJECT check_undefined_value_visitor.cpp copy_dfw.cpp docsum_field_writer.cpp + docsum_field_writer_commands.cpp docsum_field_writer_factory.cpp docsum_store_document.cpp docsumstate.cpp @@ -34,6 +35,7 @@ vespa_add_library(searchsummary_docsummary OBJECT searchdatatype.cpp simple_dfw.cpp slime_filler.cpp + slime_filler_filter.cpp struct_fields_resolver.cpp struct_map_attribute_combiner_dfw.cpp summaryfeaturesdfw.cpp diff --git a/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.cpp index 82f3d086b79..b36a2f8383e 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "annotation_converter.h" +#include "i_juniper_converter.h" #include "linguisticsannotation.h" #include <vespa/document/annotation/alternatespanlist.h> #include <vespa/document/annotation/annotation.h> @@ -12,8 +13,8 @@ #include <vespa/vespalib/util/exceptions.h> #include <utility> -using document::Annotation; using document::AlternateSpanList; +using document::Annotation; using document::FieldValue; using document::SimpleSpanList; using document::Span; @@ -27,10 +28,10 @@ namespace search::docsummary { namespace { -vespalib::string -getSpanString(const vespalib::string &s, const Span &span) +vespalib::stringref +getSpanString(vespalib::stringref s, const Span &span) { - return vespalib::string(&s[span.from()], &s[span.from() + span.length()]); + return {s.data() + span.from(), static_cast<size_t>(span.length())}; } struct SpanFinder : SpanTreeVisitor { @@ -78,6 +79,16 @@ const StringFieldValue &ensureStringFieldValue(const FieldValue &value) { } +AnnotationConverter::AnnotationConverter(IJuniperConverter& juniper_converter) + : IStringFieldConverter(), + _juniper_converter(juniper_converter), + _text(), + _out() +{ +} + +AnnotationConverter::~AnnotationConverter() = default; + template <typename ForwardIt> void AnnotationConverter::handleAnnotations(const document::Span& span, ForwardIt it, ForwardIt last) { @@ -85,28 +96,28 @@ AnnotationConverter::handleAnnotations(const document::Span& span, ForwardIt it, if (annCnt > 1 || (annCnt == 1 && it->second)) { annotateSpans(span, it, last); } else { - out << getSpanString(text, span) << juniper::separators::unit_separator_string; + _out << getSpanString(_text, span) << juniper::separators::unit_separator_string; } } template <typename ForwardIt> void AnnotationConverter::annotateSpans(const document::Span& span, ForwardIt it, ForwardIt last) { - out << juniper::separators::interlinear_annotation_anchor_string // ANCHOR - << (getSpanString(text, span)) - << juniper::separators::interlinear_annotation_separator_string; // SEPARATOR + _out << juniper::separators::interlinear_annotation_anchor_string // ANCHOR + << (getSpanString(_text, span)) + << juniper::separators::interlinear_annotation_separator_string; // SEPARATOR while (it != last) { if (it->second) { - out << ensureStringFieldValue(*it->second).getValue(); + _out << ensureStringFieldValue(*it->second).getValue(); } else { - out << getSpanString(text, span); + _out << getSpanString(_text, span); } if (++it != last) { - out << " "; + _out << " "; } } - out << juniper::separators::interlinear_annotation_terminator_string // TERMINATOR - << juniper::separators::unit_separator_string; + _out << juniper::separators::interlinear_annotation_terminator_string // TERMINATOR + << juniper::separators::unit_separator_string; } void @@ -114,11 +125,11 @@ AnnotationConverter::handleIndexingTerms(const StringFieldValue& value) { StringFieldValue::SpanTrees trees = value.getSpanTrees(); const SpanTree *tree = StringFieldValue::findTree(trees, linguistics::SPANTREE_NAME); - typedef std::pair<Span, const FieldValue *> SpanTerm; - typedef std::vector<SpanTerm> SpanTermVector; + using SpanTerm = std::pair<Span, const FieldValue *>; + using SpanTermVector = std::vector<SpanTerm>; if (!tree) { // Treat a string without annotations as a single span. - SpanTerm str(Span(0, text.size()), + SpanTerm str(Span(0, _text.size()), static_cast<const FieldValue*>(nullptr)); handleAnnotations(str.first, &str, &str + 1); return; @@ -126,7 +137,7 @@ AnnotationConverter::handleIndexingTerms(const StringFieldValue& value) SpanTermVector terms; for (const Annotation& annotation : *tree) { // For now, skip any composite spans. - const Span *span = dynamic_cast<const Span*>(annotation.getSpanNode()); + const auto *span = dynamic_cast<const Span*>(annotation.getSpanNode()); if ((span != nullptr) && annotation.valid() && (annotation.getType() == *linguistics::TERM)) { terms.push_back(std::make_pair(getSpan(*span), @@ -148,11 +159,20 @@ AnnotationConverter::handleIndexingTerms(const StringFieldValue& value) handleAnnotations(it_begin->first, it_begin, it); endPos = it_begin->first.from() + it_begin->first.length(); } - int32_t wantEndPos = text.size(); + int32_t wantEndPos = _text.size(); if (endPos < wantEndPos) { Span tmpSpan(endPos, wantEndPos - endPos); handleAnnotations(tmpSpan, ite, ite); } } +void +AnnotationConverter::convert(const StringFieldValue &input, vespalib::slime::Inserter& inserter) +{ + _out.clear(); + _text = input.getValueRef(); + handleIndexingTerms(input); + _juniper_converter.convert(_out.str(), inserter); +} + } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.h b/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.h index 37e3c18606e..59b03c64540 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.h @@ -2,34 +2,37 @@ #pragma once -#include <vespa/vespalib/stllike/string.h> +#include "i_string_field_converter.h" +#include <vespa/vespalib/stllike/asciistream.h> -namespace document -{ -class Span; -class StringFieldValue; -} +namespace document { class Span; } namespace vespalib { class asciistream; } namespace search::docsummary { +class IJuniperConverter; + /* * Class converting a string field value with annotations into a string - * with interlinear annotations used by juniper. + * with interlinear annotations used by juniper before passing it to + * the juniper converter. */ -struct AnnotationConverter { - const vespalib::string text; - vespalib::asciistream& out; +class AnnotationConverter : public IStringFieldConverter +{ + IJuniperConverter& _juniper_converter; + vespalib::stringref _text; + vespalib::asciistream _out; template <typename ForwardIt> void handleAnnotations(const document::Span& span, ForwardIt it, ForwardIt last); template <typename ForwardIt> void annotateSpans(const document::Span& span, ForwardIt it, ForwardIt last); -public: - AnnotationConverter(const vespalib::string& s, vespalib::asciistream& stream) - : text(s), out(stream) {} void handleIndexingTerms(const document::StringFieldValue& value); +public: + AnnotationConverter(IJuniperConverter& juniper_converter); + ~AnnotationConverter() override; + void convert(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp index 6e00511398c..889169f8888 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.cpp @@ -41,7 +41,7 @@ AttributeCombinerDFW::create(const vespalib::string &fieldName, IAttributeContex { StructFieldsResolver structFields(fieldName, attrCtx, true); if (structFields.has_error()) { - return std::unique_ptr<DocsumFieldWriter>(); + return {}; } else if (structFields.is_map_of_struct()) { return std::make_unique<StructMapAttributeCombinerDFW>(fieldName, structFields, filter_elements, std::move(matching_elems_fields)); } @@ -49,15 +49,15 @@ AttributeCombinerDFW::create(const vespalib::string &fieldName, IAttributeContex } void -AttributeCombinerDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType, vespalib::slime::Inserter &target) const +AttributeCombinerDFW::insertField(uint32_t docid, GetDocsumsState& state, vespalib::slime::Inserter &target) const { - auto& fieldWriterState = state->_fieldWriterStates[_stateIndex]; + auto& fieldWriterState = state._fieldWriterStates[_stateIndex]; if (!fieldWriterState) { const MatchingElements *matching_elements = nullptr; if (_filter_elements) { - matching_elements = &state->get_matching_elements(*_matching_elems_fields); + matching_elements = &state.get_matching_elements(*_matching_elems_fields); } - fieldWriterState = allocFieldWriterState(*state->_attrCtx, state->get_stash(), matching_elements); + fieldWriterState = allocFieldWriterState(*state._attrCtx, state.get_stash(), matching_elements); } fieldWriterState->insertField(docid, target); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h index 33b61718392..0e1163df5e2 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/attribute_combiner_dfw.h @@ -39,7 +39,7 @@ public: bool setFieldWriterStateIndex(uint32_t fieldWriterStateIndex) override; static std::unique_ptr<DocsumFieldWriter> create(const vespalib::string &fieldName, search::attribute::IAttributeContext &attrCtx, bool filter_elements, std::shared_ptr<MatchingElementsFields> matching_elems_fields); - void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) const override; + void insertField(uint32_t docid, GetDocsumsState& state, vespalib::slime::Inserter &target) const override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp index ce08da7f7f1..74d67aabe88 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp @@ -25,12 +25,12 @@ using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; using search::attribute::IMultiValueAttribute; using search::attribute::IMultiValueReadView; +using vespalib::Issue; using vespalib::Memory; +using vespalib::eval::Value; using vespalib::slime::Cursor; using vespalib::slime::Inserter; using vespalib::slime::Symbol; -using vespalib::eval::Value; -using vespalib::Issue; namespace search::docsummary { @@ -53,16 +53,16 @@ public: explicit SingleAttrDFW(const vespalib::string & attrName) : AttrDFW(attrName) { } - void insertField(uint32_t docid, GetDocsumsState *state, ResType, Inserter &target) const override; - bool isDefaultValue(uint32_t docid, const GetDocsumsState * state) const override { - return get_attribute(*state).isUndefined(docid); + void insertField(uint32_t docid, GetDocsumsState& state, Inserter &target) const override; + bool isDefaultValue(uint32_t docid, const GetDocsumsState& state) const override { + return get_attribute(state).isUndefined(docid); } }; void -SingleAttrDFW::insertField(uint32_t docid, GetDocsumsState * state, ResType, Inserter &target) const +SingleAttrDFW::insertField(uint32_t docid, GetDocsumsState& state, Inserter &target) const { - const auto& v = get_attribute(*state); + const auto& v = get_attribute(state); switch (v.getBasicType()) { case BasicType::Type::UINT2: case BasicType::Type::UINT4: @@ -253,7 +253,7 @@ public: } } bool setFieldWriterStateIndex(uint32_t fieldWriterStateIndex) override; - void insertField(uint32_t docid, GetDocsumsState* state, ResType, Inserter& target) const override; + void insertField(uint32_t docid, GetDocsumsState& state, Inserter& target) const override; }; bool @@ -301,16 +301,16 @@ make_field_writer_state(const vespalib::string& field_name, const IAttributeVect } void -MultiAttrDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType, vespalib::slime::Inserter &target) const +MultiAttrDFW::insertField(uint32_t docid, GetDocsumsState& state, vespalib::slime::Inserter &target) const { - auto& field_writer_state = state->_fieldWriterStates[_state_index]; + auto& field_writer_state = state._fieldWriterStates[_state_index]; if (!field_writer_state) { const MatchingElements *matching_elements = nullptr; if (_filter_elements) { - matching_elements = &state->get_matching_elements(*_matching_elems_fields); + matching_elements = &state.get_matching_elements(*_matching_elems_fields); } - const auto& attr = get_attribute(*state); - field_writer_state = make_field_writer_state(getAttributeName(), attr, state->get_stash(), matching_elements); + const auto& attr = get_attribute(state); + field_writer_state = make_field_writer_state(getAttributeName(), attr, state.get_stash(), matching_elements); } field_writer_state->insertField(docid, target); } @@ -347,7 +347,7 @@ AttributeDFWFactory::create(const IAttributeManager& attr_mgr, const auto* attr = ctx->getAttribute(attr_name); if (attr == nullptr) { Issue::report("No valid attribute vector found: '%s'", attr_name.c_str()); - return std::unique_ptr<DocsumFieldWriter>(); + return {}; } if (attr->hasMultiValue()) { return create_multi_writer(*attr, filter_elements, std::move(matching_elems_fields)); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.cpp index 2dc04c03845..94e5420881b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.cpp @@ -17,7 +17,7 @@ CopyDFW::CopyDFW(const vespalib::string& inputField) CopyDFW::~CopyDFW() = default; void -CopyDFW::insertField(uint32_t, const IDocsumStoreDocument* doc, GetDocsumsState *, ResType, +CopyDFW::insertField(uint32_t, const IDocsumStoreDocument* doc, GetDocsumsState&, vespalib::slime::Inserter &target) const { if (doc != nullptr) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.h index 76c10f47bf1..175bc5b3246 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/copy_dfw.h @@ -21,8 +21,7 @@ public: ~CopyDFW() override; bool IsGenerated() const override { return false; } - void insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState *state, ResType type, - vespalib::slime::Inserter &target) const override; + void insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState& state, vespalib::slime::Inserter &target) const override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.cpp index c698f0603c6..452ca98ea0b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.cpp @@ -13,7 +13,7 @@ DocsumFieldWriter::getAttributeName() const } bool -DocsumFieldWriter::isDefaultValue(uint32_t, const GetDocsumsState*) const +DocsumFieldWriter::isDefaultValue(uint32_t, const GetDocsumsState&) const { return false; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.h index a1af91a2b3f..77dc5d5d2d6 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer.h @@ -2,7 +2,6 @@ #pragma once -#include "res_type_utils.h" #include <vespa/vespalib/stllike/string.h> namespace vespalib::slime { struct Inserter; } @@ -24,9 +23,9 @@ public: } virtual ~DocsumFieldWriter() = default; virtual bool IsGenerated() const = 0; - virtual void insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) const = 0; + virtual void insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState& state, vespalib::slime::Inserter &target) const = 0; virtual const vespalib::string & getAttributeName() const; - virtual bool isDefaultValue(uint32_t docid, const GetDocsumsState * state) const; + virtual bool isDefaultValue(uint32_t docid, const GetDocsumsState& state) const; void setIndex(size_t v) { _index = v; } size_t getIndex() const { return _index; } virtual bool setFieldWriterStateIndex(uint32_t fieldWriterStateIndex); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.cpp new file mode 100644 index 00000000000..b04963a5907 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.cpp @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "docsum_field_writer_commands.h" + +namespace search::docsummary::command { + +const vespalib::string abs_distance("absdist"); +const vespalib::string attribute("attribute"); +const vespalib::string attribute_combiner("attributecombiner"); +const vespalib::string copy("copy"); +const vespalib::string documentid("documentid"); +const vespalib::string dynamic_teaser("dynamicteaser"); +const vespalib::string empty("empty"); +const vespalib::string geo_position("geopos"); +const vespalib::string matched_attribute_elements_filter("matchedattributeelementsfilter"); +const vespalib::string matched_elements_filter("matchedelementsfilter"); +const vespalib::string positions("positions"); +const vespalib::string rank_features("rankfeatures"); +const vespalib::string summary_features("summaryfeatures"); + +} + diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.h new file mode 100644 index 00000000000..8ca508a6b60 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.h @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace search::docsummary::command { + +/** + * This contains all commands that map to specific docsum field writer(s) when setting up a summary result class. + */ + +extern const vespalib::string abs_distance; +extern const vespalib::string attribute; +extern const vespalib::string attribute_combiner; +extern const vespalib::string copy; +extern const vespalib::string documentid; +extern const vespalib::string dynamic_teaser; +extern const vespalib::string empty; +extern const vespalib::string geo_position; +extern const vespalib::string matched_attribute_elements_filter; +extern const vespalib::string matched_elements_filter; +extern const vespalib::string positions; +extern const vespalib::string rank_features; +extern const vespalib::string summary_features; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp index b3fa6c68b87..dc215d9c2ba 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp @@ -1,8 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "docsum_field_writer_factory.h" #include "attribute_combiner_dfw.h" #include "copy_dfw.h" +#include "docsum_field_writer_commands.h" +#include "docsum_field_writer_factory.h" #include "document_id_dfw.h" #include "empty_dfw.h" #include "geoposdfw.h" @@ -34,65 +35,83 @@ DocsumFieldWriterFactory::has_attribute_manager() const noexcept return getEnvironment().getAttributeManager() != nullptr; } +namespace { + +void +throw_if_nullptr(const std::unique_ptr<DocsumFieldWriter>& writer, + const vespalib::string& command) +{ + if (writer.get() == nullptr) { + throw IllegalArgumentException("Failed to create docsum field writer for command '" + command + "'."); + } +} + +void +throw_missing_source(const vespalib::string& command) +{ + throw IllegalArgumentException("Missing source for command '" + command + "'."); +} + +} + std::unique_ptr<DocsumFieldWriter> -DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& fieldName, const vespalib::string& overrideName, const vespalib::string& argument, bool& rc) +DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& field_name, + const vespalib::string& command, + const vespalib::string& source) { - rc = false; std::unique_ptr<DocsumFieldWriter> fieldWriter; - if (overrideName == "dynamicteaser") { - if ( ! argument.empty() ) { + if (command == command::dynamic_teaser) { + if ( ! source.empty() ) { auto fw = std::make_unique<DynamicTeaserDFW>(getEnvironment().getJuniper()); auto fw_ptr = fw.get(); fieldWriter = std::move(fw); - rc = fw_ptr->Init(fieldName.c_str(), argument); + if (!fw_ptr->Init(field_name.c_str(), source)) { + throw IllegalArgumentException("Failed to initialize DynamicTeaserDFW."); + } } else { - throw IllegalArgumentException("Missing argument"); + throw_missing_source(command); } - } else if (overrideName == "summaryfeatures") { + } else if (command == command::summary_features) { fieldWriter = std::make_unique<SummaryFeaturesDFW>(); - rc = true; - } else if (overrideName == "rankfeatures") { + } else if (command == command::rank_features) { fieldWriter = std::make_unique<RankFeaturesDFW>(); - rc = true; - } else if (overrideName == "empty") { + } else if (command == command::empty) { fieldWriter = std::make_unique<EmptyDFW>(); - rc = true; - } else if (overrideName == "copy") { - if ( ! argument.empty() ) { - fieldWriter = std::make_unique<CopyDFW>(argument); - rc = true; + } else if (command == command::copy) { + if ( ! source.empty() ) { + fieldWriter = std::make_unique<CopyDFW>(source); } else { - throw IllegalArgumentException("Missing argument"); + throw_missing_source(command); } - } else if (overrideName == "absdist") { + } else if (command == command::abs_distance) { if (has_attribute_manager()) { - fieldWriter = AbsDistanceDFW::create(argument.c_str(), getEnvironment().getAttributeManager()); - rc = static_cast<bool>(fieldWriter); + fieldWriter = AbsDistanceDFW::create(source.c_str(), getEnvironment().getAttributeManager()); + throw_if_nullptr(fieldWriter, command); } - } else if (overrideName == "positions") { + } else if (command == command::positions) { if (has_attribute_manager()) { - fieldWriter = PositionsDFW::create(argument.c_str(), getEnvironment().getAttributeManager(), _use_v8_geo_positions); - rc = static_cast<bool>(fieldWriter); + fieldWriter = PositionsDFW::create(source.c_str(), getEnvironment().getAttributeManager(), _use_v8_geo_positions); + throw_if_nullptr(fieldWriter, command); } - } else if (overrideName == "geopos") { + } else if (command == command::geo_position) { if (has_attribute_manager()) { - fieldWriter = GeoPositionDFW::create(argument.c_str(), getEnvironment().getAttributeManager(), _use_v8_geo_positions); - rc = static_cast<bool>(fieldWriter); + fieldWriter = GeoPositionDFW::create(source.c_str(), getEnvironment().getAttributeManager(), _use_v8_geo_positions); + throw_if_nullptr(fieldWriter, command); } - } else if (overrideName == "attribute") { + } else if (command == command::attribute) { if (has_attribute_manager()) { - fieldWriter = AttributeDFWFactory::create(*getEnvironment().getAttributeManager(), argument); - rc = true; // Allow missing attribute vector + fieldWriter = AttributeDFWFactory::create(*getEnvironment().getAttributeManager(), source); + // Missing attribute vector is allowed, so throw_if_nullptr() is NOT used. } - } else if (overrideName == "attributecombiner") { + } else if (command == command::attribute_combiner) { if (has_attribute_manager()) { auto attr_ctx = getEnvironment().getAttributeManager()->createContext(); - const vespalib::string& source_field = argument.empty() ? fieldName : argument; + const vespalib::string& source_field = source.empty() ? field_name : source; fieldWriter = AttributeCombinerDFW::create(source_field, *attr_ctx, false, std::shared_ptr<MatchingElementsFields>()); - rc = static_cast<bool>(fieldWriter); + throw_if_nullptr(fieldWriter, command); } - } else if (overrideName == "matchedattributeelementsfilter") { - const vespalib::string& source_field = argument.empty() ? fieldName : argument; + } else if (command == command::matched_attribute_elements_filter) { + const vespalib::string& source_field = source.empty() ? field_name : source; if (has_attribute_manager()) { auto attr_ctx = getEnvironment().getAttributeManager()->createContext(); if (attr_ctx->getAttribute(source_field) != nullptr) { @@ -100,20 +119,19 @@ DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& fie } else { fieldWriter = AttributeCombinerDFW::create(source_field, *attr_ctx, true, _matching_elems_fields); } - rc = static_cast<bool>(fieldWriter); + throw_if_nullptr(fieldWriter, command); } - } else if (overrideName == "matchedelementsfilter") { - const vespalib::string& source_field = argument.empty() ? fieldName : argument; + } else if (command == command::matched_elements_filter) { + const vespalib::string& source_field = source.empty() ? field_name : source; if (has_attribute_manager()) { auto attr_ctx = getEnvironment().getAttributeManager()->createContext(); fieldWriter = MatchedElementsFilterDFW::create(source_field,*attr_ctx, _matching_elems_fields); - rc = static_cast<bool>(fieldWriter); + throw_if_nullptr(fieldWriter, command); } - } else if (overrideName == "documentid") { + } else if (command == command::documentid) { fieldWriter = std::make_unique<DocumentIdDFW>(); - rc = true; } else { - throw IllegalArgumentException("unknown override operation '" + overrideName + "' for field '" + fieldName + "'."); + throw IllegalArgumentException("Unknown command '" + command + "'."); } return fieldWriter; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.h index bab7153009d..e341f49c25b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.h @@ -24,7 +24,9 @@ protected: public: DocsumFieldWriterFactory(bool use_v8_geo_positions, const IDocsumEnvironment& env); ~DocsumFieldWriterFactory() override; - std::unique_ptr<DocsumFieldWriter> create_docsum_field_writer(const vespalib::string& fieldName, const vespalib::string& overrideName, const vespalib::string& argument, bool& rc) override; + std::unique_ptr<DocsumFieldWriter> create_docsum_field_writer(const vespalib::string& field_name, + const vespalib::string& command, + const vespalib::string& source) override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp index dca6e6f8bd3..35db818ac58 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "docsum_store_document.h" -#include "check_undefined_value_visitor.h" +#include "annotation_converter.h" #include "summaryfieldconverter.h" #include <vespa/document/base/exceptions.h> #include <vespa/document/datatype/datatype.h> @@ -50,7 +50,8 @@ DocsumStoreDocument::insert_juniper_field(const vespalib::string& field_name, ve { auto field_value = get_field_value(field_name); if (field_value) { - SummaryFieldConverter::insert_juniper_field(*field_value, inserter, true, converter); + AnnotationConverter stacked_converter(converter); + SummaryFieldConverter::insert_juniper_field(*field_value, inserter, stacked_converter); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstore.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstore.h index b112b7ab0bf..7f3a88b05eb 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstore.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstore.h @@ -15,10 +15,7 @@ class IDocsumStoreDocument; class IDocsumStore { public: - /** - * Convenience typedef. - */ - typedef std::unique_ptr<IDocsumStore> UP; + using UP = std::unique_ptr<IDocsumStore>; /** * Destructor. No cleanup needed for base class. diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp index 422400dc2ef..b4b663718bd 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.cpp @@ -14,8 +14,8 @@ LOG_SETUP(".searchlib.docsummary.docsumwriter"); using vespalib::Issue; -using vespalib::slime::ObjectInserter; using vespalib::Memory; +using vespalib::slime::ObjectInserter; namespace search::docsummary { @@ -48,7 +48,7 @@ DynamicDocsumWriter::resolveOutputClass(vespalib::stringref summaryClass) const } void -DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, uint32_t docid, GetDocsumsState *state, +DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, uint32_t docid, GetDocsumsState& state, IDocsumStore *docinfos, Inserter& topInserter) { if (rci.outputClass == nullptr) { @@ -61,10 +61,10 @@ DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, uint32_t docid, for (uint32_t i = 0; i < rci.outputClass->GetNumEntries(); ++i) { const ResConfigEntry *resCfg = rci.outputClass->GetEntry(i); const DocsumFieldWriter *writer = resCfg->_docsum_field_writer.get(); - if (state->_args.needField(resCfg->_name) && ! writer->isDefaultValue(docid, state)) { + if (state._args.needField(resCfg->_name) && ! writer->isDefaultValue(docid, state)) { const Memory field_name(resCfg->_name.data(), resCfg->_name.size()); ObjectInserter inserter(docsum, field_name); - writer->insertField(docid, nullptr, state, resCfg->_type, inserter); + writer->insertField(docid, nullptr, state, inserter); } } } else { @@ -77,13 +77,15 @@ DynamicDocsumWriter::insertDocsum(const ResolveClassInfo & rci, uint32_t docid, vespalib::slime::Cursor & docsum = topInserter.insertObject(); for (uint32_t i = 0; i < rci.outputClass->GetNumEntries(); ++i) { const ResConfigEntry *outCfg = rci.outputClass->GetEntry(i); - if ( ! state->_args.needField(outCfg->_name)) continue; + if (!state._args.needField(outCfg->_name)) { + continue; + } const DocsumFieldWriter *writer = outCfg->_docsum_field_writer.get(); const Memory field_name(outCfg->_name.data(), outCfg->_name.size()); ObjectInserter inserter(docsum, field_name); if (writer != nullptr) { if (! writer->isDefaultValue(docid, state)) { - writer->insertField(docid, doc.get(), state, outCfg->_type, inserter); + writer->insertField(docid, doc.get(), state, inserter); } } else { if (doc) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h index c0579638593..52ddeebb0d6 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumwriter.h @@ -37,7 +37,7 @@ public: virtual ~IDocsumWriter() = default; virtual void InitState(const search::IAttributeManager & attrMan, GetDocsumsState& state, const ResolveClassInfo& rci) = 0; - virtual void insertDocsum(const ResolveClassInfo & rci, uint32_t docid, GetDocsumsState *state, + virtual void insertDocsum(const ResolveClassInfo & rci, uint32_t docid, GetDocsumsState& state, IDocsumStore *docinfos, Inserter & target) = 0; virtual ResolveClassInfo resolveClassInfo(vespalib::stringref outputClassName) const = 0; }; @@ -61,7 +61,7 @@ public: const ResultConfig *GetResultConfig() { return _resultConfig.get(); } void InitState(const search::IAttributeManager & attrMan, GetDocsumsState& state, const ResolveClassInfo& rci) override; - void insertDocsum(const ResolveClassInfo & outputClassInfo, uint32_t docid, GetDocsumsState *state, + void insertDocsum(const ResolveClassInfo & outputClassInfo, uint32_t docid, GetDocsumsState& state, IDocsumStore *docinfos, Inserter & inserter) override; ResolveClassInfo resolveClassInfo(vespalib::stringref outputClassName) const override; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.cpp index 3d28dbdc6fb..911cd1acc0d 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.cpp @@ -10,7 +10,7 @@ DocumentIdDFW::DocumentIdDFW() = default; DocumentIdDFW::~DocumentIdDFW() = default; void -DocumentIdDFW::insertField(uint32_t, const IDocsumStoreDocument* doc, GetDocsumsState *, ResType, +DocumentIdDFW::insertField(uint32_t, const IDocsumStoreDocument* doc, GetDocsumsState&, vespalib::slime::Inserter &target) const { if (doc != nullptr) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.h index 27766a86e83..b6a89ff7828 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/document_id_dfw.h @@ -16,7 +16,7 @@ public: DocumentIdDFW(); ~DocumentIdDFW() override; bool IsGenerated() const override { return false; } - void insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) const override; + void insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState& state, vespalib::slime::Inserter &target) const override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp index fcdb81defbf..0ea5bc9a604 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp @@ -114,8 +114,7 @@ class JuniperConverter : public IJuniperConverter public: JuniperConverter(const DynamicTeaserDFW& writer, uint32_t doc_id, GetDocsumsState& state); ~JuniperConverter() override; - void insert_juniper_field(vespalib::stringref input, vespalib::slime::Inserter& inserter) override; - void insert_juniper_field(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) override; + void convert(vespalib::stringref input, vespalib::slime::Inserter& inserter) override; }; JuniperConverter::JuniperConverter(const DynamicTeaserDFW& writer, uint32_t doc_id, GetDocsumsState& state) @@ -129,25 +128,19 @@ JuniperConverter::JuniperConverter(const DynamicTeaserDFW& writer, uint32_t doc_ JuniperConverter::~JuniperConverter() = default; void -JuniperConverter::insert_juniper_field(vespalib::stringref input, vespalib::slime::Inserter& inserter) +JuniperConverter::convert(vespalib::stringref input, vespalib::slime::Inserter& inserter) { _writer.insert_juniper_field(_doc_id, input, _state, inserter); } -void -JuniperConverter::insert_juniper_field(const document::StringFieldValue& input, vespalib::slime::Inserter& inserter) -{ - _writer.insert_juniper_field(_doc_id, input.getValueRef(), _state, inserter); -} - } void -DynamicTeaserDFW::insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState *state, ResType, +DynamicTeaserDFW::insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState& state, vespalib::slime::Inserter &target) const { if (doc != nullptr) { - JuniperConverter converter(*this, docid, *state); + JuniperConverter converter(*this, docid, state); doc->insert_juniper_field(_input_field_name, target, converter); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/empty_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/empty_dfw.cpp index 37d2785ffa7..d7d59c06791 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/empty_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/empty_dfw.cpp @@ -9,7 +9,7 @@ EmptyDFW::EmptyDFW() = default; EmptyDFW::~EmptyDFW() = default; void -EmptyDFW::insertField(uint32_t, GetDocsumsState *, ResType, vespalib::slime::Inserter &target) const +EmptyDFW::insertField(uint32_t, GetDocsumsState&, vespalib::slime::Inserter &target) const { // insert explicitly-empty field? // target.insertNix(); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/empty_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/empty_dfw.h index 9a250450b1f..3e05e029722 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/empty_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/empty_dfw.h @@ -16,7 +16,7 @@ public: ~EmptyDFW() override; bool IsGenerated() const override { return true; } - void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) const override; + void insertField(uint32_t docid, GetDocsumsState& state, vespalib::slime::Inserter &target) const override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp index 474f329799b..6d668561651 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp @@ -55,14 +55,14 @@ void fmtZcurve(int64_t zval, vespalib::slime::Inserter &target, bool useV8geoPos } void -GeoPositionDFW::insertField(uint32_t docid, GetDocsumsState * dsState, ResType, vespalib::slime::Inserter &target) const +GeoPositionDFW::insertField(uint32_t docid, GetDocsumsState& dsState, vespalib::slime::Inserter &target) const { using vespalib::slime::Cursor; using vespalib::slime::ObjectSymbolInserter; using vespalib::slime::Symbol; using vespalib::slime::ArrayInserter; - const auto& attribute = get_attribute(*dsState); + const auto& attribute = get_attribute(dsState); if (attribute.hasMultiValue()) { uint32_t entries = attribute.getValueCount(docid); if (entries == 0 && _useV8geoPositions) return; @@ -104,21 +104,20 @@ GeoPositionDFW::create(const char *attribute_name, const IAttributeManager *attribute_manager, bool useV8geoPositions) { - GeoPositionDFW::UP ret; if (attribute_manager != nullptr) { if (!attribute_name) { LOG(warning, "create: missing attribute name '%p'", attribute_name); - return ret; + return {}; } IAttributeContext::UP context = attribute_manager->createContext(); if (!context.get()) { LOG(warning, "create: could not create context from attribute manager"); - return ret; + return {}; } const IAttributeVector *attribute = context->getAttribute(attribute_name); if (!attribute) { Issue::report("GeoPositionDFW::create: could not get attribute '%s' from context", attribute_name); - return ret; + return {}; } } return std::make_unique<GeoPositionDFW>(attribute_name, useV8geoPositions); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.h index 1bc8b523160..6e470d479ff 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.h @@ -14,9 +14,9 @@ class GeoPositionDFW : public AttrDFW private: bool _useV8geoPositions; public: - typedef std::unique_ptr<GeoPositionDFW> UP; + using UP = std::unique_ptr<GeoPositionDFW>; GeoPositionDFW(const vespalib::string & attrName, bool useV8geoPositions); - void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) const override; + void insertField(uint32_t docid, GetDocsumsState& state, vespalib::slime::Inserter &target) const override; static UP create(const char *attribute_name, const IAttributeManager *attribute_manager, bool useV8geoPositions); }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_field_writer_factory.h b/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_field_writer_factory.h index 927fef26d1a..6a5cd691857 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_field_writer_factory.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_field_writer_factory.h @@ -16,7 +16,12 @@ class IDocsumFieldWriterFactory { public: virtual ~IDocsumFieldWriterFactory() = default; - virtual std::unique_ptr<DocsumFieldWriter> create_docsum_field_writer(const vespalib::string& fieldName, const vespalib::string& overrideName, const vespalib::string& argument, bool& rc) = 0; + /** + * Implementations can throw vespalib::IllegalArgumentException if setup of field writer fails. + */ + virtual std::unique_ptr<DocsumFieldWriter> create_docsum_field_writer(const vespalib::string& field_name, + const vespalib::string& command, + const vespalib::string& source) = 0; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/i_juniper_converter.h b/searchsummary/src/vespa/searchsummary/docsummary/i_juniper_converter.h index 00751082567..a52002d37f5 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/i_juniper_converter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/i_juniper_converter.h @@ -12,17 +12,12 @@ namespace search::docsummary { /** * Interface class for inserting a dynamic string based on an * annotated full string and query context. - * - * For streaming search we use the same interface in an adapter that - * calls a snippet modifier (vsm::SnippetModifier) to add the annotation - * needed by juniper. */ class IJuniperConverter { public: virtual ~IJuniperConverter() = default; - virtual void insert_juniper_field(vespalib::stringref input, vespalib::slime::Inserter& inserter) = 0; - virtual void insert_juniper_field(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) = 0; + virtual void convert(vespalib::stringref input, vespalib::slime::Inserter& inserter) = 0; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/i_string_field_converter.h b/searchsummary/src/vespa/searchsummary/docsummary/i_string_field_converter.h new file mode 100644 index 00000000000..0e80fc28ded --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/i_string_field_converter.h @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace document { class StringFieldValue; } +namespace vespalib::slime { struct Inserter; } + +namespace search::docsummary { + +/** + * Interface class for inserting a dynamic string. + */ +class IStringFieldConverter +{ +public: + virtual ~IStringFieldConverter() = default; + virtual void convert(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) = 0; +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/juniper_query_adapter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/juniper_query_adapter.cpp index 814fe0aafe4..a13f65db5ce 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/juniper_query_adapter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/juniper_query_adapter.cpp @@ -27,8 +27,9 @@ JuniperQueryAdapter::SkipItem(search::SimpleQueryStackDumpIterator *iterator) co uint32_t skipCount = iterator->getArity(); while (skipCount > 0) { - if (!iterator->next()) + if (!iterator->next()) { return false; // stack too small + } skipCount = skipCount - 1 + iterator->getArity(); } return true; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h index 7dcf3d16e26..24c99873f58 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/juniperdfw.h @@ -48,8 +48,8 @@ class DynamicTeaserDFW : public JuniperTeaserDFW public: explicit DynamicTeaserDFW(const juniper::Juniper * juniper) : JuniperTeaserDFW(juniper) { } - void insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState *state, - ResType type, vespalib::slime::Inserter &target) const override; + void insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState& state, + vespalib::slime::Inserter &target) const override; void insert_juniper_field(uint32_t docid, vespalib::stringref input, GetDocsumsState& state, vespalib::slime::Inserter& inserter) const; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp index 0256965e7f4..e8ff3068a4c 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp @@ -100,8 +100,9 @@ KeywordExtractor::GetLegalIndexSpec() } for (const auto & index : _legalIndexes) { - if (!spec.empty()) + if (!spec.empty()) { spec.append(';'); + } spec.append(index); } return spec; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h index 5f87de762f9..9d46f0c8d89 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h @@ -23,7 +23,7 @@ public: }; private: - typedef vespalib::hash_set<vespalib::string> Set; + using Set = vespalib::hash_set<vespalib::string>; const IDocsumEnvironment *_env; std::vector<IndexPrefix> _legalPrefixes; Set _legalIndexes; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp index fe06212bcd2..1a029cfd16f 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.cpp @@ -54,7 +54,7 @@ MatchedElementsFilterDFW::create(const std::string& input_field_name, { StructFieldsResolver resolver(input_field_name, attr_ctx, false); if (resolver.has_error()) { - return std::unique_ptr<DocsumFieldWriter>(); + return {}; } resolver.apply_to(*matching_elems_fields); return std::make_unique<MatchedElementsFilterDFW>(input_field_name, std::move(matching_elems_fields)); @@ -63,12 +63,12 @@ MatchedElementsFilterDFW::create(const std::string& input_field_name, MatchedElementsFilterDFW::~MatchedElementsFilterDFW() = default; void -MatchedElementsFilterDFW::insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState *state, - ResType, vespalib::slime::Inserter& target) const +MatchedElementsFilterDFW::insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState& state, + vespalib::slime::Inserter& target) const { auto field_value = doc->get_field_value(_input_field_name); if (field_value) { - SummaryFieldConverter::insert_summary_field_with_filter(*field_value, target, get_matching_elements(docid, *state)); + SummaryFieldConverter::insert_summary_field_with_filter(*field_value, target, get_matching_elements(docid, state)); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h index 18d608440d3..7dafdbc9e6b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/matched_elements_filter_dfw.h @@ -34,8 +34,8 @@ public: std::shared_ptr<MatchingElementsFields> matching_elems_fields); ~MatchedElementsFilterDFW() override; bool IsGenerated() const override { return false; } - void insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState *state, - ResType, vespalib::slime::Inserter& target) const override; + void insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState& state, + vespalib::slime::Inserter& target) const override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp index 2dd19d8d9ea..5aba321b540 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.cpp @@ -26,24 +26,24 @@ double to_degrees(int32_t microDegrees) { } +using search::attribute::BasicType; using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; -using search::attribute::BasicType; using search::attribute::IntegerContent; -using search::common::Location; using search::common::GeoGcd; +using search::common::Location; LocationAttrDFW::AllLocations -LocationAttrDFW::getAllLocations(GetDocsumsState *state) const +LocationAttrDFW::getAllLocations(GetDocsumsState& state) const { AllLocations retval; - if (! state->_args.locations_possible()) { + if (! state._args.locations_possible()) { return retval; } - if (state->_parsedLocations.empty()) { - state->parse_locations(); + if (state._parsedLocations.empty()) { + state.parse_locations(); } - for (const auto & loc : state->_parsedLocations) { + for (const auto & loc : state._parsedLocations) { if (loc.location.valid()) { LOG(debug, "found location(field %s) for DFW(field %s)\n", loc.field_name.c_str(), getAttributeName().c_str()); @@ -56,7 +56,7 @@ LocationAttrDFW::getAllLocations(GetDocsumsState *state) const } if (retval.empty()) { // avoid doing things twice - state->_args.locations_possible(false); + state._args.locations_possible(false); } return retval; } @@ -69,13 +69,13 @@ AbsDistanceDFW::AbsDistanceDFW(const vespalib::string & attrName) { } uint64_t -AbsDistanceDFW::findMinDistance(uint32_t docid, GetDocsumsState *state, +AbsDistanceDFW::findMinDistance(uint32_t docid, GetDocsumsState& state, const std::vector<const GeoLoc *> &locations) const { // ensure result fits in Java "int" uint64_t absdist = std::numeric_limits<int32_t>::max(); uint64_t sqdist = absdist*absdist; - const auto& attribute = get_attribute(*state); + const auto& attribute = get_attribute(state); for (auto location : locations) { int32_t docx = 0; int32_t docy = 0; @@ -95,7 +95,7 @@ AbsDistanceDFW::findMinDistance(uint32_t docid, GetDocsumsState *state, } void -AbsDistanceDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType, vespalib::slime::Inserter &target) const +AbsDistanceDFW::insertField(uint32_t docid, GetDocsumsState& state, vespalib::slime::Inserter &target) const { const auto & all_locations = getAllLocations(state); if (all_locations.empty()) { @@ -220,33 +220,32 @@ void insertV8FromAttr(const attribute::IAttributeVector &attribute, uint32_t doc } // namespace void -PositionsDFW::insertField(uint32_t docid, GetDocsumsState * dsState, ResType, vespalib::slime::Inserter &target) const +PositionsDFW::insertField(uint32_t docid, GetDocsumsState& dsState, vespalib::slime::Inserter &target) const { if (_useV8geoPositions) { - insertV8FromAttr(get_attribute(*dsState), docid, target); + insertV8FromAttr(get_attribute(dsState), docid, target); } else { - insertFromAttr(get_attribute(*dsState), docid, target); + insertFromAttr(get_attribute(dsState), docid, target); } } //-------------------------------------------------------------------------- PositionsDFW::UP PositionsDFW::create(const char *attribute_name, const IAttributeManager *attribute_manager, bool useV8geoPositions) { - PositionsDFW::UP ret; if (attribute_manager != nullptr) { if (!attribute_name) { LOG(debug, "createPositionsDFW: missing attribute name '%p'", attribute_name); - return ret; + return {}; } IAttributeContext::UP context = attribute_manager->createContext(); if (!context.get()) { LOG(debug, "createPositionsDFW: could not create context from attribute manager"); - return ret; + return {}; } const IAttributeVector *attribute = context->getAttribute(attribute_name); if (!attribute) { LOG(debug, "createPositionsDFW: could not get attribute '%s' from context", attribute_name); - return ret; + return {}; } } return std::make_unique<PositionsDFW>(attribute_name, useV8geoPositions); @@ -254,21 +253,20 @@ PositionsDFW::UP PositionsDFW::create(const char *attribute_name, const IAttribu std::unique_ptr<DocsumFieldWriter> AbsDistanceDFW::create(const char *attribute_name, const IAttributeManager *attribute_manager) { - std::unique_ptr<DocsumFieldWriter> ret; if (attribute_manager != nullptr) { if (!attribute_name) { LOG(debug, "createAbsDistanceDFW: missing attribute name '%p'", attribute_name); - return ret; + return {}; } IAttributeContext::UP context = attribute_manager->createContext(); if (!context.get()) { LOG(debug, "createAbsDistanceDFW: could not create context from attribute manager"); - return ret; + return {}; } const IAttributeVector *attribute = context->getAttribute(attribute_name); if (!attribute) { LOG(debug, "createAbsDistanceDFW: could not get attribute '%s' from context", attribute_name); - return ret; + return {}; } } return std::make_unique<AbsDistanceDFW>(attribute_name); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h index 67fe0bba5fe..5ac5f0fe051 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/positionsdfw.h @@ -31,20 +31,20 @@ public: return matching.empty() ? other : matching; } }; - AllLocations getAllLocations(GetDocsumsState *state) const; + AllLocations getAllLocations(GetDocsumsState& state) const; }; class AbsDistanceDFW : public LocationAttrDFW { private: - uint64_t findMinDistance(uint32_t docid, GetDocsumsState *state, + uint64_t findMinDistance(uint32_t docid, GetDocsumsState& state, const std::vector<const GeoLoc *> &locations) const; public: explicit AbsDistanceDFW(const vespalib::string & attrName); bool IsGenerated() const override { return true; } - void insertField(uint32_t docid, GetDocsumsState *state, - ResType, vespalib::slime::Inserter &target) const override; + void insertField(uint32_t docid, GetDocsumsState& state, + vespalib::slime::Inserter &target) const override; static std::unique_ptr<DocsumFieldWriter> create(const char *attribute_name, const IAttributeManager *index_man); @@ -57,10 +57,10 @@ class PositionsDFW : public AttrDFW private: bool _useV8geoPositions; public: - typedef std::unique_ptr<PositionsDFW> UP; + using UP = std::unique_ptr<PositionsDFW>; PositionsDFW(const vespalib::string & attrName, bool useV8geoPositions); bool IsGenerated() const override { return true; } - void insertField(uint32_t docid, GetDocsumsState *state, ResType, vespalib::slime::Inserter &target) const override; + void insertField(uint32_t docid, GetDocsumsState& state, vespalib::slime::Inserter &target) const override; static UP create(const char *attribute_name, const IAttributeManager *index_man, bool useV8geoPositions); }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp index b7b10d9c1ea..08fba307e8f 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp @@ -12,17 +12,17 @@ RankFeaturesDFW::RankFeaturesDFW() = default; RankFeaturesDFW::~RankFeaturesDFW() = default; void -RankFeaturesDFW::insertField(uint32_t docid, GetDocsumsState *state, - ResType, vespalib::slime::Inserter &target) const +RankFeaturesDFW::insertField(uint32_t docid, GetDocsumsState& state, + vespalib::slime::Inserter &target) const { - if ( !state->_rankFeatures ) { - state->_callback.FillRankFeatures(*state); - if (state->_rankFeatures.get() == nullptr) { // still no rank features to write + if ( !state._rankFeatures ) { + state._callback.FillRankFeatures(state); + if (state._rankFeatures.get() == nullptr) { // still no rank features to write return; } } - const FeatureSet::StringVector & names = state->_rankFeatures->getNames(); - const FeatureSet::Value * values = state->_rankFeatures->getFeaturesByDocId(docid); + const FeatureSet::StringVector & names = state._rankFeatures->getNames(); + const FeatureSet::Value * values = state._rankFeatures->getFeaturesByDocId(docid); if (values == nullptr) { return; } vespalib::slime::Cursor& obj = target.insertObject(); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h index 7302d162b65..dbd5f3ce0b6 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.h @@ -14,7 +14,7 @@ public: RankFeaturesDFW & operator=(const RankFeaturesDFW &) = delete; ~RankFeaturesDFW() override; bool IsGenerated() const override { return true; } - void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) const override; + void insertField(uint32_t docid, GetDocsumsState& state, vespalib::slime::Inserter &target) const override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp index 781cd62a818..d19a111080f 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultclass.cpp @@ -23,15 +23,16 @@ ResultClass::~ResultClass() = default; int ResultClass::GetIndexFromName(const char* name) const { - NameIdMap::const_iterator found(_nameMap.find(name)); + auto found = _nameMap.find(name); return (found != _nameMap.end()) ? found->second : -1; } bool ResultClass::AddConfigEntry(const char *name, ResType type, std::unique_ptr<DocsumFieldWriter> docsum_field_writer) { - if (_nameMap.find(name) != _nameMap.end()) + if (_nameMap.find(name) != _nameMap.end()) { return false; + } _nameMap[name] = _entries.size(); ResConfigEntry e; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp index 77714ddd98f..4f5b5db841c 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp @@ -4,8 +4,9 @@ #include "docsum_field_writer.h" #include "docsum_field_writer_factory.h" #include "resultclass.h" -#include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/config-summary.h> +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <vespa/vespalib/util/exceptions.h> #include <atomic> #include <vespa/log/log.h> @@ -70,14 +71,14 @@ ResultConfig::set_default_result_class_id(uint32_t id) const ResultClass* ResultConfig::LookupResultClass(uint32_t id) const { - IdMap::const_iterator it(_classLookup.find(id)); + auto it = _classLookup.find(id); return (it != _classLookup.end()) ? it->second.get() : nullptr; } uint32_t ResultConfig::LookupResultClassId(const vespalib::string &name) const { - NameMap::const_iterator found(_nameLookup.find(name)); + auto found = _nameLookup.find(name); return (found != _nameLookup.end()) ? found->second : ((name.empty() || (name == "default")) ? _defaultSummaryId : NoClassID()); } @@ -126,16 +127,20 @@ ResultConfig::ReadConfig(const SummaryConfig &cfg, const char *configId, IDocsum for (unsigned int j = 0; rc && (j < cfg_class.fields.size()); j++) { const char *fieldtype = cfg_class.fields[j].type.c_str(); const char *fieldname = cfg_class.fields[j].name.c_str(); - vespalib::string override_name = cfg_class.fields[j].command; + vespalib::string command = cfg_class.fields[j].command; vespalib::string source_name = cfg_class.fields[j].source; auto res_type = ResTypeUtils::get_res_type(fieldtype); LOG(debug, "Reconfiguring class '%s' field '%s' of type '%s'", cfg_class.name.c_str(), fieldname, fieldtype); if (res_type != RES_BAD) { std::unique_ptr<DocsumFieldWriter> docsum_field_writer; - if (!override_name.empty()) { - docsum_field_writer = docsum_field_writer_factory.create_docsum_field_writer(fieldname, override_name, source_name, rc); - if (!rc) { - LOG(error, "%s override operation failed during initialization", override_name.c_str()); + if (!command.empty()) { + try { + docsum_field_writer = docsum_field_writer_factory.create_docsum_field_writer(fieldname, + command, + source_name); + } catch (const vespalib::IllegalArgumentException& ex) { + LOG(error, "Exception during setup of summary result class '%s': field='%s', command='%s', source='%s': %s", + cfg_class.name.c_str(), fieldname, command.c_str(), source_name.c_str(), ex.getMessage().c_str()); break; } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.cpp index 01e306161e7..b2a05d98f5b 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.cpp @@ -5,9 +5,9 @@ namespace search::docsummary { void -SimpleDFW::insertField(uint32_t docid, const IDocsumStoreDocument *, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) const +SimpleDFW::insertField(uint32_t docid, const IDocsumStoreDocument *, GetDocsumsState& state, vespalib::slime::Inserter &target) const { - insertField(docid, state, type, target); + insertField(docid, state, target); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.h index 52a45754c1f..4c7a4be517e 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/simple_dfw.h @@ -13,8 +13,8 @@ namespace search::docsummary { class SimpleDFW : public DocsumFieldWriter { public: - virtual void insertField(uint32_t docid, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) const = 0; - void insertField(uint32_t docid, const IDocsumStoreDocument*, GetDocsumsState *state, ResType type, vespalib::slime::Inserter &target) const override; + virtual void insertField(uint32_t docid, GetDocsumsState& state, vespalib::slime::Inserter &target) const = 0; + void insertField(uint32_t docid, const IDocsumStoreDocument*, GetDocsumsState& state, vespalib::slime::Inserter &target) const override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.cpp b/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.cpp index b3d3fde7150..94774c1bee4 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.cpp @@ -1,10 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "slime_filler.h" -#include "annotation_converter.h" #include "i_juniper_converter.h" +#include "i_string_field_converter.h" #include "resultconfig.h" #include "searchdatatype.h" +#include "slime_filler_filter.h" #include <vespa/document/datatype/positiondatatype.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/boolfieldvalue.h> @@ -65,51 +66,53 @@ private: Cursor& _array; Symbol _key_sym; Symbol _val_sym; - bool _tokenize; + std::optional<const SlimeFillerFilter*> _filter; public: - MapFieldValueInserter(Inserter& parent_inserter, bool tokenize) + MapFieldValueInserter(Inserter& parent_inserter, std::optional<const SlimeFillerFilter*> filter) : _array(parent_inserter.insertArray()), _key_sym(_array.resolve("key")), _val_sym(_array.resolve("value")), - _tokenize(tokenize) + _filter(std::move(filter)) { } void insert_entry(const FieldValue& key, const FieldValue& value) { Cursor& c = _array.addObject(); ObjectSymbolInserter ki(c, _key_sym); - ObjectSymbolInserter vi(c, _val_sym); - SlimeFiller key_conv(ki, _tokenize); - SlimeFiller val_conv(vi, _tokenize); + SlimeFiller key_conv(ki); key.accept(key_conv); - value.accept(val_conv); + if (_filter.has_value()) { + ObjectSymbolInserter vi(c, _val_sym); + SlimeFiller val_conv(vi, nullptr, _filter.value()); + value.accept(val_conv); + } } }; } -SlimeFiller::SlimeFiller(Inserter& inserter, bool tokenize) +SlimeFiller::SlimeFiller(Inserter& inserter) : _inserter(inserter), - _tokenize(tokenize), _matching_elems(nullptr), - _juniper_converter(nullptr) + _string_converter(nullptr), + _filter(nullptr) { } -SlimeFiller::SlimeFiller(Inserter& inserter, bool tokenize, const std::vector<uint32_t>* matching_elems) +SlimeFiller::SlimeFiller(Inserter& inserter, const std::vector<uint32_t>* matching_elems) : _inserter(inserter), - _tokenize(tokenize), _matching_elems(matching_elems), - _juniper_converter(nullptr) + _string_converter(nullptr), + _filter(nullptr) { } -SlimeFiller::SlimeFiller(Inserter& inserter, bool tokenize, IJuniperConverter* juniper_converter) +SlimeFiller::SlimeFiller(Inserter& inserter, IStringFieldConverter* string_converter, const SlimeFillerFilter* filter) : _inserter(inserter), - _tokenize(tokenize), _matching_elems(nullptr), - _juniper_converter(juniper_converter) + _string_converter(string_converter), + _filter(filter) { } @@ -141,7 +144,7 @@ SlimeFiller::visit(const MapFieldValue& v) if (empty_or_empty_after_filtering(v)) { return; } - MapFieldValueInserter map_inserter(_inserter, _tokenize); + MapFieldValueInserter map_inserter(_inserter, SlimeFillerFilter::get_filter(_filter, "value")); if (filter_matching_elements()) { assert(v.has_no_erased_keys()); for (uint32_t id_to_keep : (*_matching_elems)) { @@ -163,7 +166,7 @@ SlimeFiller::visit(const ArrayFieldValue& value) } Cursor& a = _inserter.insertArray(); ArrayInserter ai(a); - SlimeFiller conv(ai, _tokenize, _juniper_converter); + SlimeFiller conv(ai, _string_converter, _filter); if (filter_matching_elements()) { for (uint32_t id_to_keep : (*_matching_elems)) { value[id_to_keep].accept(conv); @@ -178,21 +181,10 @@ SlimeFiller::visit(const ArrayFieldValue& value) void SlimeFiller::visit(const StringFieldValue& value) { - if (_tokenize) { - asciistream tmp; - AnnotationConverter converter(value.getValue(), tmp); - converter.handleIndexingTerms(value); - if (_juniper_converter != nullptr) { - _juniper_converter->insert_juniper_field(tmp.str(), _inserter); - } else { - _inserter.insertString(Memory(tmp.str())); - } + if (_string_converter != nullptr) { + _string_converter->convert(value, _inserter); } else { - if (_juniper_converter != nullptr) { - _juniper_converter->insert_juniper_field(value, _inserter); - } else { - _inserter.insertString(Memory(value.getValueRef())); - } + _inserter.insertString(Memory(value.getValueRef())); } } @@ -282,11 +274,15 @@ SlimeFiller::visit(const StructFieldValue& value) } Cursor& c = _inserter.insertObject(); for (StructFieldValue::const_iterator itr = value.begin(); itr != value.end(); ++itr) { - Memory keymem(itr.field().getName()); - ObjectInserter vi(c, keymem); - SlimeFiller conv(vi, _tokenize); - FieldValue::UP nextValue(value.getValue(itr.field())); - (*nextValue).accept(conv); + auto& name = itr.field().getName(); + auto sub_filter = SlimeFillerFilter::get_filter(_filter, name); + if (sub_filter.has_value()) { + Memory keymem(name); + ObjectInserter vi(c, keymem); + SlimeFiller conv(vi, nullptr, sub_filter.value()); + FieldValue::UP nextValue(value.getValue(itr.field())); + (*nextValue).accept(conv); + } } } @@ -318,7 +314,7 @@ SlimeFiller::visit(const WeightedSetFieldValue& value) } Cursor& o = a.addObject(); ObjectSymbolInserter ki(o, isym); - SlimeFiller conv(ki, _tokenize); + SlimeFiller conv(ki); entry.first->accept(conv); int weight = static_cast<const IntFieldValue&>(*entry.second).getValue(); o.setLong(wsym, weight); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.h b/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.h index ebade8aa711..a81a20814c4 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.h @@ -10,7 +10,8 @@ namespace vespalib::slime { struct Inserter; } namespace search::docsummary { -class IJuniperConverter; +class IStringFieldConverter; +class SlimeFillerFilter; /* * Class inserting a field value into a slime object. @@ -18,9 +19,9 @@ class IJuniperConverter; class SlimeFiller : public document::ConstFieldValueVisitor { vespalib::slime::Inserter& _inserter; - bool _tokenize; const std::vector<uint32_t>* _matching_elems; - IJuniperConverter* _juniper_converter; + IStringFieldConverter* _string_converter; + const SlimeFillerFilter* _filter; bool filter_matching_elements() const { return _matching_elems != nullptr; @@ -50,9 +51,9 @@ class SlimeFiller : public document::ConstFieldValueVisitor { void visit(const document::TensorFieldValue& value) override; void visit(const document::ReferenceFieldValue& value) override; public: - SlimeFiller(vespalib::slime::Inserter& inserter, bool tokenize); - SlimeFiller(vespalib::slime::Inserter& inserter, bool tokenize, const std::vector<uint32_t>* matching_elems); - SlimeFiller(vespalib::slime::Inserter& inserter, bool tokenize, IJuniperConverter* juniper_converter); + SlimeFiller(vespalib::slime::Inserter& inserter); + SlimeFiller(vespalib::slime::Inserter& inserter, const std::vector<uint32_t>* matching_elems); + SlimeFiller(vespalib::slime::Inserter& inserter, IStringFieldConverter* string_converter, const SlimeFillerFilter* filter); ~SlimeFiller() override; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/slime_filler_filter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/slime_filler_filter.cpp new file mode 100644 index 00000000000..db28a1ae5cf --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/slime_filler_filter.cpp @@ -0,0 +1,67 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "slime_filler_filter.h" +#include <vespa/vespalib/stllike/hash_map.hpp> +#include <cassert> + +namespace search::docsummary { + +SlimeFillerFilter::SlimeFillerFilter() + : _filter() +{ +} + +SlimeFillerFilter::~SlimeFillerFilter() = default; + +std::optional<const SlimeFillerFilter*> +SlimeFillerFilter::get_filter(vespalib::stringref field_name) const +{ + auto itr = _filter.find(field_name); + if (itr == _filter.end()) { + return std::nullopt; + } + return itr->second.get(); +} + +std::optional<const SlimeFillerFilter*> +SlimeFillerFilter::get_filter(const SlimeFillerFilter* filter, vespalib::stringref field_name) +{ + return (filter != nullptr) ? filter->get_filter(field_name) : nullptr; +} + +bool +SlimeFillerFilter::empty() const { return _filter.empty(); } + +SlimeFillerFilter& +SlimeFillerFilter::add(vespalib::stringref field_path) +{ + vespalib::stringref field_name; + vespalib::stringref remaining_path; + auto dot_pos = field_path.find('.'); + if (dot_pos != vespalib::string::npos) { + field_name = field_path.substr(0, dot_pos); + remaining_path = field_path.substr(dot_pos + 1); + } else { + field_name = field_path; + } + auto itr = _filter.find(field_name); + if (itr != _filter.end()) { + if (itr->second) { + if (remaining_path.empty()) { + itr->second.reset(); + } else { + itr->second->add(remaining_path); + } + } + } else { + auto insres = _filter.insert(std::make_pair(field_name, std::unique_ptr<SlimeFillerFilter>())); + assert(insres.second); + if (!remaining_path.empty()) { + insres.first->second = std::make_unique<SlimeFillerFilter>(); + insres.first->second->add(remaining_path); + } + } + return *this; +} + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/slime_filler_filter.h b/searchsummary/src/vespa/searchsummary/docsummary/slime_filler_filter.h new file mode 100644 index 00000000000..ba7ba6fe159 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/slime_filler_filter.h @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/stllike/hash_map.h> +#include <optional> + +namespace search::docsummary { + +/* + * Class filtering which fields to render in a struct field. + */ +class SlimeFillerFilter { + vespalib::hash_map<vespalib::string, std::unique_ptr<SlimeFillerFilter>> _filter; + std::optional<const SlimeFillerFilter*> get_filter(vespalib::stringref field_name) const; +public: + SlimeFillerFilter(); + ~SlimeFillerFilter(); + /* + * If field is blocked by the filter then the return value is not set, + * otherwise it is set to the filter for the next level. + */ + static std::optional<const SlimeFillerFilter*> get_filter(const SlimeFillerFilter* filter, vespalib::stringref field_name); + bool empty() const; + SlimeFillerFilter& add(vespalib::stringref field_path); +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp index 76bae0cee97..13a3a345bf6 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp @@ -18,19 +18,19 @@ SummaryFeaturesDFW::~SummaryFeaturesDFW() = default; static vespalib::Memory _M_cached("vespa.summaryFeatures.cached"); void -SummaryFeaturesDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType, vespalib::slime::Inserter &target) const +SummaryFeaturesDFW::insertField(uint32_t docid, GetDocsumsState& state, vespalib::slime::Inserter &target) const { - if (state->_omit_summary_features) { + if (state._omit_summary_features) { return; } - if ( ! state->_summaryFeatures) { - state->_callback.FillSummaryFeatures(*state); - if ( !state->_summaryFeatures) { // still no summary features to write + if ( ! state._summaryFeatures) { + state._callback.FillSummaryFeatures(state); + if ( !state._summaryFeatures) { // still no summary features to write return; } } - const FeatureSet::StringVector &names = state->_summaryFeatures->getNames(); - const FeatureSet::Value *values = state->_summaryFeatures->getFeaturesByDocId(docid); + const FeatureSet::StringVector &names = state._summaryFeatures->getNames(); + const FeatureSet::Value *values = state._summaryFeatures->getFeaturesByDocId(docid); if (values == nullptr) { return; } vespalib::slime::Cursor& obj = target.insertObject(); @@ -42,7 +42,7 @@ SummaryFeaturesDFW::insertField(uint32_t docid, GetDocsumsState *state, ResType, obj.setDouble(name, values[i].as_double()); } } - if (state->_summaryFeaturesCached) { + if (state._summaryFeaturesCached) { obj.setDouble(_M_cached, 1.0); } else { obj.setDouble(_M_cached, 0.0); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h index ec14dc45055..661d23c2d64 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.h @@ -14,8 +14,8 @@ public: SummaryFeaturesDFW & operator=(const SummaryFeaturesDFW &) = delete; ~SummaryFeaturesDFW() override; bool IsGenerated() const override { return true; } - void insertField(uint32_t docid, GetDocsumsState *state, - ResType type, vespalib::slime::Inserter &target) const override; + void insertField(uint32_t docid, GetDocsumsState& state, + vespalib::slime::Inserter &target) const override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp index 1a21c1d3eab..dd5a59e46af 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp @@ -1,214 +1,21 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "summaryfieldconverter.h" -#include "annotation_converter.h" #include "check_undefined_value_visitor.h" -#include "searchdatatype.h" #include "slime_filler.h" -#include <vespa/document/fieldvalue/arrayfieldvalue.h> -#include <vespa/document/fieldvalue/boolfieldvalue.h> -#include <vespa/document/fieldvalue/bytefieldvalue.h> -#include <vespa/document/fieldvalue/document.h> -#include <vespa/document/fieldvalue/doublefieldvalue.h> -#include <vespa/document/fieldvalue/floatfieldvalue.h> -#include <vespa/document/fieldvalue/intfieldvalue.h> -#include <vespa/document/fieldvalue/longfieldvalue.h> -#include <vespa/document/fieldvalue/predicatefieldvalue.h> -#include <vespa/document/fieldvalue/rawfieldvalue.h> -#include <vespa/document/fieldvalue/shortfieldvalue.h> -#include <vespa/document/fieldvalue/stringfieldvalue.h> -#include <vespa/document/fieldvalue/weightedsetfieldvalue.h> -#include <vespa/document/fieldvalue/annotationreferencefieldvalue.h> -#include <vespa/document/fieldvalue/tensorfieldvalue.h> -#include <vespa/document/fieldvalue/referencefieldvalue.h> -#include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/util/size_literals.h> -#include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/data/smart_buffer.h> +#include <vespa/document/fieldvalue/fieldvalue.h> -using document::AnnotationReferenceFieldValue; -using document::ArrayFieldValue; -using document::BoolFieldValue; -using document::ByteFieldValue; -using document::Document; -using document::DoubleFieldValue; using document::FieldValue; -using document::ConstFieldValueVisitor; -using document::FloatFieldValue; -using document::IntFieldValue; -using document::LongFieldValue; -using document::MapFieldValue; -using document::PredicateFieldValue; -using document::RawFieldValue; -using document::ShortFieldValue; -using document::StringFieldValue; -using document::StructFieldValue; -using document::WeightedSetFieldValue; -using document::TensorFieldValue; -using document::ReferenceFieldValue; namespace search::docsummary { -namespace { - -struct FieldValueConverter { - virtual FieldValue::UP convert(const FieldValue &input) = 0; - virtual ~FieldValueConverter() = default; -}; - - -class SummaryFieldValueConverter : protected ConstFieldValueVisitor -{ - vespalib::asciistream _str; - bool _tokenize; - FieldValue::UP _field_value; - FieldValueConverter &_structuredFieldConverter; - - template <typename T> - void visitPrimitive(const T &t) { - _field_value.reset(t.clone()); - } - void visit(const IntFieldValue &value) override { visitPrimitive(value); } - void visit(const LongFieldValue &value) override { visitPrimitive(value); } - void visit(const ShortFieldValue &value) override { visitPrimitive(value); } - void visit(const BoolFieldValue &value) override { visitPrimitive(value); } - void visit(const ByteFieldValue &value) override { - int8_t signedValue = value.getAsByte(); - _field_value = std::make_unique<ShortFieldValue>(signedValue); - } - void visit(const DoubleFieldValue &value) override { visitPrimitive(value); } - void visit(const FloatFieldValue &value) override { visitPrimitive(value); } - - void visit(const StringFieldValue &value) override { - if (_tokenize) { - AnnotationConverter converter(value.getValue(), _str); - converter.handleIndexingTerms(value); - } else { - _str << value.getValue(); - } - } - - void visit(const AnnotationReferenceFieldValue & v ) override { - _field_value = _structuredFieldConverter.convert(v); - } - void visit(const Document & v) override { - _field_value = _structuredFieldConverter.convert(v); - } - - void visit(const PredicateFieldValue &value) override { - _str << value.toString(); - } - - void visit(const RawFieldValue &value) override { - visitPrimitive(value); - } - - void visit(const ArrayFieldValue &value) override { - if (value.size() > 0) { - _field_value = _structuredFieldConverter.convert(value); - } // else: implicit empty string - } - - void visit(const MapFieldValue & value) override { - if (value.size() > 0) { - _field_value = _structuredFieldConverter.convert(value); - } // else: implicit empty string - } - - void visit(const StructFieldValue &value) override { - if (*value.getDataType() == *SearchDataType::URI) { - FieldValue::UP uriAllValue = value.getValue("all"); - if (uriAllValue && uriAllValue->isA(FieldValue::Type::STRING)) { - uriAllValue->accept(*this); - return; - } - } - _field_value = _structuredFieldConverter.convert(value); - } - - void visit(const WeightedSetFieldValue &value) override { - if (value.size() > 0) { - _field_value = _structuredFieldConverter.convert(value); - } // else: implicit empty string - } - - void visit(const TensorFieldValue &value) override { - visitPrimitive(value); - } - - void visit(const ReferenceFieldValue& value) override { - if (value.hasValidDocumentId()) { - _str << value.getDocumentId().toString(); - } // else: implicit empty string - } - -public: - SummaryFieldValueConverter(bool tokenize, FieldValueConverter &subConverter); - ~SummaryFieldValueConverter() override; - - FieldValue::UP convert(const FieldValue &input) { - input.accept(*this); - if (_field_value.get()) { - return std::move(_field_value); - } - return StringFieldValue::make(_str.str()); - } -}; - -SummaryFieldValueConverter::SummaryFieldValueConverter(bool tokenize, FieldValueConverter &subConverter) - : _str(), _tokenize(tokenize), - _structuredFieldConverter(subConverter) -{} -SummaryFieldValueConverter::~SummaryFieldValueConverter() = default; - -using namespace vespalib::slime::convenience; - -class SlimeConverter : public FieldValueConverter { -private: - bool _tokenize; - const std::vector<uint32_t>* _matching_elems; - -public: - explicit SlimeConverter(bool tokenize) - : _tokenize(tokenize), - _matching_elems() - {} - - SlimeConverter(bool tokenize, const std::vector<uint32_t>& matching_elems) - : _tokenize(tokenize), - _matching_elems(&matching_elems) - {} - - FieldValue::UP convert(const FieldValue &input) override { - vespalib::Slime slime; - SlimeInserter inserter(slime); - SlimeFiller visitor(inserter, _tokenize, _matching_elems); - input.accept(visitor); - vespalib::SmartBuffer buffer(4_Ki); - vespalib::slime::BinaryFormat::encode(slime, buffer); - vespalib::Memory mem = buffer.obtain(); - return std::make_unique<RawFieldValue>(mem.data, mem.size); - } -}; - - -} // namespace - -FieldValue::UP -SummaryFieldConverter::convertSummaryField(bool markup, - const FieldValue &value) -{ - SlimeConverter subConv(markup); - return SummaryFieldValueConverter(markup, subConv).convert(value); -} - void SummaryFieldConverter::insert_summary_field(const FieldValue& value, vespalib::slime::Inserter& inserter) { CheckUndefinedValueVisitor check_undefined; value.accept(check_undefined); if (!check_undefined.is_undefined()) { - SlimeFiller visitor(inserter, false); + SlimeFiller visitor(inserter); value.accept(visitor); } } @@ -219,18 +26,18 @@ SummaryFieldConverter::insert_summary_field_with_filter(const FieldValue& value, CheckUndefinedValueVisitor check_undefined; value.accept(check_undefined); if (!check_undefined.is_undefined()) { - SlimeFiller visitor(inserter, false, &matching_elems); + SlimeFiller visitor(inserter, &matching_elems); value.accept(visitor); } } void -SummaryFieldConverter::insert_juniper_field(const document::FieldValue& value, vespalib::slime::Inserter& inserter, bool tokenize, IJuniperConverter& converter) +SummaryFieldConverter::insert_juniper_field(const document::FieldValue& value, vespalib::slime::Inserter& inserter, IStringFieldConverter& converter) { CheckUndefinedValueVisitor check_undefined; value.accept(check_undefined); if (!check_undefined.is_undefined()) { - SlimeFiller visitor(inserter, tokenize, &converter); + SlimeFiller visitor(inserter, &converter, nullptr); value.accept(visitor); } } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.h b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.h index 924ec6f402e..ce3bf80b365 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.h @@ -2,13 +2,16 @@ #pragma once -#include <vespa/document/fieldvalue/fieldvalue.h> +#include <cstdint> +#include <vector> + +namespace document { class FieldValue; } namespace vespalib::slime { struct Inserter; } namespace search::docsummary { -class IJuniperConverter; +class IStringFieldConverter; /** * This class converts a summary field for docsum fetching. @@ -16,18 +19,12 @@ class IJuniperConverter; class SummaryFieldConverter { public: - static document::FieldValue::UP convertSummaryField(bool markup, const document::FieldValue &value); - - static document::FieldValue::UP convert_field_with_filter(bool markup, - const document::FieldValue& value, - const std::vector<uint32_t>& matching_elems); - static void insert_summary_field(const document::FieldValue& value, vespalib::slime::Inserter& inserter); /** * Insert the given field value, but only the elements that are contained in the matching_elems vector. */ static void insert_summary_field_with_filter(const document::FieldValue& value, vespalib::slime::Inserter& inserter, const std::vector<uint32_t>& matching_elems); - static void insert_juniper_field(const document::FieldValue& value, vespalib::slime::Inserter& inserter, bool tokenize, IJuniperConverter& converter); + static void insert_juniper_field(const document::FieldValue& value, vespalib::slime::Inserter& inserter, IStringFieldConverter& converter); }; } diff --git a/storage/src/tests/frameworkimpl/status/statustest.cpp b/storage/src/tests/frameworkimpl/status/statustest.cpp index 2593eabecec..0db5f2cf6b0 100644 --- a/storage/src/tests/frameworkimpl/status/statustest.cpp +++ b/storage/src/tests/frameworkimpl/status/statustest.cpp @@ -12,6 +12,7 @@ #include <vespa/config/subscription/configuri.h> #include <vespa/vespalib/gtest/gtest.h> #include <gmock/gmock.h> +#include <string> using namespace ::testing; @@ -99,6 +100,19 @@ void StatusTest::SetUp() { _node = std::make_unique<TestServiceLayerApp>(); } +namespace { + +std::string additional_fixed_http_response_headers() { + return ("X-XSS-Protection: 1; mode=block\r\n" + "X-Frame-Options: DENY\r\n" + "Content-Security-Policy: default-src 'none'; frame-ancestors 'none'\r\n" + "X-Content-Type-Options: nosniff\r\n" + "Cache-Control: no-store\r\n" + "Pragma: no-cache\r\n"); +} + +} + TEST_F(StatusTest, index_status_page) { StatusComponent rep1(_node->getComponentRegister(), "foo", new HtmlStatusReporter( @@ -115,12 +129,7 @@ TEST_F(StatusTest, index_status_page) { "Connection: close\r\n" "Content-Type: text\\/html\r\n" "Content-Length: [0-9]+\r\n" - "X-XSS-Protection: 1; mode=block\r\n" - "X-Frame-Options: DENY\r\n" - "Content-Security-Policy: default-src 'none'; frame-ancestors 'none'\r\n" - "X-Content-Type-Options: nosniff\r\n" - "Cache-Control: no-store\r\n" - "Pragma: no-cache\r\n" + + additional_fixed_http_response_headers() + "\r\n" "<html>\n" "<head>\n" @@ -150,12 +159,33 @@ TEST_F(StatusTest, html_status) { "Connection: close\r\n" "Content-Type: text/html\r\n" "Content-Length: 117\r\n" - "X-XSS-Protection: 1; mode=block\r\n" - "X-Frame-Options: DENY\r\n" - "Content-Security-Policy: default-src 'none'; frame-ancestors 'none'\r\n" - "X-Content-Type-Options: nosniff\r\n" - "Cache-Control: no-store\r\n" - "Pragma: no-cache\r\n" + + additional_fixed_http_response_headers() + + "\r\n" + "<html>\n" + "<head>\n" + " <title>Foo impl</title>\n" + "<!-- script --></head>\n" + "<body>\n" + " <h1>Foo impl</h1>\n" + "<p>info</p></body>\n" + "</html>\n" + ); + EXPECT_EQ(expected, std::string(actual)); +} + +TEST_F(StatusTest, path_with_v1_prefix_aliases_to_handler_under_root) { + StatusComponent rep1(_node->getComponentRegister(), "foo", + new HtmlStatusReporter("fooid", "Foo impl", "<p>info</p>", "<!-- script -->")); + StatusWebServer webServer(_node->getComponentRegister(), + _node->getComponentRegister(), + config::ConfigUri("raw:httpport 0")); + auto actual = fetch(webServer.getListenPort(), "/contentnode-status/v1/fooid?unusedParam"); + std::string expected( + "HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" + "Content-Type: text/html\r\n" + "Content-Length: 117\r\n" + + additional_fixed_http_response_headers() + "\r\n" "<html>\n" "<head>\n" @@ -182,12 +212,7 @@ TEST_F(StatusTest, xml_sStatus) { "Connection: close\r\n" "Content-Type: application/xml\r\n" "Content-Length: 100\r\n" - "X-XSS-Protection: 1; mode=block\r\n" - "X-Frame-Options: DENY\r\n" - "Content-Security-Policy: default-src 'none'; frame-ancestors 'none'\r\n" - "X-Content-Type-Options: nosniff\r\n" - "Cache-Control: no-store\r\n" - "Pragma: no-cache\r\n" + + additional_fixed_http_response_headers() + "\r\n" "<?xml version=\"1.0\"?>\n" "<status id=\"fooid\" name=\"Foo impl\">\n" diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp index 166bda1adbb..51422de07e6 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp @@ -376,8 +376,8 @@ BucketManager::reportStatus(std::ostream& out, } else { framework::PartlyHtmlStatusReporter htmlReporter(*this); htmlReporter.reportHtmlHeader(out, path); - // Print menu - out << "<font size=\"-1\">[ <a href=\"/\">Back to top</a>" + // Print menu + out << "<font size=\"-1\">[ <a href=\"../\">Back to top</a>" << " | <a href=\"?showall\">Show all buckets</a> ]</font>"; htmlReporter.reportHtmlFooter(out, path); } diff --git a/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.cpp b/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.cpp index 1dc0ff15e81..e267626fd7f 100644 --- a/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.cpp +++ b/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.cpp @@ -2,6 +2,7 @@ #include "simplebucketprioritydatabase.h" #include <vespa/vespalib/stllike/hash_map.hpp> +#include <cassert> #include <ostream> #include <sstream> diff --git a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp index 7139ab0eb41..b2bce8a1241 100644 --- a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp +++ b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp @@ -169,10 +169,21 @@ void StatusWebServer::handlePage(const framework::HttpUrlPath& urlpath, vespalib::Portal::GetRequest request) { vespalib::string link(urlpath.getPath()); - if (!link.empty() && link[0] == '/') link = link.substr(1); + + // We allow a fixed path prefix that aliases down to whatever is provided after the prefix. + vespalib::stringref optional_status_path_prefix = "/contentnode-status/v1/"; + if (link.starts_with(optional_status_path_prefix)) { + link = link.substr(optional_status_path_prefix.size()); + } + + if (!link.empty() && link[0] == '/') { + link = link.substr(1); + } size_t slashPos = link.find('/'); - if (slashPos != std::string::npos) link = link.substr(0, slashPos); + if (slashPos != std::string::npos) { + link = link.substr(0, slashPos); + } if ( ! link.empty()) { const framework::StatusReporter *reporter = _reporterMap.getStatusReporter(link); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 62be96447a4..314836384ce 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -893,7 +893,7 @@ FileStorManager::reportHtmlStatus(std::ostream& out, const framework::HttpUrlPat bool showStatus = !path.hasAttribute("thread"); bool verbose = path.hasAttribute("verbose"); // Print menu - out << "<font size=\"-1\">[ <a href=\"/\">Back to top</a>" + out << "<font size=\"-1\">[ <a href=\"../\">Back to top</a>" << " | <a href=\"?" << (verbose ? "verbose" : "") << "\">Main filestor manager status page</a>" << " | <a href=\"?" << (verbose ? "notverbose" : "verbose"); diff --git a/storage/src/vespa/storage/visiting/visitormanager.cpp b/storage/src/vespa/storage/visiting/visitormanager.cpp index b305abae019..759f1f1f059 100644 --- a/storage/src/vespa/storage/visiting/visitormanager.cpp +++ b/storage/src/vespa/storage/visiting/visitormanager.cpp @@ -566,7 +566,7 @@ VisitorManager::reportHtmlStatus(std::ostream& out, bool showAll = path.hasAttribute("allvisitors"); // Print menu - out << "<font size=\"-1\">[ <a href=\"/\">Back to top</a>" + out << "<font size=\"-1\">[ <a href=\"../\">Back to top</a>" << " | <a href=\"?" << (verbose ? "verbose" : "") << "\">Main visitor manager status page</a>" << " | <a href=\"?allvisitors" << (verbose ? "&verbose" : "") diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index 0bf41f9a379..79bd5a4f77b 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -173,7 +173,7 @@ SearchVisitor::SummaryGenerator::fillSummary(AttributeVector::DocId lid, const H vespalib::Slime slime; vespalib::slime::SlimeInserter inserter(slime); auto& sds = get_streaming_docsums_state(summaryClass); - _docsumWriter->insertDocsum(sds.get_resolve_class_info(), lid, &sds.get_state(), _docsumFilter.get(), inserter); + _docsumWriter->insertDocsum(sds.get_resolve_class_info(), lid, sds.get_state(), _docsumFilter.get(), inserter); _buf.reset(); vespalib::WritableMemory magicId = _buf.reserve(4); memcpy(magicId.data, &search::docsummary::SLIME_MAGIC_ID, 4); diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp index 35410f3ec67..eaef0846536 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp @@ -1,18 +1,19 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "docsum_field_writer_factory.h" +#include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/searchsummary/docsummary/copy_dfw.h> #include <vespa/searchsummary/docsummary/docsum_field_writer.h> +#include <vespa/searchsummary/docsummary/docsum_field_writer_commands.h> #include <vespa/searchsummary/docsummary/empty_dfw.h> #include <vespa/searchsummary/docsummary/matched_elements_filter_dfw.h> -#include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/vsm/config/config-vsmfields.h> using search::MatchingElementsFields; using search::docsummary::CopyDFW; +using search::docsummary::DocsumFieldWriter; using search::docsummary::EmptyDFW; using search::docsummary::IDocsumEnvironment; -using search::docsummary::DocsumFieldWriter; using search::docsummary::MatchedElementsFilterDFW; using vespa::config::search::vsm::VsmfieldsConfig; @@ -44,35 +45,29 @@ DocsumFieldWriterFactory::DocsumFieldWriterFactory(bool use_v8_geo_positions, co DocsumFieldWriterFactory::~DocsumFieldWriterFactory() = default; std::unique_ptr<DocsumFieldWriter> -DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& fieldName, const vespalib::string& overrideName, const vespalib::string& argument, bool& rc) +DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& field_name, + const vespalib::string& command, + const vespalib::string& source) { std::unique_ptr<DocsumFieldWriter> fieldWriter; - if ((overrideName == "staticrank") || - (overrideName == "ranklog") || - (overrideName == "label") || - (overrideName == "project") || - (overrideName == "positions") || - (overrideName == "absdist") || - (overrideName == "subproject")) + using namespace search::docsummary; + if ((command == command::positions) || + (command == command::abs_distance)) { fieldWriter = std::make_unique<EmptyDFW>(); - rc = true; - } else if ((overrideName == "attribute") || - (overrideName == "attributecombiner")) { - if (!argument.empty() && argument != fieldName) { - fieldWriter = std::make_unique<CopyDFW>(argument); + } else if ((command == command::attribute) || + (command == command::attribute_combiner)) { + if (!source.empty() && source != field_name) { + fieldWriter = std::make_unique<CopyDFW>(source); } - rc = true; - } else if (overrideName == "geopos") { - rc = true; - } else if ((overrideName == "matchedattributeelementsfilter") || - (overrideName == "matchedelementsfilter")) { - vespalib::string source_field = argument.empty() ? fieldName : argument; + } else if (command == command::geo_position) { + } else if ((command == command::matched_attribute_elements_filter) || + (command == command::matched_elements_filter)) { + vespalib::string source_field = source.empty() ? field_name : source; populate_fields(*_matching_elems_fields, _vsm_fields_config, source_field); fieldWriter = MatchedElementsFilterDFW::create(source_field, _matching_elems_fields); - rc = static_cast<bool>(fieldWriter); } else { - return search::docsummary::DocsumFieldWriterFactory::create_docsum_field_writer(fieldName, overrideName, argument, rc); + return search::docsummary::DocsumFieldWriterFactory::create_docsum_field_writer(field_name, command, source); } return fieldWriter; } diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.h b/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.h index c06cb0454b3..17c270292e2 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.h +++ b/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.h @@ -19,7 +19,9 @@ public: DocsumFieldWriterFactory(bool use_v8_geo_positions, const search::docsummary::IDocsumEnvironment& env, const vespa::config::search::vsm::VsmfieldsConfig& vsm_fields_config); ~DocsumFieldWriterFactory() override; std::unique_ptr<search::docsummary::DocsumFieldWriter> - create_docsum_field_writer(const vespalib::string& fieldName, const vespalib::string& overrideName, const vespalib::string& argument, bool& rc) override; + create_docsum_field_writer(const vespalib::string& field_name, + const vespalib::string& command, + const vespalib::string& source) override; }; } diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp index 69a38e205c0..ca1de0082f0 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp @@ -6,6 +6,7 @@ #include <vespa/searchsummary/docsummary/check_undefined_value_visitor.h> #include <vespa/searchsummary/docsummary/i_docsum_store_document.h> #include <vespa/searchsummary/docsummary/i_juniper_converter.h> +#include <vespa/searchsummary/docsummary/i_string_field_converter.h> #include <vespa/searchsummary/docsummary/summaryfieldconverter.h> #include <vespa/document/base/exceptions.h> #include <vespa/document/fieldvalue/iteratorhandler.h> @@ -30,38 +31,34 @@ bool is_struct_or_multivalue_field_type(const document::DataType& data_type) * This class creates a modified field value which is then passed to * the original juniper converter. */ -class SnippetModifierJuniperConverter : public IJuniperConverter +class SnippetModifierJuniperConverter : public IStringFieldConverter { - IJuniperConverter& _orig_converter; - FieldModifier& _modifier; + IJuniperConverter& _juniper_converter; + FieldModifier* _modifier; FieldPath _empty_field_path; public: - SnippetModifierJuniperConverter(IJuniperConverter& orig_converter, FieldModifier& modifier) - : IJuniperConverter(), - _orig_converter(orig_converter), + SnippetModifierJuniperConverter(IJuniperConverter& juniper_converter, FieldModifier* modifier) + : IStringFieldConverter(), + _juniper_converter(juniper_converter), _modifier(modifier), _empty_field_path() { } ~SnippetModifierJuniperConverter() override = default; - void insert_juniper_field(vespalib::stringref input, vespalib::slime::Inserter& inserter) override; - void insert_juniper_field(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) override; + void convert(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) override; }; - -void -SnippetModifierJuniperConverter::insert_juniper_field(vespalib::stringref input, vespalib::slime::Inserter& inserter) -{ - _orig_converter.insert_juniper_field(input, inserter); -} - void -SnippetModifierJuniperConverter::insert_juniper_field(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) +SnippetModifierJuniperConverter::convert(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) { - auto fv = _modifier.modify(input, _empty_field_path); - assert(fv); - auto& modified_input = dynamic_cast<const document::StringFieldValue &>(*fv); - _orig_converter.insert_juniper_field(modified_input.getValueRef(), inserter); + if (_modifier != nullptr) { + auto fv = _modifier->modify(input, _empty_field_path); + assert(fv); + auto& modified_input = dynamic_cast<const document::StringFieldValue &>(*fv); + _juniper_converter.convert(modified_input.getValueRef(), inserter); + } else { + _juniper_converter.convert(input.getValueRef(), inserter); + } } /** @@ -155,22 +152,20 @@ DocsumStoreVsmDocument::insert_summary_field(const vespalib::string& field_name, void DocsumStoreVsmDocument::insert_juniper_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter, IJuniperConverter& converter) const { - // Markup for juniper has already been added due to FLATTENJUNIPER command in vsm summary config. auto field_value = get_field_value(field_name); if (field_value) { + FieldModifier* modifier = nullptr; if (is_struct_or_multivalue_field_type(*field_value->getDataType())) { auto entry_idx = _result_class.GetIndexFromName(field_name.c_str()); if (entry_idx >= 0) { assert((uint32_t) entry_idx < _result_class.GetNumEntries()); - auto modifier = _docsum_filter.get_field_modifier(entry_idx); - if (modifier != nullptr) { - SnippetModifierJuniperConverter stacked_converter(converter, *modifier); - SummaryFieldConverter::insert_juniper_field(*field_value, inserter, false, stacked_converter); - return; - } + modifier = _docsum_filter.get_field_modifier(entry_idx); } + } else { + // Markup for juniper has already been added due to FLATTENJUNIPER command in vsm summary config. } - SummaryFieldConverter::insert_juniper_field(*field_value, inserter, false, converter); + SnippetModifierJuniperConverter string_converter(converter, modifier); + SummaryFieldConverter::insert_juniper_field(*field_value, inserter, string_converter); } } diff --git a/vespaclient/CMakeLists.txt b/vespaclient/CMakeLists.txt index 86e30ab1051..9593304cccd 100644 --- a/vespaclient/CMakeLists.txt +++ b/vespaclient/CMakeLists.txt @@ -17,17 +17,3 @@ vespa_define_module( src/vespa/vespaclient/vdsstates src/vespa/vespaclient/vesparoute ) - -vespa_install_script(src/perl/bin/SetNodeState.pl libexec/vespa) -vespa_install_script(src/perl/bin/GetNodeState.pl libexec/vespa) -vespa_install_script(src/perl/bin/GetClusterState.pl libexec/vespa) - -install(DIRECTORY src/perl/lib/Yahoo/Vespa/ - DESTINATION lib/perl5/site_perl/Yahoo/Vespa - FILES_MATCHING - PATTERN "*.pm") - -install(DIRECTORY src/perl/lib/Yahoo/Vespa/Bin/ - DESTINATION lib/perl5/site_perl/Yahoo/Vespa/Bin - FILES_MATCHING - PATTERN "*.pm") diff --git a/vespaclient/src/perl/PERL_BEST_PRACTISES b/vespaclient/src/perl/PERL_BEST_PRACTISES deleted file mode 100644 index 29bbc3f01bf..00000000000 --- a/vespaclient/src/perl/PERL_BEST_PRACTISES +++ /dev/null @@ -1,361 +0,0 @@ -To try and make the perl tools good and consistent, here is a list of best -practises used within the modules. - -(Whether they are best can of course be debated, but what's listed is what is -currently used) - -1. Always use strict and warnings first thing. - -There is a lot of stuff legal in perl for backward compatability and ease of -writing one liners. However, these statements are frequent source of bugs in -real code. All modules and binaries should use strict and warnings to ensure -that these checks are enabled. (There is a unit test in the module grepping -source to ensure this). Thus, pretty much the first thing in all perl files -should be: - - use strict; - use warnings; - -2. Use perl modules. - -We want to group functionality into multiple files in perl too. A perl module is -just another perl file with a .pm extension, which minimally can look something -like this: - -Yahoo/Vespa/VespaModel.pm: - - package Yahoo::Vespa::VespaModel; - - use strict; - use warnings; - - my %CACHED_MODEL; # Prevent multiple fetches by caching results - - return 1; - - sub get { - ... - } - -Yahoo/Vespa/Bin/MyBinary.pl: - - use strict; - use warnings; - use Yahoo::Vespa::VespaModel; - - my $model = Yahoo::Vespa::VespaModel::get(); - -2a. Module install locations. - -Perl utilities are installed under $VESPA_HOME/lib/perl5/site_perl - -2b. Aliasing namespace. - -Perl doesn't have that great namespace handling. It's not like in C++, where we -can be in the storage::api namespace and thus address something in the -storage::lib namespace as lib::foo or even refer to another instance in the -same namespace. Thus, if the user of the VespaModel module above were -Yahoo::Vespa::MyLib, it still has to address VespaModel with full path by -default. - -It is possible to create aliases in Perl to help this. Using an alias the -MyBinary.pl code above could look like: - - ... - use Yahoo::Vespa::VespaModel; - - BEGIN { - *VespaModel:: = *Yahoo::Vespa::VespaModel:: ; - } - - my $model = VespaModel::get(); - -The alias declaration doesn't look very pretty, but it can be helpful to get -code looking simple. - -2b. Exporting members into users namespace. - -Another option to using long prefixed names or aliasing, is to export names -into the callers namespace. This can be done in a module doing something like -this: - -Yahoo/Vespa/VespaModel.pm: - - package Yahoo::Vespa::VespaModel; - - use strict; - use warnings; - - BEGIN { - use base 'Exporter'; - our @EXPORT = qw( getVespaModel ); - our @EXPORT_OK = qw( otherFunction ); - } - - my %CACHED_MODEL; - - return 1; - - sub getVespaModel { - ... - } - sub otherFunction { - ... - } - -Yahoo/Vespa/Bin/MyBinary.pl: - - use strict; - use warnings; - use Yahoo::Vespa::VespaModel; - - my $model = getVespaModel(); - -In this example, the getVespaModel function is imported by default, while -otherFunction is not, but can be included optionally. You can specify what to -include by adding arguments to the use statements: - -use Yahoo::Vespa::VespaMode; # Import defaults -use Yahoo::Vespa::VespaModel (); # Import nothing - # Import other function but not getVespaModel -use Yahoo::Vespa::VespaModel qw( otherFunction ); - -(The qw(...) function is just a function to generate an array from a whitespace separated string. Writing qw( foo bar ) is equivalent to writing ('foo', 'bar')) - -You can also export/import variables, but then you need to prefix the names -with the type, as in "our @EXPORT = qw( $number, @list, %hash );". - -Note that you should prefer to export as little functions as possible as they -can clash with names used in caller. Also, the tokens you do export should have -fairly descriptive names to reduce the chance of this happening. An exported -name does not have a module name tagged to it to include context. Thus, if you -don't export you can for instance use Json::encode, but if you do export you -likely need to call the function encodeJson or similar instead. - -2c. Prefer private variables (my instead of our) - -When declaring variables with 'my' they become private to the module, and you -know outsiders can't alter it. This makes it easier when debugging as there are -less possibilities for what can happen. - -2d. Prefer calling functions or exported variables rather than referencing -global variables in a module from the outside. - -Referencing non-declared variables in another module does not seem to create -compiler warnings, nor does using private (my) declared variables. Thus it's -better to refer to imported variables or call a function, such that the -compiler will tell you when this doesn't work anymore. - -2e. Put all function declarations at the bottom. - -When a perl module is loaded, the code within it run. If that doesn't return -true, that means the module fails to load. Thus, traditionally, perl modules -often end with 1; (equivalent to return 1;) to ensure this. However, this mean -you have to read through the entire module to look for module code run. - -By doing exit(...) call in main prog before function declaration and return; in -modules before function declarations, it is easier for reader to see that you -haven't hidden other code between the function declarations. (Unless you've -hacked it into a BEGIN{} block to enforce it to run before everything else) - -2f. Make it easy to reinitialize in unit tests. - -By putting initialization steps in a separate init function, rather than doing -it on load, unit tests can easily call it to reinitialize the module between -tests. Also this separates declarations of what exist from the initialization so -it is easier to see what variables are there. - -3. Confess instead of die. - -The typical perl assert is use of the 'die' function, as in: - - defined $foo or die "We expected 'foo' to be defined here"; - -The Utils package contains a confess function to be used instead (Wrapping an -external dependency), which will do the same as 'die', but will add a -stacktrace too, such that when encountered, it is much easier to find the -culprit. - -4. Do not call exit() in libraries. - -We want to be able to unit test all types of functions in unit tests, also -functionality that makes application abort and exit. The Utils defines an -exitApplication that is mocked for unit tests. Assertion types of exits with -die/confess can also be catched in unit tests. - -5. Code conventions. - - - Upper case, underscore divided, module level variables. - - Camel case function names. - - Four space indent. - -6. Naming function arguments. - -For perl, a function is just a call to a subroutine with a list containing -whatever arguments, called @_. Using this directly makes the code hard to read. -Naming variables makes this a bit easier.. - - sub getVespaModel { # (ConfigServerHost, ConfigServerPort) - return Json::parse(Http::get("http://$_[0]:$_[1]/foo"#)); - } - - sub getVespaModel { # (ConfigServerHost, ConfigServerPort) -> ObjTree - my ($host, $port) = @_; - return Json::parse(Http::get("http://$host:$port/foo"#)); - } - -In the latter example it is easier to read the code. - -The argument comment is something I usually add for function declarations to -look better with vim folding.. When I fold functions in vim, the below line will -look like - -+-- 4 lines: sub getVespaModel (ConfigServerHost, ConfigServerPort) -> ObjTree - -Using such a convention it is thus easier to read the code, as you may be able -to see all your other function declarations while working on the function you -have expanded. - -6b. Functions with many arguments. - -If you create functions with loads of parameters you can end up with a messy -function, and a hard time to adjust all the uses of it when you want to extend -it. At these times you may use hashes to name variables, such that the order -is no longer important.. - - sub getVespaModel { # (ConfigServerHost, ConfigServerPort) -> ObjTree - my $args = $_[0]; - return Json::parse(Http::get("http://$$args{'host':$$args{'port'}/foo"#)); - } - - getVespaModel({ 'host' => 'myhost', 'port' => 80 }); - -Using this trick, you can have defaults for various arguments that can be -ignored by users not caring, rather than having to pass undef at many positions -to ensure order of parameters is correct. - -Note however, that this looks a bit more messy in the function itself, and it -makes it more important to make comments of what arguments are actually handled -and which ones are not optional.. I prefer to try and have short argument -lists instead. - -7. Constants - -Sometimes you want to declare constants. Valid flag values for instance. You -can of course just declare global variables, but you have no way of ensuring -that they never change, which can be confusing. To define constants you can -do the following: - - use constant MY_FLAG => 8; - -This constant is referred to without the usual $ prefix too, so it is easy to -distinguish it from variables. These constants can also be exported, enabling -you to create function calls like: - - MyModule::foo("bar", OPTION_ARGH | OPTION_BAZ); - -Though this of course pollutes callers namespace again, so he has to -specifically not include them if he otherwise would have a name clash. - -8. Libraries not in search path - -Sometimes people install perl libraries in non-default locations. If temporary -you can fix this by add directory to PERLLIB on command line, but if permanent, -the recommended way to find the libraries is to add the directory to the search -path where you include it, like the Yahoo installation for the JSON library: - - use lib '$VESPA_HOME/lib64/perl5/site_perl/5.14/'; - use JSON; - -9. Perl references - -In perl you can create references to variables by prefixing a backslash '\'. - - my @foo ; my $listref = \@foo; - my $var ; my $scalarref = \$var; - my %bar ; my $hashref = \%bar; - -You can also create references to lists and hashes directly: - - my $listref = [ 1, 2, 4 ]; # [] instead of () to get ref instead of list. - my $hashref = { 'foo' => 3, 'bar' => 'hmm' }; # {} instead of () - -To check what a variable is you can use the ref() function: - - ref($scalarref) eq 'SCALAR' - ref($listref) eq 'ARRAY' - ref($hashref) eq 'HASH' - ref($var) == undef - -To dereference a reference you can add a deref clause around it: - my @foo = @{ $listref }; - my %bar = %{ $hashref }; - my $scalar = ${ $scalarref }; - -If the insides of the clause is easy, you also omit it. - my $scalar = $$scalarref; - my %bar = %$hashref; - my $value = $$hashref{'foo'} - -You can also dereference using the -> operator. - my $value = $hashref->{'foo'}; - my $value2 = $listref->[3]; # Element 3 in the list - -The -> operator is typically used when traversing object structures. - -10. Perl structs - -Perl object programming requires some blessing and doesn't look that awesome, -so I typically mostly program functionally. However, at the bare minimum one -needs to be able to create some structs to contain data that isn't bare -primitives. - -Perl's Class::Struct module implements a way to define structs in a simple -fashion without needing to know how bless works, module inheritation and so -forth. - -An example use case here is Yahoo::Vespa::ClusterState - - use Class::Struct; - - struct( ClusterState => { - globalState => '$', - distributor => '%', - storage => '%' - }); - - struct( Node => { - group => '$', - unit => 'State', - generated => 'State', - user => 'State' - }); - - struct( State => { - state => '$', - reason => '$', - timestamp => '$', - source => '$' - }); - -# Some file using it. - - use Yahoo::Vespa::ClusterState; - - my $clusterState = new ClusterState; - $clusterState->globalState('UP'); - my $node = new Node; - $node->group('Foo'); - $clusterState->distributor('0', $node); - - ... - - my $group = $clusterState->distributor->{'0'}->group; - my $nodetype = 'storage'; - my $group = $clusterState->$nodetype->{'0'}->group; - -Some notes: - - The names of the structs are automatically imported. Thus you don't need to - worry about prefixing or aliasing, but be aware names can collide for user. - - $, % or @ indicates if content is scalar, hash or list. A name indicates the - name of another struct that should have the content. diff --git a/vespaclient/src/perl/bin/GetClusterState.pl b/vespaclient/src/perl/bin/GetClusterState.pl deleted file mode 100755 index c462eb77ade..00000000000 --- a/vespaclient/src/perl/bin/GetClusterState.pl +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env perl -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -# BEGIN perl environment bootstrap section -# Do not edit between here and END as this section should stay identical in all scripts - -use File::Basename; -use File::Path; - -sub findpath { - my $myfullname = ${0}; - my($myname, $mypath) = fileparse($myfullname); - - return $mypath if ( $mypath && -d $mypath ); - $mypath=`pwd`; - - my $pwdfullname = $mypath . "/" . $myname; - return $mypath if ( -f $pwdfullname ); - return 0; -} - -# Returns the argument path if it seems to point to VESPA_HOME, 0 otherwise -sub is_vespa_home { - my($VESPA_HOME) = shift; - my $COMMON_ENV="libexec/vespa/common-env.sh"; - if ( $VESPA_HOME && -d $VESPA_HOME ) { - my $common_env = $VESPA_HOME . "/" . $COMMON_ENV; - return $VESPA_HOME if -f $common_env; - } - return 0; -} - -# Returns the home of Vespa, or dies if it cannot -sub findhome { - # Try the VESPA_HOME env variable - return $ENV{'VESPA_HOME'} if is_vespa_home($ENV{'VESPA_HOME'}); - if ( $ENV{'VESPA_HOME'} ) { # was set, but not correctly - die "FATAL: bad VESPA_HOME value '" . $ENV{'VESPA_HOME'} . "'\n"; - } - - # Try the ROOT env variable - my $ROOT = $ENV{'ROOT'}; - return $ROOT if is_vespa_home($ROOT); - - # Try the script location or current dir - my $mypath = findpath(); - if ($mypath) { - while ( $mypath =~ s|/[^/]*$|| ) { - return $mypath if is_vespa_home($mypath); - } - } - die "FATAL: Missing VESPA_HOME environment variable\n"; -} - -sub findhost { - my $tmp = $ENV{'VESPA_HOSTNAME'}; - my $bin = $ENV{'VESPA_HOME'} . "/bin"; - if (!defined $tmp) { - $tmp = `${bin}/vespa-detect-hostname || hostname -f || hostname || echo "localhost"`; - chomp $tmp; - } - my $validate = "${bin}/vespa-validate-hostname"; - if (-f "${validate}") { - system("${validate} $tmp"); - ( $? == 0 ) or die "Could not validate hostname\n"; - } - return $tmp; -} - -BEGIN { - my $tmp = findhome(); - $ENV{'VESPA_HOME'} = $tmp; - $tmp = findhost(); - $ENV{'VESPA_HOSTNAME'} = $tmp; -} -my $VESPA_HOME = $ENV{'VESPA_HOME'}; - -use lib $ENV{'VESPA_HOME'} . "/lib/perl5/site_perl"; - -# END perl environment bootstrap section - -use Yahoo::Vespa::Defaults; -readConfFile(); - -use strict; -use warnings; - -use Yahoo::Vespa::Bin::GetClusterState; - -exit(getClusterState(\@ARGV)); diff --git a/vespaclient/src/perl/bin/GetNodeState.pl b/vespaclient/src/perl/bin/GetNodeState.pl deleted file mode 100755 index 03947e22199..00000000000 --- a/vespaclient/src/perl/bin/GetNodeState.pl +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env perl -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -# BEGIN perl environment bootstrap section -# Do not edit between here and END as this section should stay identical in all scripts - -use File::Basename; -use File::Path; - -sub findpath { - my $myfullname = ${0}; - my($myname, $mypath) = fileparse($myfullname); - - return $mypath if ( $mypath && -d $mypath ); - $mypath=`pwd`; - - my $pwdfullname = $mypath . "/" . $myname; - return $mypath if ( -f $pwdfullname ); - return 0; -} - -# Returns the argument path if it seems to point to VESPA_HOME, 0 otherwise -sub is_vespa_home { - my($VESPA_HOME) = shift; - my $COMMON_ENV="libexec/vespa/common-env.sh"; - if ( $VESPA_HOME && -d $VESPA_HOME ) { - my $common_env = $VESPA_HOME . "/" . $COMMON_ENV; - return $VESPA_HOME if -f $common_env; - } - return 0; -} - -# Returns the home of Vespa, or dies if it cannot -sub findhome { - # Try the VESPA_HOME env variable - return $ENV{'VESPA_HOME'} if is_vespa_home($ENV{'VESPA_HOME'}); - if ( $ENV{'VESPA_HOME'} ) { # was set, but not correctly - die "FATAL: bad VESPA_HOME value '" . $ENV{'VESPA_HOME'} . "'\n"; - } - - # Try the ROOT env variable - my $ROOT = $ENV{'ROOT'}; - return $ROOT if is_vespa_home($ROOT); - - # Try the script location or current dir - my $mypath = findpath(); - if ($mypath) { - while ( $mypath =~ s|/[^/]*$|| ) { - return $mypath if is_vespa_home($mypath); - } - } - die "FATAL: Missing VESPA_HOME environment variable\n"; -} - -sub findhost { - my $tmp = $ENV{'VESPA_HOSTNAME'}; - my $bin = $ENV{'VESPA_HOME'} . "/bin"; - if (!defined $tmp) { - $tmp = `${bin}/vespa-detect-hostname || hostname -f || hostname || echo "localhost"`; - chomp $tmp; - } - my $validate = "${bin}/vespa-validate-hostname"; - if (-f "${validate}") { - system("${validate} $tmp"); - ( $? == 0 ) or die "Could not validate hostname\n"; - } - return $tmp; -} - -BEGIN { - my $tmp = findhome(); - $ENV{'VESPA_HOME'} = $tmp; - $tmp = findhost(); - $ENV{'VESPA_HOSTNAME'} = $tmp; -} -my $VESPA_HOME = $ENV{'VESPA_HOME'}; - -use lib $ENV{'VESPA_HOME'} . "/lib/perl5/site_perl"; - -# END perl environment bootstrap section - -use Yahoo::Vespa::Defaults; -readConfFile(); - -use strict; -use warnings; - -use Yahoo::Vespa::Bin::GetNodeState; - -exit(getNodeState(\@ARGV)); diff --git a/vespaclient/src/perl/bin/SetNodeState.pl b/vespaclient/src/perl/bin/SetNodeState.pl deleted file mode 100755 index 4375aa96112..00000000000 --- a/vespaclient/src/perl/bin/SetNodeState.pl +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env perl -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -# BEGIN perl environment bootstrap section -# Do not edit between here and END as this section should stay identical in all scripts - -use File::Basename; -use File::Path; - -sub findpath { - my $myfullname = ${0}; - my($myname, $mypath) = fileparse($myfullname); - - return $mypath if ( $mypath && -d $mypath ); - $mypath=`pwd`; - - my $pwdfullname = $mypath . "/" . $myname; - return $mypath if ( -f $pwdfullname ); - return 0; -} - -# Returns the argument path if it seems to point to VESPA_HOME, 0 otherwise -sub is_vespa_home { - my($VESPA_HOME) = shift; - my $COMMON_ENV="libexec/vespa/common-env.sh"; - if ( $VESPA_HOME && -d $VESPA_HOME ) { - my $common_env = $VESPA_HOME . "/" . $COMMON_ENV; - return $VESPA_HOME if -f $common_env; - } - return 0; -} - -# Returns the home of Vespa, or dies if it cannot -sub findhome { - # Try the VESPA_HOME env variable - return $ENV{'VESPA_HOME'} if is_vespa_home($ENV{'VESPA_HOME'}); - if ( $ENV{'VESPA_HOME'} ) { # was set, but not correctly - die "FATAL: bad VESPA_HOME value '" . $ENV{'VESPA_HOME'} . "'\n"; - } - - # Try the ROOT env variable - my $ROOT = $ENV{'ROOT'}; - return $ROOT if is_vespa_home($ROOT); - - # Try the script location or current dir - my $mypath = findpath(); - if ($mypath) { - while ( $mypath =~ s|/[^/]*$|| ) { - return $mypath if is_vespa_home($mypath); - } - } - die "FATAL: Missing VESPA_HOME environment variable\n"; -} - -sub findhost { - my $tmp = $ENV{'VESPA_HOSTNAME'}; - my $bin = $ENV{'VESPA_HOME'} . "/bin"; - if (!defined $tmp) { - $tmp = `${bin}/vespa-detect-hostname || hostname -f || hostname || echo "localhost"`; - chomp $tmp; - } - my $validate = "${bin}/vespa-validate-hostname"; - if (-f "${validate}") { - system("${validate} $tmp"); - ( $? == 0 ) or die "Could not validate hostname\n"; - } - return $tmp; -} - -BEGIN { - my $tmp = findhome(); - $ENV{'VESPA_HOME'} = $tmp; - $tmp = findhost(); - $ENV{'VESPA_HOSTNAME'} = $tmp; -} -my $VESPA_HOME = $ENV{'VESPA_HOME'}; - -use lib $ENV{'VESPA_HOME'} . "/lib/perl5/site_perl"; - -# END perl environment bootstrap section - -use Yahoo::Vespa::Defaults; -readConfFile(); - -use strict; -use warnings; -use Yahoo::Vespa::Bin::SetNodeState; -exit(setNodeState(\@ARGV)); diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/ArgParser.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/ArgParser.pm deleted file mode 100644 index 5abcd0b1fbb..00000000000 --- a/vespaclient/src/perl/lib/Yahoo/Vespa/ArgParser.pm +++ /dev/null @@ -1,689 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# -# Argument parser. -# -# Intentions: -# - Make it very easy for programs to get info from command line. -# - Allow shared libraries to register own options, such that a program can -# delegate command line options to libraries used. (For instance, verbosity -# arguments will be automatically delegated to console output module without -# program needing to care much. -# - Create a unified looking syntax page for all command line tools. -# - Be able to reuse input validation. For instance that an integer don't -# have a decimal point and that a hostname can be resolved. -# - -package Yahoo::Vespa::ArgParser; - -use strict; -use warnings; -use Yahoo::Vespa::ConsoleOutput; -use Yahoo::Vespa::Utils; - -BEGIN { # - Define exports and dependency aliases for module. - use base 'Exporter'; - our @EXPORT = qw( - addArgParserValidator - setProgramBinaryName setProgramDescription - setArgument setOptionHeader - setFlagOption setHostOption setPortOption setStringOption - setIntegerOption setFloatOption setUpCountingOption setDownCountingOption - handleCommandLineArguments - OPTION_SECRET OPTION_INVERTEDFLAG OPTION_REQUIRED - ); - # Alias so we can avoid writing the entire package name - *ConsoleOutput:: = *Yahoo::Vespa::ConsoleOutput:: -} - -my @ARGUMENTS; -my $DESCRIPTION; -my $BINARY_NAME; -my @ARG_SPEC_ARRAY; -my %OPTION_SPEC; -my @OPTION_SPEC_ARRAY; -my $SYNTAX_PAGE; -my $SHOW_HIDDEN; -my @VALIDATORS; -use constant OPTION_SECRET => 1; -use constant OPTION_INVERTEDFLAG => 2; -use constant OPTION_ADDFIRST => 4; -use constant OPTION_REQUIRED => 8; - -# These variables are properties needed by ConsoleOutput module. ArgParser -# handles that modules argument settings as it cannot possibly depend upon -# ArgParser itself. -my $VERBOSITY; # Default verbosity before parsing arguments -my $ANSI_COLORS; # Whether to use ansi colors or not. - -&initialize(); - -return 1; - -########################## Default exported functions ######################## - -sub handleCommandLineArguments { # () Parses and sets all values - my ($args, $validate_args_sub) = @_; - - ®isterInternalParameters(); - if (!&parseCommandLineArguments($args)) { - &writeSyntaxPage(); - exitApplication(1); - } - if (defined $validate_args_sub && !&$validate_args_sub()) { - &writeSyntaxPage(); - exitApplication(1); - } - if ($SYNTAX_PAGE) { - &writeSyntaxPage(); - exitApplication(0); - } -} - -sub addArgParserValidator { # (Validator) Add callback to verify parsing - # Using such callbacks you can verify more than is supported natively by - # argument parser, such that you can fail argument parsing at same step as - # internally supported checks are handled. - scalar @_ == 1 or confess "Invalid number of arguments given."; - push @VALIDATORS, $_[0]; -} -sub setProgramBinaryName { # (Name) Defaults to name used on command line - scalar @_ == 1 or confess "Invalid number of arguments given."; - ($BINARY_NAME) = @_; -} -sub setProgramDescription { # (Description) - scalar @_ == 1 or confess "Invalid number of arguments given."; - ($DESCRIPTION) = @_; -} - -sub setOptionHeader { # (Description) - my ($desc) = @_; - push @OPTION_SPEC_ARRAY, $desc; -} - -sub setFlagOption { # (ids[], Result&, Description, Flags) - scalar @_ >= 3 or confess "Invalid number of arguments given."; - my ($ids, $result, $description, $flags) = @_; - if (!defined $flags) { $flags = 0; } - my %optionspec = ( - 'result' => $result, - 'flags' => $flags, - 'ids' => $ids, - 'description' => $description, - 'arg_count' => 0, - 'initializer' => sub { - $$result = (($flags & OPTION_INVERTEDFLAG) == 0 ? 0 : 1); - return 1; - }, - 'result_evaluator' => sub { - $$result = (($flags & OPTION_INVERTEDFLAG) == 0 ? 1 : 0); - return 1; - } - ); - setGenericOption($ids, \%optionspec); -} -sub setHostOption { # (ids[], Result&, Description, Flags) - my ($ids, $result, $description, $flags) = @_; - my %optionspec = ( - 'result' => $result, - 'flags' => $flags, - 'ids' => $ids, - 'description' => $description, - 'arg_count' => 1, - 'result_evaluator' => sub { - my ($id, $args) = @_; - scalar @$args == 1 or confess "Should have one arg here."; - my $host = $$args[0]; - if (!&validHost($host)) { - printError "Invalid host '$host' given to option '$id'. " - . "Not a valid host\n"; - return 0; - } - printSpam "Set value of '$id' to $host.\n"; - $$result = $host; - return 1; - } - ); - setGenericOption($ids, \%optionspec); -} -sub setPortOption { # (ids[], Result&, Description, Flags) - my ($ids, $result, $description, $flags) = @_; - my %optionspec = ( - 'result' => $result, - 'flags' => $flags, - 'ids' => $ids, - 'description' => $description, - 'arg_count' => 1, - 'result_evaluator' => sub { - my ($id, $args) = @_; - scalar @$args == 1 or confess "Should have one arg here."; - my $val = $$args[0]; - if ($val !~ /^\d+$/ || $val < 0 || $val >= 65536) { - printError "Invalid value '$val' given to port option '$id'." - . " Must be an unsigned 16 bit integer.\n"; - return 0; - } - printSpam "Set value of '$id' to $val.\n"; - $$result = $val; - return 1; - } - ); - setGenericOption($ids, \%optionspec); -} -sub setIntegerOption { # (ids[], Result&, Description, Flags) - my ($ids, $result, $description, $flags) = @_; - my %optionspec = ( - 'result' => $result, - 'flags' => $flags, - 'ids' => $ids, - 'description' => $description, - 'arg_count' => 1, - 'result_evaluator' => sub { - my ($id, $args) = @_; - scalar @$args == 1 or confess "Should have one arg here."; - my $val = $$args[0]; - if ($val !~ /^(?:[-\+])?\d+$/) { - printError "Invalid value '$val' given to integer option " - . "'$id'.\n"; - return 0; - } - printSpam "Set value of '$id' to $val.\n"; - $$result = $val; - return 1; - } - ); - setGenericOption($ids, \%optionspec); -} -sub setFloatOption { # (ids[], Result&, Description, Flags) - my ($ids, $result, $description, $flags) = @_; - my %optionspec = ( - 'result' => $result, - 'flags' => $flags, - 'ids' => $ids, - 'description' => $description, - 'arg_count' => 1, - 'result_evaluator' => sub { - my ($id, $args) = @_; - scalar @$args == 1 or confess "Should have one arg here."; - my $val = $$args[0]; - if ($val !~ /^(?:[-\+])?\d+(?:\.\d+)?$/) { - printError "Invalid value '$val' given to float option " - . "'$id'.\n"; - return 0; - } - printSpam "Set value of '$id' to $val.\n"; - $$result = $val; - return 1; - } - ); - setGenericOption($ids, \%optionspec); -} -sub setStringOption { # (ids[], Result&, Description, Flags) - my ($ids, $result, $description, $flags) = @_; - my %optionspec = ( - 'result' => $result, - 'flags' => $flags, - 'ids' => $ids, - 'description' => $description, - 'arg_count' => 1, - 'result_evaluator' => sub { - my ($id, $args) = @_; - scalar @$args == 1 or confess "Should have one arg here."; - my $val = $$args[0]; - printSpam "Set value of '$id' to $val.\n"; - $$result = $val; - return 1; - } - ); - setGenericOption($ids, \%optionspec); -} -sub setUpCountingOption { # (ids[], Result&, Description, Flags) - my ($ids, $result, $description, $flags) = @_; - my $org = $$result; - my %optionspec = ( - 'result' => $result, - 'flags' => $flags, - 'ids' => $ids, - 'description' => $description, - 'arg_count' => 0, - 'initializer' => sub { - $$result = $org; - return 1; - }, - 'result_evaluator' => sub { - if (!defined $$result) { - $$result = 0; - } - ++$$result; - return 1; - } - ); - setGenericOption($ids, \%optionspec); -} -sub setDownCountingOption { # (ids[], Result&, Description, Flags) - my ($ids, $result, $description, $flags) = @_; - my $org = $$result; - my %optionspec = ( - 'result' => $result, - 'flags' => $flags, - 'ids' => $ids, - 'description' => $description, - 'arg_count' => 0, - 'initializer' => sub { - $$result = $org; - return 1; - }, - 'result_evaluator' => sub { - if (!defined $$result) { - $$result = 0; - } - --$$result; - return 1; - } - ); - setGenericOption($ids, \%optionspec); -} - -sub setArgument { # (Result&, Name, Description) - my ($result, $name, $description, $flags) = @_; - if (!defined $flags) { $flags = 0; } - if (scalar @ARG_SPEC_ARRAY > 0 && ($flags & OPTION_REQUIRED) != 0) { - my $last = $ARG_SPEC_ARRAY[scalar @ARG_SPEC_ARRAY - 1]; - if (($$last{'flags'} & OPTION_REQUIRED) == 0) { - confess "Cannot add required argument after optional argument"; - } - } - my %argspec = ( - 'result' => $result, - 'flags' => $flags, - 'name' => $name, - 'description' => $description, - 'result_evaluator' => sub { - my ($arg) = @_; - $$result = $arg; - return 1; - } - ); - push @ARG_SPEC_ARRAY, \%argspec; -} - -######################## Externally usable functions ####################### - -sub registerInternalParameters { # () - # Register console output parameters too, as the output module can't depend - # on this tool. - setFlagOption( - ['show-hidden'], - \$SHOW_HIDDEN, - 'Also show hidden undocumented debug options.', - OPTION_ADDFIRST); - setDownCountingOption( - ['s'], - \$VERBOSITY, - 'Create less verbose output.', - OPTION_ADDFIRST); - setUpCountingOption( - ['v'], - \$VERBOSITY, - 'Create more verbose output.', - OPTION_ADDFIRST); - setFlagOption( - ['h', 'help'], - \$SYNTAX_PAGE, - 'Show this help page.', - OPTION_ADDFIRST); - - # If color use is supported and turned on by default, give option to not use - if ($ANSI_COLORS) { - setOptionHeader(''); - setFlagOption( - ['nocolors'], - \$ANSI_COLORS, - 'Do not use ansi colors in print.', - OPTION_SECRET | OPTION_INVERTEDFLAG); - } -} -sub setShowHidden { # (Bool) - $SHOW_HIDDEN = ($_[0] ? 1 : 0); -} - -############## Utility functions - Not intended for external use ############# - -sub initialize { # () - $VERBOSITY = 3; - $ANSI_COLORS = Yahoo::Vespa::ConsoleOutput::ansiColorsSupported(); - $DESCRIPTION = undef; - $BINARY_NAME = $0; - if ($BINARY_NAME =~ /\/([^\/]+)$/) { - $BINARY_NAME = $1; - } - %OPTION_SPEC = (); - @OPTION_SPEC_ARRAY = (); - @ARG_SPEC_ARRAY = (); - @VALIDATORS = (); - $SYNTAX_PAGE = undef; - $SHOW_HIDDEN = undef; - @ARGUMENTS = undef; -} -sub parseCommandLineArguments { # (ArgumentListRef) - printDebug "Parsing command line arguments\n"; - @ARGUMENTS = @{ $_[0] }; - foreach my $spec (@OPTION_SPEC_ARRAY) { - if (ref($spec) && exists $$spec{'initializer'}) { - my $initsub = $$spec{'initializer'}; - &$initsub(); - } - } - my %eaten_args; - if (!&parseOptions(\%eaten_args)) { - printDebug "Option parsing failed\n"; - return 0; - } - if (!&parseArguments(\%eaten_args)) { - printDebug "Argument parsing failed\n"; - return 0; - } - ConsoleOutput::setVerbosity($VERBOSITY); - ConsoleOutput::setUseAnsiColors($ANSI_COLORS); - return 1; -} -sub writeSyntaxPage { # () - if (defined $DESCRIPTION) { - printResult $DESCRIPTION . "\n\n"; - } - printResult "Usage: " . $BINARY_NAME; - if (scalar keys %OPTION_SPEC > 0) { - printResult " [Options]"; - } - foreach my $arg (@ARG_SPEC_ARRAY) { - if (($$arg{'flags'} & OPTION_REQUIRED) != 0) { - printResult " <" . $$arg{'name'} . ">"; - } else { - printResult " [" . $$arg{'name'} . "]"; - } - } - printResult "\n"; - - if (scalar @ARG_SPEC_ARRAY > 0) { - &writeArgumentSyntax(); - } - if (scalar keys %OPTION_SPEC > 0) { - &writeOptionSyntax(); - } -} -sub setGenericOption { # (ids[], Optionspec) - my ($ids, $spec) = @_; - if (!defined $$spec{'flags'}) { - $$spec{'flags'} = 0; - } - foreach my $id (@$ids) { - if (length $id == 1 && $id =~ /[0-9]/) { - confess "A short option can not be a digit. Reserved so we can parse " - . "-4 as a negative number argument rather than an option 4"; - } - } - foreach my $id (@$ids) { - $OPTION_SPEC{$id} = $spec; - } - if (($$spec{'flags'} & OPTION_ADDFIRST) == 0) { - push @OPTION_SPEC_ARRAY, $spec; - } else { - unshift @OPTION_SPEC_ARRAY, $spec; - } -} -sub parseArguments { # (EatenArgs) - my ($eaten_args) = @_; - my $stopIndex = 10000000; - my $argIndex = 0; - printSpam "Parsing arguments\n"; - for (my $i=0; $i<scalar @ARGUMENTS; ++$i) { - printSpam "Processing arg '$ARGUMENTS[$i]'.\n"; - if ($i <= $stopIndex && $ARGUMENTS[$i] eq '--') { - printSpam "Found --. Further dash prefixed args will be args\n"; - $stopIndex = $i; - } elsif ($i <= $stopIndex && $ARGUMENTS[$i] =~ /^-/) { - printSpam "Option declaration. Ignoring\n"; - } elsif (exists $$eaten_args{$i}) { - printSpam "Already eaten argument. Ignoring\n"; - } elsif ($argIndex < scalar @ARG_SPEC_ARRAY) { - my $spec = $ARG_SPEC_ARRAY[$argIndex]; - my $name = $$spec{'name'}; - if (!&{$$spec{'result_evaluator'}}($ARGUMENTS[$i])) { - printDebug "Failed evaluate result of arg $name. Aborting\n"; - return 0; - } - printSpam "Successful parsing of argument '$name'.\n"; - $$eaten_args{$i} = 1; - ++$argIndex; - } else { - printError "Unhandled argument '$ARGUMENTS[$i]'.\n"; - return 0; - } - } - if ($SYNTAX_PAGE) { # Ignore required arg check if syntax page is to be shown - return 1; - } - for (my $i=$argIndex; $i<scalar @ARG_SPEC_ARRAY; ++$i) { - my $spec = $ARG_SPEC_ARRAY[$i]; - if (($$spec{'flags'} & OPTION_REQUIRED) != 0) { - my $name = $$spec{'name'}; - printError "Argument $name is required but not specified.\n"; - return 0; - } - } - return 1; -} -sub getOptionArguments { # (Count, MinIndex, EatenArgs) - my ($count, $minIndex, $eaten_args) = @_; - my $stopIndex = 10000000; - my @result; - if ($count == 0) { return \@result; } - for (my $i=0; $i<scalar @ARGUMENTS; ++$i) { - printSpam "Processing arg '$ARGUMENTS[$i]'.\n"; - if ($i <= $stopIndex && $ARGUMENTS[$i] eq '--') { - printSpam "Found --. Further dash prefixed args will be args\n"; - $stopIndex = $i; - } elsif ($i <= $stopIndex && $ARGUMENTS[$i] =~ /^-[^0-9]/) { - printSpam "Option declaration. Ignoring\n"; - } elsif (exists $$eaten_args{$i}) { - printSpam "Already eaten argument. Ignoring\n"; - } elsif ($i < $minIndex) { - printSpam "Not eaten, but too low index to be option arg.\n"; - } else { - printSpam "Using argument\n"; - push @result, $ARGUMENTS[$i]; - $$eaten_args{$i} = 1; - if (scalar @result == $count) { - return \@result; - } - } - } - printSpam "Too few option arguments found. Returning undef\n"; - return; -} -sub parseOption { # (Id, EatenArgs, Index) - my ($id, $eaten_args, $index) = @_; - if (!exists $OPTION_SPEC{$id}) { - printError "Unknown option '$id'.\n"; - return 0; - } - my $spec = $OPTION_SPEC{$id}; - my $args = getOptionArguments($$spec{'arg_count'}, $index, $eaten_args); - if (!defined $args) { - printError "Too few arguments for option '$id'.\n"; - return 0; - } - printSpam, "Found " . (scalar @$args) . " args\n"; - if (!&{$$spec{'result_evaluator'}}($id, $args)) { - printDebug "Failed evaluate result of option '$id'. Aborting\n"; - return 0; - } - printSpam "Successful parsing of option '$id'.\n"; - return 1; -} -sub parseOptions { # (EatenArgs) - my ($eaten_args) = @_; - for (my $i=0; $i<scalar @ARGUMENTS; ++$i) { - if ($ARGUMENTS[$i] =~ /^--(.+)$/) { - my $id = $1; - printSpam "Parsing long option '$id'.\n"; - if (!&parseOption($id, $eaten_args, $i)) { - return 0; - } - } elsif ($ARGUMENTS[$i] =~ /^-([^0-9].*)$/) { - my $shortids = $1; - while ($shortids =~ /^(.)(.*)$/) { - my ($id, $rest) = ($1, $2); - printSpam "Parsing short option '$id'.\n"; - if (!&parseOption($id, $eaten_args, $i)) { - return 0; - } - $shortids = $rest; - } - } - } - printSpam "Successful parsing of all options.\n"; - return 1; -} -sub writeArgumentSyntax { # () - printResult "\nArguments:\n"; - my $max_name_length = &getMaxNameLength(); - if ($max_name_length > 30) { $max_name_length = 30; } - foreach my $spec (@ARG_SPEC_ARRAY) { - &writeArgumentName($$spec{'name'}, $max_name_length); - &writeOptionDescription($spec, $max_name_length + 3); - } -} -sub getMaxNameLength { # () - my $max = 0; - foreach my $spec (@ARG_SPEC_ARRAY) { - my $len = 1 + length $$spec{'name'}; - if ($len > $max) { $max = $len; } - } - return $max; -} -sub writeArgumentName { # (Name, MaxNameLength) - my ($name, $maxnamelen) = @_; - printResult " $name"; - my $totalLength = 1 + length $name; - if ($totalLength <= $maxnamelen) { - for (my $i=$totalLength; $i<$maxnamelen; ++$i) { - printResult ' '; - } - } else { - printResult "\n"; - for (my $i=0; $i<$maxnamelen; ++$i) { - printResult ' '; - } - } - printResult " : "; -} -sub writeOptionSyntax { # () - printResult "\nOptions:\n"; - my $max_id_length = &getMaxIdLength(); - if ($max_id_length > 30) { $max_id_length = 30; } - my $cachedHeader; - foreach my $spec (@OPTION_SPEC_ARRAY) { - if (ref($spec) eq 'HASH') { - my $flags = $$spec{'flags'}; - if ($SHOW_HIDDEN || ($flags & OPTION_SECRET) == 0) { - if (defined $cachedHeader) { - printResult "\n"; - if ($cachedHeader ne '') { - &writeOptionHeader($cachedHeader); - } - $cachedHeader = undef; - } - &writeOptionId($spec, $max_id_length); - &writeOptionDescription($spec, $max_id_length + 3); - } - } else { - $cachedHeader = $spec; - } - } -} -sub getMaxIdLength { # () - my $max = 0; - foreach my $spec (@OPTION_SPEC_ARRAY) { - if (!ref($spec)) { next; } # Ignore option headers - my $size = 0; - foreach my $id (@{ $$spec{'ids'} }) { - my $len = length $id; - if ($len == 1) { - $size += 3; - } else { - $size += 3 + $len; - } - } - if ($size > $max) { $max = $size; } - } - return $max; -} -sub writeOptionId { # (Spec, MaxNameLength) - my ($spec, $maxidlen) = @_; - my $totalLength = 0; - foreach my $id (@{ $$spec{'ids'} }) { - my $len = length $id; - if ($len == 1) { - printResult " -" . $id; - $totalLength += 3; - } else { - printResult " --" . $id; - $totalLength += 3 + $len; - } - } - if ($totalLength <= $maxidlen) { - for (my $i=$totalLength; $i<$maxidlen; ++$i) { - printResult ' '; - } - } else { - printResult "\n"; - for (my $i=0; $i<$maxidlen; ++$i) { - printResult ' '; - } - } - printResult " : "; -} -sub writeOptionDescription { # (Spec, MaxNameLength) - my ($spec, $maxidlen) = @_; - my $width = ConsoleOutput::getTerminalWidth() - $maxidlen; - my $desc = $$spec{'description'}; - my $min = int ($width / 2); - while (length $desc > $width) { - if ($desc =~ /^(.{$min,$width}) (.*)$/s) { - my ($first, $rest) = ($1, $2); - printResult $first . "\n"; - for (my $i=0; $i<$maxidlen; ++$i) { - printResult ' '; - } - $desc = $rest; - } else { - last; - } - } - printResult $desc . "\n"; -} -sub writeOptionHeader { # (Description) - my ($desc) = @_; - my $width = ConsoleOutput::getTerminalWidth(); - my $min = 2 * $width / 3; - while (length $desc > $width) { - if ($desc =~ /^(.{$min,$width}) (.*)$/s) { - my ($first, $rest) = ($1, $2); - printResult $first . "\n"; - $desc = $rest; - } else { - last; - } - } - printResult $desc . "\n"; -} -sub validHost { # (Hostname) - my ($host) = @_; - if ($host !~ /^[a-zA-Z][-_a-zA-Z0-9\.]*$/) { - return 0; - } - if (system("host $host >/dev/null 2>/dev/null") != 0) { - return 0; - } - return 1; -} diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/Bin/GetClusterState.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/Bin/GetClusterState.pm deleted file mode 100644 index afb36a418ae..00000000000 --- a/vespaclient/src/perl/lib/Yahoo/Vespa/Bin/GetClusterState.pm +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package Yahoo::Vespa::Bin::GetClusterState; - -use strict; -use warnings; -use Yahoo::Vespa::ArgParser; -use Yahoo::Vespa::ClusterController; -use Yahoo::Vespa::ConsoleOutput; -use Yahoo::Vespa::ContentNodeSelection; -use Yahoo::Vespa::Utils; -use Yahoo::Vespa::VespaModel; - -BEGIN { - use base 'Exporter'; - our @EXPORT = qw( - getClusterState - ); -} - -my %cluster_states; - -return &init(); - -sub init { - %cluster_states = (); - return 1; -} - -# Run the get node state tool -sub getClusterState { # (Command line arguments) - my ($argsref) = @_; - &handleCommandLine($argsref); - detectClusterController(); - &showSettings(); - &showNodeStates(); -} - -# Parse command line arguments -sub handleCommandLine { # (Command line arguments) - my ($args) = @_; - my $description = <<EOS; -Get the cluster state of a given cluster. - -EOS - $description =~ s/(\S)\n(\S)/$1 $2/gs; - chomp $description; - - setProgramDescription($description); - Yahoo::Vespa::ContentNodeSelection::registerCommandLineArguments( - NO_LOCALHOST_CONSTRAINT | CLUSTER_ONLY_LIMITATION); - Yahoo::Vespa::VespaModel::registerCommandLineArguments(); - handleCommandLineArguments($args); -} - -# Show what settings this tool is running with (if verbosity is high enough) -sub showSettings { # () - &Yahoo::Vespa::ClusterController::showSettings(); -} - -# Print all state we want to show for this request -sub showNodeStates { # () - - Yahoo::Vespa::ContentNodeSelection::visit(\&showNodeStateForNode); -} - -# Get the node state from cluster controller, unless already cached -sub getStateForNode { # (Type, Index, Cluster) - my ($type, $index, $cluster) = @_; - if (!exists $cluster_states{$cluster}) { - my $state = getContentClusterState($cluster); - $cluster_states{$cluster} = $state; - if ($state->globalState eq "up") { - printResult "\nCluster $cluster:\n"; - } else { - printResult "\nCluster $cluster is " . COLOR_ERR - . $state->globalState . COLOR_RESET - . ". Too few nodes available.\n"; - } - } - return $cluster_states{$cluster}->$type->{$index}; -} - -# Print all states for a given node -sub showNodeStateForNode { # (Service, Index, NodeState, Model, ClusterName) - my ($info) = @_; - my ($cluster, $type, $index) = ( - $$info{'cluster'}, $$info{'type'}, $$info{'index'}); - my $nodestate = &getStateForNode($type, $index, $cluster); - defined $nodestate or confess "No nodestate for $type $index $cluster"; - my $generated = $nodestate->generated; - my $id = $cluster . "/"; - if (defined $nodestate->group) { - $id .= $nodestate->group; - } - my $msg = "$cluster/$type/$index: "; - if ($generated->state ne 'up') { - $msg .= COLOR_ERR; - } - $msg .= $generated->state; - if ($generated->state ne 'up') { - $msg .= COLOR_RESET; - } - # TODO: Make the Cluster Controller always populate the reason for the - # generated state. Until then we'll avoid printing it to avoid confusion. - # Use vespa-get-node-state to see the reasons on generated, user, and unit. - # - # if (length $generated->reason > 0) { - # $msg .= ': ' . $generated->reason; - # } - printResult $msg . "\n"; -} - -# ClusterState(Version: 7, Cluster state: Up, Distribution bits: 1) { -# Group 0: mygroup. 1 node [0] { -# All nodes in group up and available. -# } -# } - -# ClusterState(Version: 7, Cluster state: Up, Distribution bits: 1) { -# Group 0: mygroup. 1 node [0] { -# storage.0: Retired: foo -# } -# } diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/Bin/GetNodeState.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/Bin/GetNodeState.pm deleted file mode 100644 index 35b3f49649e..00000000000 --- a/vespaclient/src/perl/lib/Yahoo/Vespa/Bin/GetNodeState.pm +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package Yahoo::Vespa::Bin::GetNodeState; - -use strict; -use warnings; -use Yahoo::Vespa::ArgParser; -use Yahoo::Vespa::ClusterController; -use Yahoo::Vespa::ConsoleOutput; -use Yahoo::Vespa::ContentNodeSelection; -use Yahoo::Vespa::Utils; - -BEGIN { - use base 'Exporter'; - our @EXPORT = qw( - getNodeState - ); -} - -our $resultdesc; -our %cluster_states; - -return 1; - -# Run the get node state tool -sub getNodeState { # (Command line arguments) - my ($argsref) = @_; - &handleCommandLine($argsref); - detectClusterController(); - &showSettings(); - &showNodeStates(); -} - -# Parse command line arguments -sub handleCommandLine { # (Command line arguments) - my ($args) = @_; - $resultdesc = <<EOS; -Shows the various states of one or more nodes in a Vespa Storage cluster. -There exist three different type of node states. They are: - - Unit state - The state of the node seen from the cluster controller. - User state - The state we want the node to be in. By default up. Can be - set by administrators or by cluster controller when it - detects nodes that are behaving badly. - Generated state - The state of a given node in the current cluster state. - This is the state all the other nodes know about. This - state is a product of the other two states and cluster - controller logic to keep the cluster stable. -EOS - $resultdesc =~ s/\s*\n(\S.)/ $1/gs; - chomp $resultdesc; - my $description = <<EOS; -Retrieve the state of one or more storage services from the fleet controller. -Will list the state of the locally running services, possibly restricted to -less by options. - -$resultdesc - -EOS - $description =~ s/(\S)\n(\S)/$1 $2/gs; - chomp $description; - - setProgramDescription($description); - Yahoo::Vespa::ContentNodeSelection::registerCommandLineArguments(); - Yahoo::Vespa::VespaModel::registerCommandLineArguments(); - handleCommandLineArguments($args); -} - -# Show what settings this tool is running with (if verbosity is high enough) -sub showSettings { # () - &Yahoo::Vespa::ClusterController::showSettings(); - &Yahoo::Vespa::ContentNodeSelection::showSettings(); -} - -# Print all state we want to show for this request -sub showNodeStates { # () - printInfo $resultdesc . "\n"; - Yahoo::Vespa::ContentNodeSelection::visit(\&showNodeStateForNode); -} - -# Get the node state from cluster controller, unless already cached -sub getStateForNode { # (Type, Index, Cluster) - my ($type, $index, $cluster) = @_; - if (!exists $cluster_states{$cluster}) { - $cluster_states{$cluster} = getContentClusterState($cluster); - } - return $cluster_states{$cluster}->$type->{$index}; -} - -# Print all states for a given node -sub showNodeStateForNode { # (Service, Index, NodeState, Model, ClusterName) - my ($info) = @_; - my ($cluster, $type, $index) = ( - $$info{'cluster'}, $$info{'type'}, $$info{'index'}); - printResult "\n$cluster/$type.$index:\n"; - my $nodestate = &getStateForNode($type, $index, $cluster); - printState('Unit', $nodestate->unit); - printState('Generated', $nodestate->generated); - printState('User', $nodestate->user); -} - -# Print the value of a single state type for a node -sub printState { # (State name, State) - my ($name, $state) = @_; - if (!defined $state) { - printResult $name . ": UNKNOWN\n"; - } else { - my $msg = $name . ": "; - if ($state->state ne 'up') { - $msg .= COLOR_ERR; - } - $msg .= $state->state; - if ($state->state ne 'up') { - $msg .= COLOR_RESET; - } - $msg .= ": " . $state->reason . "\n"; - printResult $msg; - } -} diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/Bin/SetNodeState.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/Bin/SetNodeState.pm deleted file mode 100644 index b1daebff03c..00000000000 --- a/vespaclient/src/perl/lib/Yahoo/Vespa/Bin/SetNodeState.pm +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package Yahoo::Vespa::Bin::SetNodeState; - -use strict; -use warnings; -use Yahoo::Vespa::ClusterController; -use Yahoo::Vespa::ConsoleOutput; -use Yahoo::Vespa::ContentNodeSelection; -use Yahoo::Vespa::ArgParser; -use Yahoo::Vespa::Utils; - -BEGIN { - use base 'Exporter'; - our @EXPORT = qw( - setNodeState - ); -} - -our $wanted_state; -our $wanted_state_description; -our $nodes_attempted_set; -our $success; -our $no_wait; -our $safe_mode; - -return 1; - -# Run the set node state tool -sub setNodeState { # (Command line arguments) - my ($argsref) = @_; - &handleCommandLine($argsref); - detectClusterController(); - &showSettings(); - &maybeRequireClusterSelection(); - &execute(); -} - -# Parse command line arguments -sub handleCommandLine { # (Command line arguments) - my ($args) = @_; - my $description = <<EOS; -Set the user state of a node. This will set the generated state to the user -state if the user state is "better" than the generated state that would have -been created if the user state was up. For instance, a node that is currently -in initializing state can be forced into down state, while a node that is -currently down can not be forced into retired state, but can be forced into -maintenance state. -EOS - $description =~ s/(\S)\n(\S)/$1 $2/gs; - chomp $description; - - setProgramDescription($description); - - setArgument(\$wanted_state, "Wanted State", - "User state to set. This must be one of " - . "up, down, maintenance or retired.", - OPTION_REQUIRED); - setArgument(\$wanted_state_description, "Description", - "Give a reason for why you are altering the user state, which " - . "will show up in various admin tools. (Use double quotes to " - . "give a reason with whitespace in it)"); - - setOptionHeader("Options related to operation visibility:"); - setFlagOption(['n', 'no-wait'], \$no_wait, "Do not wait for node state " - . "changes to be visible in the cluster before returning."); - setFlagOption(['a', 'safe'], \$safe_mode, "Only carries out state changes " - . "if deemed safe by the cluster controller. For maintenance mode, " - . "will also set the distributor with the same distribution key " - . "to down atomically as part of the same state change. For up " - . "mode, transition is only allowed if the content node reports " - . "itself as up. Only supported for type storage."); - - Yahoo::Vespa::ContentNodeSelection::registerCommandLineArguments(); - Yahoo::Vespa::VespaModel::registerCommandLineArguments(); - handleCommandLineArguments($args); - - if (!Yahoo::Vespa::ContentNodeSelection::validateCommandLineArguments( - $wanted_state)) { - exitApplication(1); - } -} - -# Show what settings this tool is running with (if verbosity is high enough) -sub showSettings { # () - Yahoo::Vespa::ClusterController::showSettings(); -} - -sub maybeRequireClusterSelection -{ - return if Yahoo::Vespa::ContentNodeSelection::hasClusterSelection(); - my %clusters; - VespaModel::visitServices(sub { - my ($info) = @_; - if ($$info{'type'} =~ /^(?:distributor|storage|storagenode)$/ ) { - $clusters{$$info{'cluster'}} = 1; - } - }); - my $clusterCount = scalar keys %clusters; - if ($clusterCount > 1) { - printWarning "More than one cluster is present but no cluster is selected\n"; - exitApplication(1); - } -} - -# Sets the node state -sub execute { # () - $success = 1; - $nodes_attempted_set = 0; - Yahoo::Vespa::ContentNodeSelection::visit(\&setNodeStateForNode); - if ($nodes_attempted_set == 0) { - printWarning("Attempted setting of user state for no nodes"); - exitApplication(1); - } - if (!$success) { - exitApplication(1); - } -} - -sub setNodeStateForNode { - my ($info) = @_; - my ($cluster, $type, $index) = ( - $$info{'cluster'}, $$info{'type'}, $$info{'index'}); - $success &&= setNodeUserState($cluster, $type, $index, $wanted_state, - $wanted_state_description, $no_wait, $safe_mode); - ++$nodes_attempted_set; -} diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/ClusterController.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/ClusterController.pm deleted file mode 100644 index 9b594d780fe..00000000000 --- a/vespaclient/src/perl/lib/Yahoo/Vespa/ClusterController.pm +++ /dev/null @@ -1,279 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# -# Handles Rest API requests to State Rest API in cluster controller, making -# wanted data programmatically available. -# -package Yahoo::Vespa::ClusterController; - -use strict; -use warnings; -use Class::Struct; -use Yahoo::Vespa::ArgParser; -use Yahoo::Vespa::ClusterState; -use Yahoo::Vespa::ConsoleOutput; -use Yahoo::Vespa::Http; -use Yahoo::Vespa::Json; -use Yahoo::Vespa::Utils; -use Yahoo::Vespa::VespaModel; - -BEGIN { # - Exports and aliases for the module - use base 'Exporter'; - our $VERSION = '1.0'; - our @EXPORT = qw( - detectClusterController - getContentClusterState - setNodeUserState - ); # Exported unless specifically left out by user - # Alias namespaces - *VespaModel:: = *Yahoo::Vespa::VespaModel:: ; - *Http:: = *Yahoo::Vespa::Http:: ; - *Json:: = *Yahoo::Vespa::Json:: ; -} - -struct( ClusterController => { - index => '$', # Logical index of the cluster controller - host => '$', # Host on which cluster controller runs - port => '$' # Port where cluster controller is available -}); - -my %CACHED_CLUSTER_STATES; -my @CLUSTER_CONTROLLERS; - -return &init(); - -########################## Default exported functions ######################## - -sub init { - %CACHED_CLUSTER_STATES = (); - @CLUSTER_CONTROLLERS = (); - return 1; -} - -sub detectClusterController { # () - if (scalar @CLUSTER_CONTROLLERS == 0) { - use Yahoo::Vespa::VespaModel; - printDebug "Attempting to auto-detect cluster controller location\n"; - my $sockets = VespaModel::getSocketForService( - type => 'container-clustercontroller', tag => 'state'); - foreach my $sock (sort { $a->{'index'} <=> $b->{'index'} } @$sockets) { - my $cc = new ClusterController; - $cc->index($sock->{'index'}); - $cc->host($sock->{'host'}); - $cc->port($sock->{'port'}); - push @CLUSTER_CONTROLLERS, $cc; - } - if (scalar @$sockets == 0) { - my $oldVal = enableAutomaticLineBreaks(0); - printSpam dumpStructure(VespaModel::get()); - enableAutomaticLineBreaks($oldVal); - printError "Failed to detect cluster controller to talk to. " - . "Resolve issue that failed automatic detection or " - . "provide cluster controller socket through command " - . "line options. (See --help)\n"; - exitApplication(1); - } - &showSettings(); - printSpam "Content of vespa model inspected to find cluster " - . "controller:\n"; - my $oldVal = enableAutomaticLineBreaks(0); - printSpam dumpStructure(VespaModel::get()); - enableAutomaticLineBreaks($oldVal); - } -} -sub setNodeUserState { # (ClusterName, NodeType, Index, State, Reason, NoWait, SafeMode) - my ($cluster, $service, $index, $state, $reason, $no_wait, $safe_mode) = @_; - my @params = (); - my @headers = ( - 'Content-Type' => 'application/json' - ); - $state =~ tr/A-Z/a-z/; - $state =~ /(?:up|down|maintenance|retired)$/ - or confess "Invalid state '$state' attempted set.\n"; - if (!defined $reason) { - $reason = ""; - } - my $request = { - "state" => { - "user" => { - "state" => $state, - "reason" => $reason - } - } - }; - if ($no_wait) { - $request->{'response-wait'} = 'no-wait'; - } - if ($safe_mode) { - $request->{'condition'} = 'safe'; - } - my $content = Json::encode($request); - - my $path = &getPathToNode($cluster, $service, $index); - my %response = &requestCC('POST', $path, \@params, $content, \@headers); - if (defined $response{'all'}) { printSpam $response{'all'}; } - printDebug $response{'code'} . " " . $response{'status'} . "\n"; - printInfo exists($response{'content'}) ? $response{'content'} : ''; - if ($response{'code'} >= 200 && $response{'code'} < 300) { - printResult "$response{'status'}\n"; - return 1 - } else { - printWarning "Failed to set node state for node " - . "$cluster/$service/$index: " - . "$response{'code'} $response{'status'}\n"; - return 0 - } -} -sub getContentClusterState { # (ClusterName) -> ClusterState - my ($cluster) = @_; - if (!exists $CACHED_CLUSTER_STATES{$cluster}) { - $CACHED_CLUSTER_STATES{$cluster} = &fetchContentClusterState($cluster); - } - return $CACHED_CLUSTER_STATES{$cluster}; -} - -######################## Externally usable functions ####################### - -sub getClusterControllers { # () - return \@CLUSTER_CONTROLLERS; -} -sub showSettings { # () - printDebug "Cluster controllers:\n"; - foreach my $cc (@CLUSTER_CONTROLLERS) { - printDebug " " . $cc->index . ": " - . $cc->host . ":" . $cc->port . "\n"; - } -} - -############## Utility functions - Not intended for external use ############# - -sub fetchContentClusterState { # (ClusterName) -> ClusterState - my ($cluster) = @_; - my @params = ( - 'recursive' => 'true' - ); - my %response = &getCC("/cluster/v2/$cluster/", \@params); - if ($response{'code'} != 200) { - printError "Failed to fetch cluster state of content cluster " - . "'$cluster':\n" . $response{'all'} . "\n"; - exitApplication(1); - } - my $json = Json::parse($response{'content'}); - my $result = new ClusterState; - &fillInGlobalState($cluster, $result, $json); - &fillInNodes($result, 'distributor', - &getJsonValue($json, ['service', 'distributor', 'node'])); - &fillInNodes($result, 'storage', - &getJsonValue($json, ['service', 'storage', 'node'])); - return $result; -} -sub fillInGlobalState { # (ClusterName, StateToFillIn, JsonToParse) - my ($cluster, $state, $json) = @_; - my $e = &getJsonValue($json, ['state', 'generated', 'state']); - if (defined $e) { - $state->globalState($e); - if (!Yahoo::Vespa::ClusterState::legalState($state->globalState())) { - printWarning "Illegal global cluster state $e found.\n"; - } - } else { - printDebug dumpStructure($json) . "\n"; - printWarning "Found no global cluster state\n"; - } -} -sub getPathToNode { # (ClusterName, NodeType, Index) - my ($cluster, $service, $index) = @_; - return "/cluster/v2/$cluster/$service/$index"; -} -sub listContentClusters { # () -> (ContentClusterName, ...) - my %result = &getCC("/cluster/v2/"); - if ($result{'code'} != 200) { - printError "Failed to fetch list of content clusters:\n" - . $result{'all'} . "\n"; - exitApplication(1); - } - my $json = Json::parse($result{'content'}); - return keys %{ $json->{'cluster'} }; -} -sub fillInNodes { # (StateToFillIn, ServiceType, json) - my ($state, $service, $json) = @_; - foreach my $index (%{ $json }) { - my $node = new Node; - &parseNode($node, $json->{$index}); - $state->$service($index, $node); - } -} -sub parseNode { # (StateToFillIn, JsonToParse) - my ($node, $json) = @_; - my $group = &getJsonValue($json, ['attributes', 'hierarchical-group']); - if (defined $group && $group =~ /^[^\.]*\.(.*)$/) { - $node->group($1); - } - parseState($node, $json, 'unit'); - parseState($node, $json, 'generated'); - parseState($node, $json, 'user'); - my $partitions = $json->{'partition'}; - if (defined $partitions) { - foreach my $index (%{ $json->{'partition'} }) { - my $partition = new Partition; - parsePartition($partition, $json->{'partition'}->{$index}); - $node->partition($index, $partition); - } - } -} -sub parsePartition { # (StateToFillIn, JsonToParse) - my ($partition, $json) = @_; - my $buckets = &getJsonValue($json, ['metrics', 'bucket-count']); - my $doccount = &getJsonValue($json, ['metrics', 'unique-document-count']); - my $size = &getJsonValue($json, ['metrics', 'unique-document-total-size']); - $partition->bucketcount($buckets); - $partition->doccount($doccount); - $partition->totaldocsize($size); -} -sub parseState { # (StateToFillIn, JsonToParse, StateType) - my ($node, $json, $type) = @_; - my $value = &getJsonValue($json, ['state', $type, 'state']); - my $reason = &getJsonValue($json, ['state', $type, 'reason']); - if (defined $value) { - my $state = new State; - $state->state($value); - $state->reason($reason); - $state->source($type); - $node->$type($state); - } -} -sub getJsonValue { # (json, [ keys ]) - my ($json, $keys) = @_; - foreach my $key (@$keys) { - if (!defined $json) { return; } - $json = $json->{$key}; - } - return $json; -} -sub getCC { # (Path, Params, Headers) -> Response - my ($path, $params, $headers) = @_; - return requestCC('GET', $path, $params, undef, $headers); -} -sub requestCC { # (Type, Path, Params, Content, Headers) -> Response - my ($type, $path, $params, $content, $headers) = @_; - my %response; - foreach my $cc (@CLUSTER_CONTROLLERS) { - %response = Http::request($type, $cc->host, $cc->port, $path, - $params, $content, $headers); - if ($response{'code'} == 200) { - return %response; - } elsif ($response{'code'} == 307) { - my %headers = $response{'headers'}; - my $masterlocation = $headers{'Location'}; - if (defined $masterlocation) { - if ($masterlocation =~ /http:\/\/([^\/:]+):(\d+)\//) { - my ($host, $port) = ($1, $2); - return Http::request($type, $host, $port, $path, - $params, $content, $headers); - } else { - printError("Unhandled relocaiton URI '$masterlocation'."); - exitApplication(1); - } - } - } - } - return %response; -} diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/ClusterState.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/ClusterState.pm deleted file mode 100644 index 3c1b9acf4d1..00000000000 --- a/vespaclient/src/perl/lib/Yahoo/Vespa/ClusterState.pm +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# -# Defines structs to represent a cluster state -# -package Yahoo::Vespa::ClusterState; - -use strict; -use warnings; -use Class::Struct; - -struct( ClusterState => { - globalState => '$', # A state primitive - distributor => '%', # Index to Node map - storage => '%' # Index to Node map -}); - -struct( Node => { - group => '$', # Hierarchical group node belongs to - unit => 'State', - generated => 'State', - user => 'State', - partition => '%' -}); - -struct( Partition => { - generated => 'State', - bucketcount => '$', - doccount => '$', - totaldocsize => '$' -}); - -struct( State => { - state => '$', # A state primitive - reason => '$', # Textual reason for it to be set. - timestamp => '$', # Timestamp of the time it got set. - source => '$' # What type of state is it (unit/generated/user) -}); - -return 1; - -sub legalState { # (State) -> Bool - my ($state) = @_; - return ($state =~ /^(up|down|maintenance|retired|stopping|initializing)$/); -} - diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/ConsoleOutput.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/ConsoleOutput.pm deleted file mode 100644 index 60db964e7f7..00000000000 --- a/vespaclient/src/perl/lib/Yahoo/Vespa/ConsoleOutput.pm +++ /dev/null @@ -1,331 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# -# Output handler -# -# Intentions: -# - Make it easy for unit tests to redirect output. -# - Allow programmers to add all sorts of debug information into tools usable -# for debugging, while hiding it by default for real users. -# - Allow generic functionality that can be reused by all. For instance color -# coding of very important information. -# -# Ideas for improvement: -# - Could possibly detect terminal width and do proper line breaking of long -# lines -# -# A note about colors: -# - This module will detect if terminal supports colors. If not, it will not -# print any. (Color support can be turned off by giving --nocolors argument -# through argument parser, by setting a TERM value that does not support -# colors or programmatically call setUseAnsiColors(0). -# - Currently only red and grey are used in addition to default. These colors -# should work well for both light and dark backgrounds. -# - -package Yahoo::Vespa::ConsoleOutput; - -use strict; -use warnings; -use Yahoo::Vespa::Utils; - -BEGIN { # - Define exports for modul - use base 'Exporter'; - our @EXPORT = qw( - printResult printError printWarning printInfo printDebug printSpam - enableAutomaticLineBreaks - COLOR_RESET COLOR_WARN COLOR_ERR COLOR_ANON - ); - our @EXPORT_OK = qw( - getTerminalWidth getVerbosity usingAnsiColors ansiColorsSupported - setVerbosity - ); -} - -my %TYPES = ( - 'result' => 0, # Output from a tool. Expected when app runs successfully. - 'error' => 1, # Error found, typically aborting the script with a failure. - 'warning' => 2, # An issue that may or may not cause the program to fail. - 'info' => 3, # Useful information to get from the script. - 'debug' => 4, # Debug information useful to debug script or to see - # internals of what is happening. - 'spam' => 5, # Spammy information used when large amounts of details is - # wanted. Typically to debug some failure. -); -my $VERBOSITY; # Current verbosity level -my $ANSI_COLORS_SUPPORTED; # True if terminal supports colors -my $ANSI_COLORS; # True if we want to use colors (and support it) -my %ATTRIBUTE_PREFIX; # Ansi escape prefixes for verbosity levels -my %ATTRIBUTE_POSTFIX; # Ansi escape postfixes for verbosity levels -my %OUTPUT_STREAM; # Where to write different verbosity levels (stdout|stderr) -my $TERMINAL_WIDTH; # With of terminal in columns -my $COLUMN_POSITION; # Current index of cursor in terminal -my $ENABLE_AUTO_LINE_BREAKS; - -use constant COLOR_RESET => "\e[0m"; -use constant COLOR_ERR => "\e[91m"; -use constant COLOR_WARN => "\e[93m"; -use constant COLOR_ANON => "\e[90m"; - -&initialize(*STDOUT, *STDERR); - -return 1; - -########################## Default exported functions ######################## - -sub printResult { # (Output...) - printAtLevel('result', @_); -} -sub printError { # (Output...) - printAtLevel('error', @_); -} -sub printWarning { # (Output...) - printAtLevel('warning', @_); -} -sub printInfo { # (Output...) - printAtLevel('info', @_); -} -sub printDebug { # (Output...) - printAtLevel('debug', @_); -} -sub printSpam { # (Output...) - printAtLevel('spam', @_); -} -sub enableAutomaticLineBreaks { # (Bool) -> (OldValue) - my $oldval = $ENABLE_AUTO_LINE_BREAKS; - $ENABLE_AUTO_LINE_BREAKS = ($_[0] ? 1 : 0); - return $oldval; -} - -######################## Optionally exported functions ####################### - -sub getTerminalWidth { # () -> ColumnCount - # May be undefined if someone prints before initialized - return (defined $TERMINAL_WIDTH ? $TERMINAL_WIDTH : 80); -} -sub getVerbosity { # () -> VerbosityLevel - return $VERBOSITY; -} -sub usingAnsiColors { # () -> Bool - return $ANSI_COLORS; -} -sub ansiColorsSupported { # () -> Bool - return $ANSI_COLORS_SUPPORTED; -} -sub setVerbosity { # (VerbosityLevel) - $VERBOSITY = $_[0]; -} - -################## Functions for unit tests to mock internals ################ - -sub setTerminalWidth { # (ColumnCount) - $TERMINAL_WIDTH = $_[0]; -} -sub setUseAnsiColors { # (Bool) - if ($ANSI_COLORS_SUPPORTED && $_[0]) { - $ANSI_COLORS = 1; - } else { - $ANSI_COLORS = 0; - } -} - -############## Utility functions - Not intended for external use ############# - -sub initialize { # () - my ($stdout, $stderr, $use_colors_by_default) = @_; - if (!defined $VERBOSITY) { - $VERBOSITY = &getDefaultVerbosity(); - } - $COLUMN_POSITION = 0; - $ENABLE_AUTO_LINE_BREAKS = 1; - %ATTRIBUTE_PREFIX = map { $_ => '' } keys %TYPES; - %ATTRIBUTE_POSTFIX = map { $_ => '' } keys %TYPES; - &setAttribute('error', COLOR_ERR, COLOR_RESET); - &setAttribute('warning', COLOR_WARN, COLOR_RESET); - &setAttribute('debug', COLOR_ANON, COLOR_RESET); - &setAttribute('spam', COLOR_ANON, COLOR_RESET); - %OUTPUT_STREAM = map { $_ => $stdout } keys %TYPES; - $OUTPUT_STREAM{'error'} = $stderr; - $OUTPUT_STREAM{'warning'} = $stderr; - if (defined $use_colors_by_default) { - $ANSI_COLORS_SUPPORTED = $use_colors_by_default; - $ANSI_COLORS = $ANSI_COLORS_SUPPORTED; - } else { - &detectTerminalColorSupport(); - } - if (!defined $TERMINAL_WIDTH) { - $TERMINAL_WIDTH = &detectTerminalWidth(); - } -} -sub setAttribute { # (type, prefox, postfix) - my ($type, $prefix, $postfix) = @_; - $ATTRIBUTE_PREFIX{$type} = $prefix; - $ATTRIBUTE_POSTFIX{$type} = $postfix; -} -sub stripAnsiEscapes { # (Line) -> (StrippedLine) - $_[0] =~ s/\e\[[^m]*m//g; - return $_[0]; -} -sub getDefaultVerbosity { # () -> VerbosityLevel - # We can not print at correct verbosity levels before argument parsing has - # completed. We try some simple arg parsing here assuming default options - # used to set verbosity, such that we likely guess correctly, allowing - # correct verbosity from the start. - my $default = 3; - foreach my $arg (@ARGV) { - if ($arg eq '--') { return $default; } - if ($arg =~ /^-([^-]+)/) { - my $optstring = $1; - while ($optstring =~ /^(.)(.*)$/) { - my $char = $1; - $optstring = $2; - if ($char eq 'v') { - ++$default; - } - if ($char eq 's') { - if ($default > 0) { - --$default; - } - } - } - } - } - return $default; -} -sub detectTerminalWidth { #() -> ColumnCount - my $cols = &checkConsoleFeature('cols'); - if (!defined $cols) { - printDebug "Assuming terminal width of 80.\n"; - return 80; - } - if ($cols =~ /^\d+$/ && $cols > 10 && $cols < 500) { - printDebug "Detected terminal width of $cols.\n"; - return $cols; - } else { - printDebug "Unexpected terminal width of '$cols' given. " - . "Assuming size of 80.\n"; - return 80; - } -} -sub detectTerminalColorSupport { # () -> Bool - my $colorcount = &checkConsoleFeature('colors'); - if (!defined $colorcount) { - $ANSI_COLORS_SUPPORTED = 0; - printDebug "Assuming no color support.\n"; - return 0; - } - if ($colorcount =~ /^\d+$/ && $colorcount >= 8) { - $ANSI_COLORS_SUPPORTED = 1; - if (!defined $ANSI_COLORS) { - $ANSI_COLORS = $ANSI_COLORS_SUPPORTED; - } - printDebug "Color support detected.\n"; - return 1; - } -} -sub checkConsoleFeature { # (Feature) -> Bool - my ($feature) = @_; - # Unit tests must mock. Can't depend on TERM being set. - assertNotUnitTest(); - if (!exists $ENV{'TERM'}) { - printDebug "Terminal not set. Unknown.\n"; - return; - } - if (-f '/usr/bin/tput') { - my ($fh, $result); - if (open ($fh, "tput $feature 2>/dev/null |")) { - $result = <$fh>; - close $fh; - } else { - printDebug "Failed to open tput pipe.\n"; - return; - } - if ($? != 0) { - printDebug "Failed tput call to detect feature $feature $!\n"; - return; - } - chomp $result; - #printSpam "Console feature $feature: '$result'\n"; - return $result; - } else { - printDebug "No tput binary. Dont know how to detect feature.\n"; - return; - } -} -sub printAtLevel { # (Level, Output...) - # Prints an array of data that may contain newlines - my $level = shift @_; - exists $TYPES{$level} or confess "Unknown print level '$level'."; - if ($TYPES{$level} > $VERBOSITY) { - return; - } - my $buffer = ''; - my $width = &getTerminalWidth(); - foreach my $printable (@_) { - my @lines = split(/\n/, $printable, -1); - my $current = 0; - for (my $i=0; $i < scalar @lines; ++$i) { - if ($i != 0) { - $buffer .= "\n"; - $COLUMN_POSITION = 0; - } - my $last = ($i + 1 == scalar @lines); - printLineAtLevel($level, $lines[$i], \$buffer, $last); - } - } - my $stream = $OUTPUT_STREAM{$level}; - print $stream $buffer; -} -sub printLineAtLevel { # (Level, Line, Buffer, Last) - # Prints a single line, which might still have to be broken into multiple - # lines - my ($level, $data, $buffer, $last) = @_; - if (!$ANSI_COLORS) { - $data = &stripAnsiEscapes($data); - } - my $width = &getTerminalWidth(); - while (1) { - my $remaining = $width - $COLUMN_POSITION; - if (&prefixLineWithLevel($level)) { - $remaining -= (2 + length $level); - } - if ($ENABLE_AUTO_LINE_BREAKS && $remaining < length $data) { - my $min = int (2 * $width / 3) - $COLUMN_POSITION; - if ($min < 1) { $min = 1; } - if ($data =~ /^(.{$min,$remaining}) (.*?)$/s) { - my ($first, $rest) = ($1, $2); - &printLinePartAtLevel($level, $first, $buffer); - $$buffer .= "\n"; - $data = $rest; - $COLUMN_POSITION = 0; - } else { - last; - } - } else { - last; - } - } - if (!$last || length $data > 0) { - &printLinePartAtLevel($level, $data, $buffer); - } -} -sub printLinePartAtLevel { # ($Level, Line, Buffer) - # Print a single line that should fit on one line - my ($level, $data, $buffer) = @_; - if ($ANSI_COLORS) { - $$buffer .= $ATTRIBUTE_PREFIX{$level}; - } - if (&prefixLineWithLevel($level)) { - $$buffer .= $level . ": "; - $COLUMN_POSITION = (length $level) + 2; - } - $$buffer .= $data; - $COLUMN_POSITION += length $data; - if ($ANSI_COLORS) { - $$buffer .= $ATTRIBUTE_POSTFIX{$level}; - } -} -sub prefixLineWithLevel { # (Level) -> Bool - my ($level) = @_; - return ($TYPES{$level} > 2 && $VERBOSITY >= 4 && $COLUMN_POSITION == 0); -} - diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/ContentNodeSelection.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/ContentNodeSelection.pm deleted file mode 100644 index 0886cb50da0..00000000000 --- a/vespaclient/src/perl/lib/Yahoo/Vespa/ContentNodeSelection.pm +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# -# This module implements a way to select a subset of nodes from a Vespa -# application. -# - -package Yahoo::Vespa::ContentNodeSelection; - -use strict; -use warnings; -use Yahoo::Vespa::ArgParser; -use Yahoo::Vespa::ConsoleOutput; -use Yahoo::Vespa::Utils; -use Yahoo::Vespa::VespaModel; - -BEGIN { # - Declare exports and dependency aliases for module - use base 'Exporter'; - our @EXPORT = qw( - NO_LOCALHOST_CONSTRAINT - CLUSTER_ONLY_LIMITATION - ); - # Package aliases - *VespaModel:: = *Yahoo::Vespa::VespaModel:: ; -} - -my $CLUSTER; -my $NODE_TYPE; -my $INDEX; -my $FORCE = 0; -our $LOCALHOST; - -use constant NO_LOCALHOST_CONSTRAINT => 1; -use constant CLUSTER_ONLY_LIMITATION => 2; - -return 1; - -######################## Externally usable functions ####################### - -sub registerCommandLineArguments { # (Flags) - my ($flags) = @_; - if (!defined $flags) { $flags = 0; } - if (($flags & NO_LOCALHOST_CONSTRAINT) == 0) { - $LOCALHOST = getHostname(); - } else { - $LOCALHOST = undef; - } - if (($flags & CLUSTER_ONLY_LIMITATION) == 0) { - setOptionHeader("Node selection options. By default, nodes running " - . "locally will be selected:"); - } - setStringOption( - ['c', 'cluster'], - \$CLUSTER, - 'Cluster name. ' - . 'If unspecified, and vespa is installed on current node, ' - . 'information will be attempted auto-extracted'); - setFlagOption( - ['f', 'force'], - \$FORCE, - 'Force execution'); - if (($flags & CLUSTER_ONLY_LIMITATION) == 0) { - setStringOption( - ['t', 'type'], - \$NODE_TYPE, - 'Node type - can either be \'storage\' or ' - . '\'distributor\'. If not specified, the operation will use ' - . 'state for both types.'); - setIntegerOption( - ['i', 'index'], - \$INDEX, - 'Node index. If not specified, all nodes ' - . 'found running on this host will be used.'); - } -} -sub visit { # (Callback) - my ($callback) = @_; - printDebug "Visiting selected services: " - . "Cluster " . (defined $CLUSTER ? $CLUSTER : 'undef') - . " node type " . (defined $NODE_TYPE ? $NODE_TYPE : 'undef') - . " index " . (defined $INDEX ? $INDEX : 'undef') - . " localhost only ? " . ($LOCALHOST ? "true" : "false") . "\n"; - VespaModel::visitServices(sub { - my ($info) = @_; - $$info{'type'} = &convertType($$info{'type'}); - if (!&validType($$info{'type'})) { return; } - if (defined $CLUSTER && $CLUSTER ne $$info{'cluster'}) { return; } - if (defined $NODE_TYPE && $NODE_TYPE ne $$info{'type'}) { return; } - if (defined $INDEX && $INDEX ne $$info{'index'}) { return; } - if (!defined $INDEX && defined $LOCALHOST - && $LOCALHOST ne $$info{'host'}) - { - return; - } - # printResult "Ok $$info{'cluster'} $$info{'type'} $$info{'index'}\n"; - &$callback($info); - }); -} -sub showSettings { # () - printDebug "Visiting selected services: " - . "Cluster " . (defined $CLUSTER ? $CLUSTER : 'undef') - . " node type " . (defined $NODE_TYPE ? $NODE_TYPE : 'undef') - . " index " . (defined $INDEX ? $INDEX : 'undef') - . " localhost only ? " . ($LOCALHOST ? "true" : "false") . "\n"; -} - -sub validateCommandLineArguments { # (WantedState) - my ($wanted_state) = @_; - - if (defined $NODE_TYPE) { - if ($NODE_TYPE !~ /^(distributor|storage)$/) { - printWarning "Invalid value '$NODE_TYPE' given for node type.\n"; - return 0; - } - } - - if (!$FORCE && - (!defined $NODE_TYPE || $NODE_TYPE eq "distributor") && - $wanted_state eq "maintenance") { - printWarning "Setting the distributor to maintenance mode may have " - . "severe consequences for feeding!\n" - . "Please specify -t storage to only set the storage node to " - . "maintenance mode, or -f to override this error.\n"; - return 0; - } - - printDebug "Command line arguments validates ok\n"; - return 1; -} - -sub hasClusterSelection { - return defined $CLUSTER; -} - -############## Utility functions - Not intended for external use ############# - -sub validType { # (ServiceType) -> Bool - my ($type) = @_; - return $type =~ /^(?:distributor|storage)$/; -} -sub convertType { # (ServiceType) -> Bool - my ($type) = @_; - if ($type eq 'storagenode') { return 'storage'; } - return $type; -} - diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm deleted file mode 100644 index 6b5e5380540..00000000000 --- a/vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm +++ /dev/null @@ -1,179 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# -# Simple HTTP wrapper library -# -# Intentions: -# - Make it very easy for programs to do HTTP requests towards Rest APIs. -# - Allow unit tests to fake returned data -# - Allow using another external dependency for HTTP without affecting apps -# -# An HTTP request returns a Response that is a hash containing: -# code - The HTTP status code -# status - The HTTP status string that comes with the code -# content - The content of the reply -# all - The entire response coming over the TCP connection -# This is here for debugging and testing. If you need specifics like -# HTTP headers, we should just add specific fields for them rather than -# to parse all content. -# -# Examples: -# -# my @headers = ( -# "X-Foo" => 'Bar' -# ); -# my @params = ( -# "verbose" => 1 -# ); -# -# $response = Http::get('localhost', 80, '/status.html'); -# $response = Http::get('localhost', 80, '/status.html', \@params, \@headers); -# $response = Http::request('POST', 'localhost', 80, '/test', \@params, -# "Some content", \@headers); -# - -package Yahoo::Vespa::Http; - -use strict; -use warnings; - -use Net::INET6Glue::INET_is_INET6; -use LWP::Simple (); -use URI (); -use URI::Escape qw( uri_escape ); -use Yahoo::Vespa::ConsoleOutput; -use Yahoo::Vespa::Utils; - -my %LEGAL_TYPES; -my $BROWSER; -my $EXECUTE; - -&initialize(); - -return 1; - -######################## Externally usable functions ####################### - -sub get { # (Host, Port, Path, Params, Headers) -> Response - my ($host, $port, $path, $params, $headers) = @_; - return &request('GET', $host, $port, $path, $params, undef, $headers); -} -sub request { # (Type, Host, Port, Path, Params, Content, Headers) -> Response - my ($type, $host, $port, $path, $params, $content, $headers) = @_; - if (!exists $LEGAL_TYPES{$type}) { - confess "Invalid HTTP type '$type' specified."; - } - if (defined $params && ref($params) ne "ARRAY") { - confess 'HTTP request attempted without array ref for params'; - } - if (defined $headers && ref($headers) ne "ARRAY") { - confess 'HTTP request attempted without array ref for headers'; - } - return &$EXECUTE( - $type, $host, $port, $path, $params, $content, $headers); -} -sub encodeForm { # (KeyValueMap) -> RawString - my $data; - for (my $i=0; $i < scalar @_; $i += 2) { - my ($key, $value) = ($_[$i], $_[$i+1]); - if ($i != 0) { - $data .= '&'; - } - $data .= uri_escape($key); - if (defined $value) { - $data .= '=' . uri_escape($value); - } - } - return $data; -} - -################## Functions for unit tests to mock internals ################ - -sub setHttpExecutor { # (Function) - $EXECUTE = $_[0] -} - -############## Utility functions - Not intended for external use ############# - -sub initialize { # () - %LEGAL_TYPES = map { $_ => 1 } ( 'GET', 'POST', 'PUT', 'DELETE'); - $BROWSER = LWP::UserAgent->new; - my $tls_enabled = $ENV{'VESPA_TLS_ENABLED'}; - if (defined $tls_enabled and $tls_enabled eq '1') { - $BROWSER->ssl_opts( SSL_version => 'TLSv12'); - my $hostname_verification_disabled = $ENV{'VESPA_TLS_HOSTNAME_VALIDATION_DISABLED'}; - if (defined $hostname_verification_disabled and $hostname_verification_disabled eq '1') { - $BROWSER->ssl_opts( verify_hostname => 0); - } - $BROWSER->ssl_opts( SSL_cipher_list => 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-CHACHA20-POLY1305:TLS13-AES-128-GCM-SHA256:TLS13-AES-256-GCM-SHA384:TLS13-CHACHA20-POLY1305-SHA256' ); - } - if (defined $ENV{'VESPA_TLS_CA_CERT'}) { - $BROWSER->ssl_opts( SSL_ca_file => $ENV{'VESPA_TLS_CA_CERT'} ); - } - if (defined $ENV{'VESPA_TLS_CERT'}) { - $BROWSER->ssl_opts( SSL_cert_file => $ENV{'VESPA_TLS_CERT'} ); - } - if (defined $ENV{'VESPA_TLS_PRIVATE_KEY'}) { - $BROWSER->ssl_opts( SSL_key_file => $ENV{'VESPA_TLS_PRIVATE_KEY'} ); - } - $BROWSER->agent('Vespa-perl-script'); - $EXECUTE = \&execute; -} -sub execute { # (Type, Host, Port, Path, Params, Content, Headers) -> Response - my ($type, $host, $port, $path, $params, $content, $headers) = @_; - if (!defined $headers) { $headers = []; } - if (!defined $params) { $params = []; } - my $url = URI->new(&buildUri($host, $port, $path)); - if (defined $params) { - $url->query_form(@$params); - } - printSpam "Performing HTTP request $type '$url'.\n"; - my $response; - if ($type eq 'GET') { - !defined $content or confess "$type requests cannot have content"; - $response = $BROWSER->get($url, @$headers); - } elsif ($type eq 'POST') { - if (defined $content) { - $response = $BROWSER->post($url, $params, @$headers, - 'Content' => $content); - } else { - $response = $BROWSER->post($url, $params, @$headers); - } - } elsif ($type eq 'PUT') { - if (defined $content) { - $response = $BROWSER->put($url, $params, @$headers, - 'Content' => $content); - } else { - $response = $BROWSER->put($url, $params, @$headers); - } - } elsif ($type eq 'DELETE') { - !defined $content or confess "$type requests cannot have content"; - $response = $BROWSER->put($url, $params, @$headers); - } else { - confess "Unknown type $type"; - } - my $autoLineBreak = enableAutomaticLineBreaks(0); - printSpam "Got HTTP result: '" . $response->as_string . "'\n"; - enableAutomaticLineBreaks($autoLineBreak); - return ( - 'code' => $response->code, - 'headers' => $response->headers(), - 'status' => $response->message, - 'content' => $response->content, - 'all' => $response->as_string - ); -} -sub buildUri { # (Host, Port, Path) -> UriString - my ($host, $port, $path) = @_; - my $tls_enabled = $ENV{'VESPA_TLS_ENABLED'}; - my $uri = (defined $tls_enabled and $tls_enabled eq '1') ? "https:" : "http:"; - if (defined $host) { - $uri .= '//' . $host; - if (defined $port) { - $uri .= ':' . $port; - } - } - if (defined $path) { - $uri .= $path; - } - return $uri; -} diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/Json.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/Json.pm deleted file mode 100644 index d811c24ed7b..00000000000 --- a/vespaclient/src/perl/lib/Yahoo/Vespa/Json.pm +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# -# Minimal JSON wrapper. -# -# Intentions: -# - If needed, be able to switch the implementation of the JSON parser -# without components using this class seeing it. -# - Make API as simple as possible to use. -# -# Currently uses JSON.pm from ypan/perl-JSON -# -# Example usage: -# -# my $json = <<EOS; -# { -# 'foo' : [ -# { 'key1' : 2 }, -# { 'key2' : 5 } -# ] -# } -# -# my $result = Json::parse($json); -# my $firstkey = $result->{'foo'}->[0]->{'key1'} -# my @keys = @{ $result->{'foo'} }; -# -# See JsonTest for more usage. Add tests there if unsure. -# - -package Yahoo::Vespa::Json; - -use strict; -use warnings; - # Location of JSON.pm is not in default search path on tested Yahoo nodes. -use lib ($ENV{'VESPA_HOME'} . '/lib64/perl5/site_perl/5.14/'); -use JSON; - -return 1; - -# Parses a string with json data returning an object tree -sub parse { # (RawString) -> ObjTree - my ($raw) = @_; - my $json = decode_json($raw); - return $json; -} - -# Encodes an object tree as returned from parse back to a raw string -sub encode { # (ObjTree) -> RawString - my ($json) = @_; - my $JSON = JSON->new->allow_nonref; - my $encoded = $JSON->pretty->encode($json); - return $encoded; -} diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/Utils.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/Utils.pm deleted file mode 100644 index 609ec97f385..00000000000 --- a/vespaclient/src/perl/lib/Yahoo/Vespa/Utils.pm +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# -# Some simple utilities to allow unit tests to mock behavior. -# - -package Yahoo::Vespa::Utils; - -use strict; -use warnings; -use Carp (); - -BEGIN { # - Define exports from this module - use base 'Exporter'; - our @EXPORT = qw( - exitApplication - getHostname - confess - assertNotUnitTest - dumpStructure - ); -} - -my $HOSTNAME; -my $EXIT_HANDLER; -my $IS_UNIT_TEST; - -&initialize(); - -return 1; - -########################## Default exported functions ######################## - -# Use this function to get hostname to allow unit test mocking for tests to be -# independent of computer they run on. -sub getHostname { # () - if (!defined $HOSTNAME) { - &assertNotUnitTest(); - $HOSTNAME = `vespa-print-default hostname`; - chomp $HOSTNAME; - } - return $HOSTNAME; -} - -# Use instead of exit() to allow unit tests to mock the call to avoid aborting -sub exitApplication { #(ExitCode) - if ($IS_UNIT_TEST && $EXIT_HANDLER == \&defaultExitHandler) { - &confess("Exit handler not overridden in unit test"); - } - &$EXIT_HANDLER(@_); -} - -# Use instead of die to get backtrace when dieing -sub confess { # (Reason) - Carp::confess(@_); -} - -# Call for behavior that you want to ensure is not used in unit tests. -# Typically unit tests have to mock commands that for instance fetch host name -# or require that terminal is set etc. Unit tests use mocks for this. This -# command can be used in code, such that unit tests die if they reach the -# non-mocked code. -sub assertNotUnitTest { # () - if ($IS_UNIT_TEST) { - confess "Unit tests should not reach here. Mock required. " - . "Initialize mock"; - } -} - -# Use to look at content of a perl struct. -sub dumpStructure { # (ObjTree) -> ReadableString - my ($var) = @_; - use Data::Dumper; - local $Data::Dumper::Indent = 1; - local $Data::Dumper::Sortkeys = 1; - return Dumper($var); -} - -################## Functions for unit tests to mock internals ################ - -sub initializeUnitTest { # (Hostname, ExitHandler) - my ($host, $exitHandler) = @_; - $IS_UNIT_TEST = 1; - $HOSTNAME = $host; - $EXIT_HANDLER = $exitHandler; -} - -############## Utility functions - Not intended for external use ############# - -sub initialize { # () - $EXIT_HANDLER = \&defaultExitHandler; -} -sub defaultExitHandler { # () - my ($exitcode) = @_; - exit($exitcode); -} diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/VespaModel.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/VespaModel.pm deleted file mode 100644 index 844ab6653b0..00000000000 --- a/vespaclient/src/perl/lib/Yahoo/Vespa/VespaModel.pm +++ /dev/null @@ -1,354 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# -# Vespa model -# -# Make vespa model information available for tools. To for instance get an -# overview of where services are running. -# -# Possible improvements: -# -# - Depending on config Rest API and config server might be better than -# depending on vespa-get-config tool and config format. -# - Support direct communication with config server if config proxy is not -# running (unless vespa-get-config does that for us) -# - Support specifying config server, to be able to run tool external from the -# vespa system to talk to. -# - Return a list of all matching sockets instead of first found. -# - Be able to specify a set of port tags needed for a match. -# - -package Yahoo::Vespa::VespaModel; - -use strict; -use warnings; -use Yahoo::Vespa::ArgParser; -use Yahoo::Vespa::ConsoleOutput; -use Yahoo::Vespa::Utils; - -my $RETRIEVE_MODEL_CONFIG; # Allow unit tests to switch source of config info -my $MODEL; -my $CONFIG_SERVER_HOST; -my $CONFIG_SERVER_PORT; -my $CONFIG_REQUEST_TIMEOUT; - -&initialize(); - -return 1; - -######################## Externally usable functions ####################### - -sub registerCommandLineArguments { # () - setOptionHeader("Config retrieval options:"); - setHostOption( - ['config-server'], - \$CONFIG_SERVER_HOST, - 'Host name of config server to query'); - setPortOption( - ['config-server-port'], - \$CONFIG_SERVER_PORT, - 'Port to connect to config server on'); - setFloatOption( - ['config-request-timeout'], - \$CONFIG_REQUEST_TIMEOUT, - 'Timeout of config request'); -} - -sub visitServices { # (Callback) - my $model = &get(); - my ($callback) = @_; - my @services = @{ &getServices($model); }; - foreach my $service (sort serviceOrder @services) { - &$callback($service); - } -} - -sub getServices { - my $model = &get(); - my @result; - foreach my $hostindex (keys %{ $$model{'hosts'} }) { - my $host = ${ $$model{'hosts'} }{ $hostindex }; - foreach my $serviceindex (keys %{ $$host{'services'} }) { - my $service = ${ $$host{'services'} }{ $serviceindex }; - my %info = ( - 'name' => $$service{'name'}, - 'type' => $$service{'type'}, - 'configid' => $$service{'configid'}, - 'cluster' => $$service{'clustername'}, - 'host' => $$host{'name'} - ); - if (exists $$service{'index'}) { - $info{'index'} = $$service{'index'}; - } - push @result, \%info; - } - } - return \@result; -} - -# Get socket for given service matching given conditions (Given as a hash) -# Legal conditions: -# type - Service type -# tag - Port tag -# index - Service index -# clustername - Name of cluster. -# Example: getSocketForService( 'type' => 'distributor', 'index' => 3, -# 'tag' => 'http', 'tag' => 'state' ); -sub getSocketForService { # (Conditions) => [{host=>$,port=>$,index=>$}...] - my $model = &get(); - my $conditions = \@_; - printDebug "Looking at model to find socket for a service.\n"; - &validateConditions($conditions); - my $hosts = $$model{'hosts'}; - if (!defined $hosts) { return; } - my @results; - foreach my $hostindex (keys %$hosts) { - my $host = $$hosts{$hostindex}; - my $services = $$host{'services'}; - if (defined $services) { - printSpam "Searching services on host $$host{'name'}\n"; - foreach my $serviceindex (keys %$services) { - my $service = $$services{$serviceindex}; - my $type = $$service{'type'}; - my $cluster = $$service{'clustername'}; - if (!&serviceTypeMatchConditions($conditions, $type)) { - printSpam "Unwanted service '$type'.\n"; - next; - } - if (!&indexMatchConditions($conditions, $$service{'index'})) { - printSpam "Unwanted index '$$service{'index'}'.\n"; - next; - } - if (!&clusterNameMatchConditions($conditions, $cluster)) { - printSpam "Unwanted index '$$service{'index'}'.\n"; - next; - } - my $ports = $$service{'ports'}; - if (defined $ports) { - my $resultcount = 0; - foreach my $portindex (keys %$ports) { - my $port = $$ports{$portindex}; - my $tags = $$port{'tags'}; - if (defined $tags) { - if (!&tagsMatchConditions($conditions, $tags)) { - next; - } - } - push @results, { 'host' => $$host{'name'}, - 'port' => $$port{'number'}, - 'index' => $$service{'index'} }; - ++$resultcount; - } - if ($resultcount == 0) { - printSpam "No ports with acceptable tags found. " - . "Ignoring $type.$$service{'index'}\n"; - } - } else { - printSpam "No ports defined. " - . "Ignoring $type.$$service{'index'}\n"; - } - } - } - } - return \@results; -} - -############## Utility functions - Not intended for external use ############# - -sub initialize { # () - $RETRIEVE_MODEL_CONFIG = \&retrieveModelConfigDefault; -} -sub setModelRetrievalFunction { # (Function) - $RETRIEVE_MODEL_CONFIG = $_[0]; -} -sub retrieveModelConfigDefault { # () - my $VESPA_HOME= $ENV{'VESPA_HOME'}; - my $cmd = ${VESPA_HOME} . '/bin/vespa-get-config -l -n cloud.config.model -i admin/model'; - - if (defined $CONFIG_REQUEST_TIMEOUT) { - $cmd .= " -w $CONFIG_REQUEST_TIMEOUT"; - } - - if (!defined $CONFIG_SERVER_HOST) { - my $temp = `${VESPA_HOME}/bin/vespa-print-default configservers`; - chomp($temp); - $CONFIG_SERVER_HOST = $temp; - } - - if (!defined $CONFIG_SERVER_PORT) { - my $temp = `${VESPA_HOME}/bin/vespa-print-default configserver_rpc_port`; - chomp($temp); - $CONFIG_SERVER_PORT = $temp; - } - $cmd .= " -p $CONFIG_SERVER_PORT"; - - my $errors = ""; - foreach my $cfshost (split(' ', $CONFIG_SERVER_HOST)) { - my $hostcmd = $cmd . " -s $cfshost"; - - printDebug "Fetching model config '$hostcmd'.\n"; - my @data = `$hostcmd 2>&1`; - if ($? != 0 || join(' ', @data) =~ /^error/) { - $errors .= "Failed to get model config from config command line tool:\n" - . "Command: $hostcmd\n" - . "Exit code: $?\n" - . "Output: " . join("\n", @data) . "\n"; - } else { - return @data; - } - } - printError $errors; - exitApplication(1); -} -sub fetch { # () - my @data = &$RETRIEVE_MODEL_CONFIG(); - $MODEL = &parseConfig(@data); - return $MODEL; -} -sub get { # () - if (!defined $MODEL) { - return &fetch(); - } - return $MODEL; -} -sub validateConditions { # (ConditionArrayRef) - my ($condition) = @_; - for (my $i=0, my $n=scalar @$condition; $i<$n; $i += 2) { - if ($$condition[$i] !~ /^(type|tag|index|clustername)$/) { - printError "Invalid socket for service condition " - . "'$$condition[$i]' given.\n"; - exitApplication(1); - } - } -} -sub tagsMatchConditions { # (Condition, TagList) -> Bool - my ($condition, $taglist) = @_; - my %tags = map { $_ => 1 } @$taglist; - for (my $i=0, my $n=scalar @$condition; $i<$n; $i += 2) { - if ($$condition[$i] eq 'tag' && !exists $tags{$$condition[$i + 1]}) { - return 0; - } - } - return 1; -} -sub serviceTypeMatchConditions { # (Condition, ServiceType) -> Bool - my ($condition, $type) = @_; - for (my $i=0, my $n=scalar @$condition; $i<$n; $i += 2) { - if ($$condition[$i] eq 'type' && $$condition[$i + 1] ne $type) { - return 0; - } - } - return 1; -} -sub clusterNameMatchConditions { # (Condition, ClusterName) -> Bool - my ($condition, $cluster) = @_; - for (my $i=0, my $n=scalar @$condition; $i<$n; $i += 2) { - if ($$condition[$i] eq 'clustername' && $$condition[$i + 1] ne $cluster) - { - return 0; - } - } - return 1; -} -sub indexMatchConditions { # (Condition, Index) -> Bool - my ($condition, $index) = @_; - for (my $i=0, my $n=scalar @$condition; $i<$n; $i += 2) { - if ($$condition[$i] eq 'index' && $$condition[$i + 1] ne $index) { - return 0; - } - } - return 1; -} -sub parseConfig { # () - my $model = {}; - printDebug "Parsing vespa model raw config to create object tree\n"; - my $autoLineBreak = enableAutomaticLineBreaks(0); - foreach my $line (@_) { - chomp $line; - printSpam "Parsing line '$line'\n"; - if ($line =~ /^hosts\[(\d+)\]\.(([a-z]+).*)$/) { - my ($hostindex, $tag, $rest) = ($1, $3, $2); - my $host = &getHost($hostindex, $model); - if ($tag eq 'services') { - &parseService($host, $rest); - } else { - &parseValue($host, $rest); - } - } - } - enableAutomaticLineBreaks($autoLineBreak); - return $model; -} -sub parseService { # (Host, Line) - my ($host, $line) = @_; - if ($line =~ /^services\[(\d+)\].(([a-z]+).*)$/) { - my ($serviceindex, $tag, $rest) = ($1, $3, $2); - my $service = &getService($serviceindex, $host); - if ($tag eq 'ports') { - &parsePort($service, $rest); - } else { - &parseValue($service, $rest); - } - } -} -sub parsePort { # (Service, Line) - my ($service, $line) = @_; - if ($line =~ /^ports\[(\d+)\].(([a-z]+).*)$/) { - my ($portindex, $tag, $rest) = ($1, $3, $2); - my $port = &getPort($portindex, $service); - &parseValue($port, $rest); - } -} -sub parseValue { # (Entity, Line) - my ($entity, $line) = @_; - $line =~ /^(\S+) (?:\"(.*)\"|(\d+))$/ or confess "Unexpected line '$line'."; - my ($id, $string, $number) = ($1, $2, $3); - if ($id eq 'tags' && defined $string) { - my @tags = split(/\s+/, $string); - $$entity{$id} = \@tags; - } elsif (defined $string) { - $$entity{$id} = $string; - } else { - defined $number or confess "Should not happen"; - $$entity{$id} = $number; - } -} -sub getEntity { # (Type, Index, ParentEntity) - my ($type, $index, $parent) = @_; - if (!exists $$parent{$type}) { - $$parent{$type} = {}; - } - my $list = $$parent{$type}; - if (!exists $$list{$index}) { - $$list{$index} = {}; - } - return $$list{$index}; -} -sub getHost { # (Index, Model) - return &getEntity('hosts', $_[0], $_[1]); -} -sub getService { # (Index, Host) - return &getEntity('services', $_[0], $_[1]); -} -sub getPort { # (Index, Service) - return &getEntity('ports', $_[0], $_[1]); -} -sub serviceOrder { - if ($a->{'cluster'} ne $b->{'cluster'}) { - return $a->{'cluster'} cmp $b->{'cluster'}; - } - if ($a->{'type'} ne $b->{'type'}) { - return $a->{'type'} cmp $b->{'type'}; - } - if ($a->{'index'} != $b->{'index'}) { - return $a->{'index'} <=> $b->{'index'}; - } - if ($a->{'host'} ne $b->{'host'}) { - return $a->{'host'} cmp $b->{'host'}; - } - if ($a->{'configid'} ne $b->{'configid'}) { - return $a->{'configid'} cmp $b->{'configid'}; - } - confess "Unsortable elements: " . dumpStructure($a) . "\n" - . dumpStructure($b) . "\n"; -} - diff --git a/vespaclient/src/perl/test/Generic/UseTest.pl b/vespaclient/src/perl/test/Generic/UseTest.pl deleted file mode 100644 index 3b56873c775..00000000000 --- a/vespaclient/src/perl/test/Generic/UseTest.pl +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# -# That that all perl files use strict and warnings -# - -use Test::More; -use TestUtils::VespaTest; - -use strict; -use warnings; - -my @dirs = ( - '../bin', - '../lib', - 'Yahoo/Vespa/Mocks' -); - -my $checkdirs = join(' ', @dirs); - -my @files = `find $checkdirs -name \\*.pm -or -name \\*.pl`; -chomp @files; - -printTest "Checking " . (scalar @files) . " files for includes.\n"; - -foreach my $file (@files) { - ok( system("cat $file | grep 'use strict;' >/dev/null") == 0, - "$file use strict" ); - ok( system("cat $file | grep 'use warnings;' >/dev/null") == 0, - "$file use warnings" ); -} - -done_testing(); - -exit(0); diff --git a/vespaclient/src/perl/test/TestUtils/OutputCapturer.pm b/vespaclient/src/perl/test/TestUtils/OutputCapturer.pm deleted file mode 100644 index b6c67c81ed7..00000000000 --- a/vespaclient/src/perl/test/TestUtils/OutputCapturer.pm +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package TestUtils::OutputCapturer; - -use Test::More; -use Yahoo::Vespa::ConsoleOutput; - -BEGIN { - use base 'Exporter'; - our @EXPORT = qw( - getOutput - isOutput - matchesOutput - ); -} - -Yahoo::Vespa::ConsoleOutput::setTerminalWidth(79); - -our ($stdout, $stderr); -my $USE_COLORS = 1; - -&openStreams(); - -END { - &closeStreams(); -} - -return 1; - -sub useColors { - $USE_COLORS = $_[0]; - &closeStreams(); - &openStreams(); -} - -sub isOutput { # (stdout, stderr, test) - my ($expected_cout, $expected_cerr, $test) = @_; - my ($cout, $cerr) = &getOutput(); - &diff($expected_cout, $cout); - ok ($cout eq $expected_cout, $test . " - stdout"); - &diff($expected_cerr, $cerr); - ok ($cerr eq $expected_cerr, $test . " - stderr"); -} - -sub matchesOutput { # (stdout_pattern, stderr_pattern, test) - my ($cout_pat, $cerr_pat, $test) = @_; - my ($cout, $cerr) = &getOutput(); - if ($cout !~ $cout_pat) { - diag("Output did not match standard out pattern:\n/$cout_pat/:\n$cout"); - } - ok ($cout =~ $cout_pat, $test . " - stdout"); - if ($cerr !~ $cerr_pat) { - diag("Stderr output did not match standard err pattern:\n" - . "/$cerr_pat/:\n$cerr"); - } - ok ($cerr =~ $cerr_pat, $test . " - stdout"); -} - -sub getOutput { - my $cout = &getStdOut(); - my $cerr = &getStdErr(); - &closeStreams(); - &openStreams(); - return ($cout, $cerr); -} - -sub openStreams { - open ($stdout, ">/tmp/vespaclient.perltest.stdout.log") - or die "Failed to create tmp file for stdout"; - open ($stderr, ">/tmp/vespaclient.perltest.stderr.log") - or die "Failed to create tmp file for stdout"; - Yahoo::Vespa::ConsoleOutput::initialize($stdout, $stderr, $USE_COLORS); -} - -sub closeStreams { - close $stdout; - close $stderr; - system("rm /tmp/vespaclient.perltest.stdout.log"); - system("rm /tmp/vespaclient.perltest.stderr.log"); -} - -sub getStdOut { - my $data = `cat /tmp/vespaclient.perltest.stdout.log`; - if (!defined $data) { $data = ''; } - return $data; -} - -sub getStdErr { - my $data = `cat /tmp/vespaclient.perltest.stderr.log`; - if (!defined $data) { $data = ''; } - return $data; -} - -sub diff { - my ($expected, $actual) = @_; - if ($expected eq $actual) { return; } - &writeToFile("/tmp/vespaclient.perltest.expected", $expected); - &writeToFile("/tmp/vespaclient.perltest.actual", $actual); - print "Output differs. Diff:\n"; - system("diff -u /tmp/vespaclient.perltest.expected " - . "/tmp/vespaclient.perltest.actual"); - system("rm -f /tmp/vespaclient.perltest.expected"); - system("rm -f /tmp/vespaclient.perltest.actual"); -} - -sub writeToFile { - my ($file, $data) = @_; - my $fh; - open ($fh, ">$file") or die "Failed to open temp file for writing."; - print $fh $data; - close $fh; -} diff --git a/vespaclient/src/perl/test/TestUtils/VespaTest.pm b/vespaclient/src/perl/test/TestUtils/VespaTest.pm deleted file mode 100644 index 92a3cd0b1e0..00000000000 --- a/vespaclient/src/perl/test/TestUtils/VespaTest.pm +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package TestUtils::VespaTest; - -use Test::More; -use TestUtils::OutputCapturer; -use Yahoo::Vespa::Utils; - -BEGIN { - use base 'Exporter'; - our @EXPORT = qw( - isOutput - matchesOutput - setApplication - assertRun - assertRunMatches - printTest - useColors - setLocalHost - ); -} - -my $APPLICATION; - -&initialize(); - -return 1; - -sub initialize { - Yahoo::Vespa::Utils::initializeUnitTest( - 'testhost.yahoo.com', \&mockedExitHandler); -} - -sub setLocalHost { - my ($host) = @_; - Yahoo::Vespa::Utils::initializeUnitTest( - $host, \&mockedExitHandler); -} - -sub useColors { - TestUtils::OutputCapturer::useColors(@_); -} - -sub mockedExitHandler { - my ($exitcode) = @_; - die "Application exited with exitcode $exitcode."; -} - -sub setApplication { - my ($main_func) = @_; - $APPLICATION = $main_func; -} - -sub assertRun { - my ($testname, $argstring, - $expected_exitcode, $expected_stdout, $expected_stderr) = @_; - my $exitcode = &run($argstring); - is( $exitcode, $expected_exitcode, "$testname - exitcode" ); - # print OutputCapturer::getStdOut(); - isOutput($expected_stdout, $expected_stderr, $testname); -} - -sub assertRunMatches { - my ($testname, $argstring, - $expected_exitcode, $expected_stdout, $expected_stderr) = @_; - my $exitcode = &run($argstring); - is( $exitcode, $expected_exitcode, "$testname - exitcode" ); - # print OutputCapturer::getStdOut(); - matchesOutput($expected_stdout, $expected_stderr, $testname); -} - -sub run { - my ($argstring) = @_; - my @args = split(/\s+/, $argstring); - eval { - Yahoo::Vespa::ArgParser::initialize(); - &$APPLICATION(\@args); - }; - my $exitcode = 0; - if ($@) { - if ($@ =~ /Application exited with exitcode (\d+)\./) { - $exitcode = 1; - } else { - print "Unknown die signal '" . $@ . "'\n"; - } - } - return $exitcode; -} - -sub printTest { - print "Test: ", @_; -} diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/ArgParserTest.pl b/vespaclient/src/perl/test/Yahoo/Vespa/ArgParserTest.pl deleted file mode 100644 index 870f31c1dfa..00000000000 --- a/vespaclient/src/perl/test/Yahoo/Vespa/ArgParserTest.pl +++ /dev/null @@ -1,313 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -use Test::More; - -BEGIN { use_ok( 'Yahoo::Vespa::ArgParser' ); } -require_ok( 'Yahoo::Vespa::ArgParser' ); - -BEGIN { *ArgParser:: = *Yahoo::Vespa::ArgParser:: } - -use TestUtils::OutputCapturer; - -TestUtils::OutputCapturer::useColors(1); - -&testSyntaxPage(); - -TestUtils::OutputCapturer::useColors(0); - -&testStringOption(); -&testIntegerOption(); -&testHostOption(); -&testPortOption(); -&testFlagOption(); -&testCountOption(); -&testComplexParsing(); -&testArguments(); - -done_testing(); - -exit(0); - -sub testSyntaxPage { - # Empty - ArgParser::writeSyntaxPage(); - my $expected = <<EOS; -Usage: ArgParserTest.pl -EOS - isOutput($expected, '', 'Empty syntax page'); - - # Built in only - Yahoo::Vespa::ArgParser::registerInternalParameters(); - ArgParser::writeSyntaxPage(); - $expected = <<EOS; -Usage: ArgParserTest.pl [Options] - -Options: - -h --help : Show this help page. - -v : Create more verbose output. - -s : Create less verbose output. - --show-hidden : Also show hidden undocumented debug options. -EOS - isOutput($expected, '', 'Syntax page with default args'); - - # Actual example - ArgParser::initialize(); - - setProgramBinaryName("testprog"); - setProgramDescription( - "This is a multiline description of what the program is that " - . "should be split accordingly to look nice. For now probably hard " - . "coded, but can later be extended to detect terminal width."); - my $arg; - setArgument(\$arg, "Test Arg", "This argument is not used for anything.", - OPTION_REQUIRED); - my $optionalArg; - setArgument(\$arg, "Another Test Arg", - "This argument is not used for anything either."); - - setOptionHeader("My prog headers. Also a long line just to check that it " - . "is also split accordingly."); - my $stringval; - my $flag; - my $intval; - setStringOption(['string', 'j'], \$stringval, "A random string"); - setFlagOption(['flag', 'f'], \$flag, "A flag option with a pretty long " - . "description that might need to be split into multiple lines."); - setOptionHeader("More options"); - setIntegerOption(['integer', 'i'], \$intval, "A secret integer option.", - OPTION_SECRET); - Yahoo::Vespa::ArgParser::registerInternalParameters(); - ArgParser::writeSyntaxPage(); - $expected = <<EOS; -This is a multiline description of what the program is that should be split -accordingly to look nice. For now probably hard coded, but can later be -extended to detect terminal width. - -Usage: testprog [Options] <Test Arg> [Another Test Arg] - -Arguments: - Test Arg : This argument is not used for anything. - Another Test Arg : This argument is not used for anything either. - -Options: - -h --help : Show this help page. - -v : Create more verbose output. - -s : Create less verbose output. - --show-hidden : Also show hidden undocumented debug options. - -My prog headers. Also a long line just to check that it is also split -accordingly. - --string -j : A random string - --flag -f : A flag option with a pretty long description that might need - to be split into multiple lines. -EOS - isOutput($expected, '', 'Actual syntax page example'); - - ArgParser::setShowHidden(1); - ArgParser::writeSyntaxPage(); - $expected = <<EOS; -This is a multiline description of what the program is that should be split -accordingly to look nice. For now probably hard coded, but can later be -extended to detect terminal width. - -Usage: testprog [Options] <Test Arg> [Another Test Arg] - -Arguments: - Test Arg : This argument is not used for anything. - Another Test Arg : This argument is not used for anything either. - -Options: - -h --help : Show this help page. - -v : Create more verbose output. - -s : Create less verbose output. - --show-hidden : Also show hidden undocumented debug options. - -My prog headers. Also a long line just to check that it is also split -accordingly. - --string -j : A random string - --flag -f : A flag option with a pretty long description that might need - to be split into multiple lines. - -More options - --integer -i : A secret integer option. - - --nocolors : Do not use ansi colors in print. -EOS - isOutput($expected, '', 'Actual syntax page example with hidden'); -} - -sub setUpParseTest { - Yahoo::Vespa::ArgParser::initialize(); -} - -sub parseFail { - my ($optstring, $expectedError) = @_; - my @args = split(/\s+/, $optstring); - my $name = $expectedError; - chomp $name; - if (length $name > 40 && $name =~ /^(.{20,70}?)\./) { - $name = $1; - } elsif (length $name > 55 && $name =~ /^(.{40,55})\s/) { - $name = $1; - } - ok( !ArgParser::parseCommandLineArguments(\@args), - "Expected parse failure: $name"); - isOutput('', $expectedError, $name); -} - -sub parseSuccess { - my ($optstring, $testname) = @_; - my @args = split(/\s+/, $optstring); - ok( ArgParser::parseCommandLineArguments(\@args), - "Expected parse success: $testname"); - isOutput('', '', $testname); -} - -sub testStringOption { - &setUpParseTest(); - my $val; - setStringOption(['s'], \$val, 'foo'); - parseFail("-s", "Too few arguments for option 's'\.\n"); - ok( !defined $val, 'String value unset on failure' ); - parseSuccess("-s foo", "String option"); - ok( $val eq 'foo', "String value set" ); -} - -sub testIntegerOption { - &setUpParseTest(); - my $val; - setIntegerOption(['i'], \$val, 'foo'); - parseFail("-i", "Too few arguments for option 'i'\.\n"); - ok( !defined $val, 'Integer value unset on failure' ); - parseFail("-i foo", "Invalid value 'foo' given to integer option 'i'\.\n"); - parseFail("-i 0.5", "Invalid value '0.5' given to integer option 'i'\.\n"); - parseSuccess("-i 5", "Integer option"); - ok( $val == 5, "Integer value set" ); - # Don't allow numbers as first char in id, so this can be detected as - # argument for integer. - parseSuccess("-i -8", "Negative integer option"); - ok( $val == -8, "Integer value set" ); - # Test big numbers - parseSuccess("-i 8000000000", "Big integer option"); - ok( $val / 1000000 == 8000, "Integer value set" ); - parseSuccess("-i -8000000000", "Big negative integer option"); - ok( $val / 1000000 == -8000, "Integer value set" ); -} - -sub testHostOption { - &setUpParseTest(); - my $val; - setHostOption(['h'], \$val, 'foo'); - parseFail("-h", "Too few arguments for option 'h'\.\n"); - ok( !defined $val, 'Host value unset on failure' ); - parseFail("-h 5", "Invalid host '5' given to option 'h'\. Not a valid host\n"); - parseFail("-h non.existing.host.no", "Invalid host 'non.existing.host.no' given to option 'h'\. Not a valid host\n"); - parseSuccess("-h localhost", "Host option set"); - is( $val, 'localhost', 'Host value set' ); -} - -sub testPortOption { - &setUpParseTest(); - my $val; - setPortOption(['p'], \$val, 'foo'); - parseFail("-p", "Too few arguments for option 'p'\.\n"); - ok( !defined $val, 'Host value unset on failure' ); - parseFail("-p -1", "Invalid value '-1' given to port option 'p'\. Must be an unsigned 16 bit\ninteger\.\n"); - parseFail("-p 65536", "Invalid value '65536' given to port option 'p'\. Must be an unsigned 16 bit\ninteger\.\n"); - parseSuccess("-p 65535", "Port option set"); - is( $val, 65535, 'Port value set' ); -} - -sub testFlagOption { - &setUpParseTest(); - my $val; - setFlagOption(['f'], \$val, 'foo'); - setFlagOption(['g'], \$val2, 'foo', OPTION_INVERTEDFLAG); - parseFail("-f 3", "Unhandled argument '3'\.\n"); - parseSuccess("-f", "First flag option set"); - is( $val, 1, 'Flag value set' ); - is( $val2, 1, 'Flag value set' ); - parseSuccess("-f", "First flag option reset"); - is( $val, 1, 'Flag value set' ); - is( $val2, 1, 'Flag value set' ); - parseSuccess("-g", "Second flag option set"); - is( $val, 0, 'Flag value set' ); - is( $val2, 0, 'Flag value set' ); - parseSuccess("-fg", "Both flag options set"); - is( $val, 1, 'Flag value set' ); - is( $val2, 0, 'Flag value set' ); -} - -sub testCountOption { - &setUpParseTest(); - my $val; - setUpCountingOption(['u'], \$val, 'foo'); - setDownCountingOption(['d'], \$val, 'foo'); - parseSuccess("", "Count not set"); - ok( !defined $val, 'Count value not set if not specified' ); - parseSuccess("-u", "Counting undefined"); - is( $val, 1, 'Count value set' ); - parseSuccess("-d", "Counting undefined - down"); - is( $val, -1, 'Count value set' ); - parseSuccess("-uuuud", "Counting both ways"); - is( $val, 3, 'Count value set' ); -} - -sub testComplexParsing { - &setUpParseTest(); - my $count; - my $int; - my $string; - setUpCountingOption(['u', 'up'], \$count, 'foo'); - setIntegerOption(['i', 'integer'], \$int, 'bar'); - setStringOption(['s', 'string'], \$string, 'baz'); - parseSuccess("-uis 3 foo", "Complex parsing managed"); - is( $count, 1, 'count counted' ); - is( $int, 3, 'integer set' ); - is( $string, 'foo', 'string set' ); - parseSuccess("-uiusi 3 foo 5", "Complex parsing managed 2"); - is( $count, 2, 'count counted' ); - is( $int, 5, 'integer set' ); - is( $string, 'foo', 'string set' ); - parseSuccess("-s -i foo -u 3", "Complex parsing managed 3"); - is( $count, 1, 'count counted' ); - is( $int, 3, 'integer set' ); - is( $string, 'foo', 'string set' ); -} - -sub testArguments { - &testOptionalArgument(); - &testRequiredArgument(); - &testRequiredArgumentAfterOptional(); -} - -sub testOptionalArgument { - &setUpParseTest(); - my $val; - setArgument(\$val, "Name", "Description"); - parseSuccess("", "Unset optional argument"); - ok( !defined $val, "Argument unset if not specified" ); - parseSuccess("myval", "Optional argument set"); - is( $val, 'myval', 'Optional argument set to correct value' ); -} - -sub testRequiredArgument { - &setUpParseTest(); - my $val; - setArgument(\$val, "Name", "Description", OPTION_REQUIRED); - parseFail("", "Argument Name is required but not specified\.\n"); - ok( !defined $val, "Argument unset on failure" ); - parseSuccess("myval", "Required argument set"); - is( $val, 'myval', 'Required argument set to correct value' ); -} - -sub testRequiredArgumentAfterOptional { - &setUpParseTest(); - my ($val, $val2); - setArgument(\$val, "Name", "Description"); - eval { - setArgument(\$val2, "Name2", "Description2", OPTION_REQUIRED); - }; - like( $@, qr/Cannot add required argument after optional/, - 'Fails adding required arg after optional' ); -} diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/Bin/GetClusterStateTest.pl b/vespaclient/src/perl/test/Yahoo/Vespa/Bin/GetClusterStateTest.pl deleted file mode 100644 index 6e2ec20c712..00000000000 --- a/vespaclient/src/perl/test/Yahoo/Vespa/Bin/GetClusterStateTest.pl +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -use Test::More; -use strict; -use warnings; - -BEGIN { use_ok( 'Yahoo::Vespa::Bin::GetClusterState' ); } -require_ok( 'Yahoo::Vespa::Bin::GetClusterState' ); - -use TestUtils::VespaTest; -use Yahoo::Vespa::Mocks::ClusterControllerMock; -use Yahoo::Vespa::Mocks::VespaModelMock; - -# Set which application is called on assertRun / assertRunMatches calls -setApplication( \&getClusterState ); - -useColors(0); - -&testSimple(); -&testSyntaxPage(); -&testClusterDown(); - -done_testing(); - -exit(0); - -sub testSimple { - my $stdout = <<EOS; - -Cluster books: -books/storage/0: down -books/storage/1: up - -Cluster music: -music/distributor/0: down -music/distributor/1: up -music/storage/0: retired -EOS - assertRun("Default - no arguments", "", 0, $stdout, ""); -} - -sub testClusterDown { - Yahoo::Vespa::Mocks::ClusterControllerMock::setClusterDown(); - Yahoo::Vespa::ClusterController::init(); - Yahoo::Vespa::Bin::GetClusterState::init(); - my $stdout = <<EOS; - -Cluster books: -books/storage/0: down -books/storage/1: up - -Cluster music is down. Too few nodes available. -music/distributor/0: down -music/distributor/1: up -music/storage/0: retired -EOS - assertRun("Music cluster down", "", 0, $stdout, ""); -} - -sub testSyntaxPage { - my $stdout = <<EOS; -EOS - my $pat = qr/^Get the cluster state of a given cluster.*Usage:.*GetClusterState.*Options.*--help.*/s; - assertRunMatches("Syntax page", "--help", 1, $pat, qr/^$/); -} diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/Bin/GetNodeStateTest.pl b/vespaclient/src/perl/test/Yahoo/Vespa/Bin/GetNodeStateTest.pl deleted file mode 100644 index 626be8d37e7..00000000000 --- a/vespaclient/src/perl/test/Yahoo/Vespa/Bin/GetNodeStateTest.pl +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -use Test::More; -use strict; -use warnings; - -BEGIN { use_ok( 'Yahoo::Vespa::Bin::GetNodeState' ); } -require_ok( 'Yahoo::Vespa::Bin::GetNodeState' ); - -use TestUtils::VespaTest; -use Yahoo::Vespa::Mocks::ClusterControllerMock; -use Yahoo::Vespa::Mocks::VespaModelMock; - -useColors(0); - -# Set which application is called on assertRun / assertRunMatches calls -setApplication( \&getNodeState ); - -&testSimple(); -&testSyntaxPage(); -&testRetired(); - -done_testing(); - -exit(0); - -sub testSimple { - my $stdout = <<EOS; -Shows the various states of one or more nodes in a Vespa Storage cluster. There -exist three different type of node states. They are: - - Unit state - The state of the node seen from the cluster controller. - User state - The state we want the node to be in. By default up. Can be - set by administrators or by cluster controller when it - detects nodes that are behaving badly. - Generated state - The state of a given node in the current cluster state. - This is the state all the other nodes know about. This - state is a product of the other two states and cluster - controller logic to keep the cluster stable. - -books/storage.0: -Unit: down: Not in slobrok -Generated: down: Not seen -User: down: default - -music/distributor.0: -Unit: up: Now reporting state U -Generated: down: Setting it down -User: down: Setting it down -EOS - assertRun("Default - no arguments", "", 0, $stdout, ""); -} - -sub testRetired { - setLocalHost("other.host.yahoo.com"); - my $stdout = <<EOS; - -music/storage.0: -Unit: up: Now reporting state U -Generated: retired: Stop using -User: retired: Stop using -EOS - assertRun("Other node", "-c music -t storage -i 0 -s", 0, $stdout, ""); -} - -sub testSyntaxPage { - my $stdout = <<EOS; -EOS - my $pat = qr/^Retrieve the state of one or more.*Usage:.*GetNodeState.*Options.*--help.*/s; - assertRunMatches("Syntax page", "--help", 1, $pat, qr/^$/); -} diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/Bin/SetNodeStateTest.pl b/vespaclient/src/perl/test/Yahoo/Vespa/Bin/SetNodeStateTest.pl deleted file mode 100644 index 7546c8f4ef6..00000000000 --- a/vespaclient/src/perl/test/Yahoo/Vespa/Bin/SetNodeStateTest.pl +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -use Test::More; -use strict; -use warnings; - -BEGIN { use_ok( 'Yahoo::Vespa::Bin::SetNodeState' ); } -require_ok( 'Yahoo::Vespa::Bin::SetNodeState' ); - -use TestUtils::VespaTest; -use Yahoo::Vespa::Mocks::ClusterControllerMock; -use Yahoo::Vespa::Mocks::VespaModelMock; - -# Set which application is called on assertRun / assertRunMatches calls -setApplication( \&setNodeState ); - -&testSimple(); -&testSyntaxPage(); -&testHelp(); -&testDownState(); -&testDownFailure(); -&testDefaultMaintenanceFails(); -&testForcedMaintenanceSucceeds(); - -done_testing(); - -exit(0); - -sub testSimple { - my $stdout = <<EOS; -Set user state for books/storage/0 to 'up' with reason '' -Set user state for music/distributor/0 to 'up' with reason '' -EOS - assertRun("Default - Min arguments", "up", 0, $stdout, ""); -} - -sub testSyntaxPage { - my $stdout = <<EOS; -EOS - my $pat = qr/^Set the user state of a node.*Usage:.*SetNodeState.*Arguments:.*Options:.*--help.*/s; - assertRunMatches("Syntax page", "--help", 1, $pat, qr/^$/); -} - -sub testHelp { - my $stdout = <<EOS; -Set the user state of a node. This will set the generated state to the user -state if the user state is "better" than the generated state that would have -been created if the user state was up. For instance, a node that is currently -in initializing state can be forced into down state, while a node that is -currently down can not be forced into retired state, but can be forced into -maintenance state. - -Usage: SetNodeStateTest.pl [Options] <Wanted State> [Description] - -Arguments: - Wanted State : User state to set. This must be one of up, down, maintenance or - retired. - Description : Give a reason for why you are altering the user state, which - will show up in various admin tools. (Use double quotes to give - a reason with whitespace in it) - -Options: - -h --help : Show this help page. - -v : Create more verbose output. - -s : Create less verbose output. - --show-hidden : Also show hidden undocumented debug options. - -Options related to operation visibility: - -n --no-wait : Do not wait for node state changes to be visible in - the cluster before returning. - -Node selection options. By default, nodes running locally will be selected: - -c --cluster : Cluster name. If unspecified, - and vespa is installed on current node, information - will be attempted auto-extracted - -f --force : Force execution - -t --type : Node type - can either be 'storage' or - 'distributor'. If not specified, the operation will - use state for both types. - -i --index : Node index. If not specified, - all nodes found running on this host will be used. - -Config retrieval options: - --config-server : Host name of config server to query - --config-server-port : Port to connect to config server on - --config-request-timeout : Timeout of config request -EOS - - assertRun("Help text", "-h", 1, $stdout, ""); -} - -sub testDownState { - my $stdout = <<EOS; -Set user state for books/storage/0 to 'down' with reason 'testing' -Set user state for music/distributor/0 to 'down' with reason 'testing' -EOS - assertRun("Down state", "down testing", 0, $stdout, ""); -} - -sub testDownFailure { - $Yahoo::Vespa::Mocks::ClusterControllerMock::forceInternalServerError = 1; - - my $stderr = <<EOS; -Failed to set node state for node books/storage/0: 500 Internal Server Error -(forced) -EOS - - assertRun("Down failure", "--nocolors down testing", 1, "", $stderr); - - $Yahoo::Vespa::Mocks::ClusterControllerMock::forceInternalServerError = 0; -} - -sub testDefaultMaintenanceFails { - my $stderr = <<EOS; -Setting the distributor to maintenance mode may have severe consequences for -feeding! -Please specify -t storage to only set the storage node to maintenance mode, or --f to override this error. -EOS - - assertRun("Default maintenance fails", "--nocolors maintenance testing", - 1, "", $stderr); -} - -sub testForcedMaintenanceSucceeds { - my $stdout = <<EOS; -Set user state for books/storage/0 to 'maintenance' with reason 'testing' -Set user state for music/distributor/0 to 'maintenance' with reason 'testing' -EOS - - assertRun("Forced maintenance succeeds", "-f maintenance testing", - 0, $stdout, ""); -} diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/ClusterControllerTest.pl b/vespaclient/src/perl/test/Yahoo/Vespa/ClusterControllerTest.pl deleted file mode 100644 index a833fad4087..00000000000 --- a/vespaclient/src/perl/test/Yahoo/Vespa/ClusterControllerTest.pl +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -use Test::More; -use Data::Dumper; - -BEGIN { use_ok( 'Yahoo::Vespa::ClusterController' ); } -require_ok( 'Yahoo::Vespa::ClusterController' ); - -use TestUtils::OutputCapturer; -use Yahoo::Vespa::Mocks::ClusterControllerMock; -use Yahoo::Vespa::Mocks::VespaModelMock; - -Yahoo::Vespa::ConsoleOutput::setVerbosity(0); # Squelch output when running test -detectClusterController(); -Yahoo::Vespa::ConsoleOutput::setVerbosity(3); - -my $cclist = Yahoo::Vespa::ClusterController::getClusterControllers(); -is( scalar @$cclist, 1, "Cluster controllers detected" ); -is( $$cclist[0]->host, 'testhost.yahoo.com', 'Host autodetected' ); -is( $$cclist[0]->port, 19050, 'Port autodetected' ); - -is( join (' - ', Yahoo::Vespa::ClusterController::listContentClusters()), - "music - books", 'Content clusters' ); - -my $state = getContentClusterState('music'); - -$Data::Dumper::Indent = 1; -# print Dumper($state); - -is( $state->globalState, 'up', 'Generated state for music' ); - -is( $state->distributor->{'0'}->unit->state, 'up', 'Unit state for music' ); -is( $state->distributor->{'1'}->unit->state, 'up', 'Unit state for music' ); -is( $state->storage->{'0'}->unit->state, 'up', 'Unit state for music' ); -is( $state->storage->{'1'}->unit->state, 'up', 'Unit state for music' ); -is( $state->distributor->{'0'}->generated->state, 'down', 'Generated state' ); -is( $state->distributor->{'1'}->generated->state, 'up', 'Generated state' ); -is( $state->storage->{'0'}->generated->state, 'retired', 'Generated state' ); -is( $state->storage->{'1'}->generated->state, 'up', 'Generated state' ); -is( $state->distributor->{'0'}->user->state, 'down', 'User state' ); -is( $state->distributor->{'1'}->user->state, 'up', 'User state' ); -is( $state->storage->{'0'}->user->state, 'retired', 'User state' ); -is( $state->storage->{'1'}->user->state, 'up', 'User state' ); - -is( $state->storage->{'1'}->unit->reason, 'Now reporting state U', 'Reason' ); - -done_testing(); - -exit(0); diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/ConsoleOutputTest.pl b/vespaclient/src/perl/test/Yahoo/Vespa/ConsoleOutputTest.pl deleted file mode 100644 index 2089bf80d8e..00000000000 --- a/vespaclient/src/perl/test/Yahoo/Vespa/ConsoleOutputTest.pl +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -use Test::More; - -BEGIN { use_ok( 'Yahoo::Vespa::ConsoleOutput' ); } -require_ok( 'Yahoo::Vespa::ConsoleOutput' ); - -ok( Yahoo::Vespa::ConsoleOutput::getVerbosity() == 3, - 'Default verbosity is 3' ); -ok( Yahoo::Vespa::ConsoleOutput::usingAnsiColors(), - 'Using ansi colors by default' ); - -use TestUtils::VespaTest; - -printSpam "test\n"; -isOutput('', '', "No spam at level 3"); - -printDebug "test\n"; -isOutput('', '', "No spam at level 3"); - -printInfo "info test\n"; -isOutput("info test\n", '', "Info at level 3"); - -printWarning "foo\n"; -isOutput("", "\e[93mfoo\e[0m\n", "Stderr output for warning"); - -useColors(0); -printWarning "foo\n"; -isOutput("", "foo\n", "Stderr output without ansi colors"); - -Yahoo::Vespa::ConsoleOutput::setVerbosity(4); -printSpam "test\n"; -isOutput('', '', "No spam at level 4"); - -printDebug "test\n"; -isOutput("debug: test\n", '', "Debug at level 4"); - -Yahoo::Vespa::ConsoleOutput::setVerbosity(5); -printSpam "test\n"; -isOutput("spam: test\n", '', "Spam at level 5"); - -printInfo "info test\n"; -isOutput("info: info test\n", '', "Type prefix at high verbosity"); - -done_testing(); - -exit(0); diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/HttpTest.pl b/vespaclient/src/perl/test/Yahoo/Vespa/HttpTest.pl deleted file mode 100644 index 1e3e97bb829..00000000000 --- a/vespaclient/src/perl/test/Yahoo/Vespa/HttpTest.pl +++ /dev/null @@ -1,140 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# -# Tests of the Http wrapper library.. -# -# NOTE: Test server set up does not support content not ending in newline. -# - -use strict; -use Test::More; -use Yahoo::Vespa::Mocks::HttpServerMock; - -BEGIN { - use_ok( 'Yahoo::Vespa::Http' ); - *Http:: = *Yahoo::Vespa::Http:: -} -require_ok( 'Yahoo::Vespa::Http' ); - -my $httpTestServerPort = setupTestHttpServer(); -ok(defined $httpTestServerPort, "Test server set up"); - -&testSimpleGet(); -&testAdvancedGet(); -&testFailingGet(); -&testSimplePost(); -&testJsonReturnInPost(); - -done_testing(); - -exit(0); - -sub filterRequest { - my ($request) = @_; - $request =~ s/\r//g; - $request =~ s/(Content-Length:\s*)\d+/$1##/g; - $request =~ s/(Host: localhost:)\d+/$1##/g; - $request =~ s/(?:Connection|TE|Client-[^:]+):[^\n]*\n//g; - - return $request; -} - -sub testSimpleGet { - my %r = Http::get('localhost', $httpTestServerPort, '/foo'); - is( $r{'code'}, 200, "Get request code" ); - is( $r{'status'}, 'OK', "Get request status" ); - - my $expected = <<EOS; -HTTP/1.1 200 OK -Content-Length: ## -Content-Type: text/plain; charset=utf-8 - -GET /foo HTTP/1.1 -Host: localhost:## -User-Agent: Vespa-perl-script -EOS - is( &filterRequest($r{'all'}), $expected, 'Get result' ); -} - -sub testAdvancedGet { - my @headers = ("X-Foo" => 'Bar'); - my @uri_param = ("uricrap" => 'special=?&%value', - "other" => 'hmm'); - my %r = Http::request('GET', 'localhost', $httpTestServerPort, '/foo', - \@uri_param, undef, \@headers); - is( $r{'code'}, 200, "Get request code" ); - is( $r{'status'}, 'OK', "Get request status" ); - - my $expected = <<EOS; -HTTP/1.1 200 OK -Content-Length: ## -Content-Type: text/plain; charset=utf-8 - -GET /foo?uricrap=special%3D%3F%26%25value&other=hmm HTTP/1.1 -Host: localhost:## -User-Agent: Vespa-perl-script -X-Foo: Bar -EOS - is( &filterRequest($r{'all'}), $expected, 'Get result' ); -} - -sub testFailingGet { - my @uri_param = ("code" => '501', - "status" => 'Works'); - my %r = Http::request('GET', 'localhost', $httpTestServerPort, '/foo', - \@uri_param); - is( $r{'code'}, 501, "Get request code" ); - is( $r{'status'}, 'Works', "Get request status" ); - - my $expected = <<EOS; -HTTP/1.1 501 Works -Content-Length: ## -Content-Type: text/plain; charset=utf-8 - -GET /foo?code=501&status=Works HTTP/1.1 -Host: localhost:## -User-Agent: Vespa-perl-script -EOS - is( &filterRequest($r{'all'}), $expected, 'Get result' ); -} - -sub testSimplePost { - my @uri_param = ("uricrap" => 'Rrr' ); - my %r = Http::request('POST', 'localhost', $httpTestServerPort, '/foo', - \@uri_param, "Some content\n"); - is( $r{'code'}, 200, "Get request code" ); - is( $r{'status'}, 'OK', "Get request status" ); - - my $expected = <<EOS; -HTTP/1.1 200 OK -Content-Length: ## -Content-Type: text/plain; charset=utf-8 - -POST /foo?uricrap=Rrr HTTP/1.1 -Host: localhost:## -User-Agent: Vespa-perl-script -Content-Length: ## -Content-Type: application/x-www-form-urlencoded - -Some content -EOS - is( &filterRequest($r{'all'}), $expected, 'Get result' ); -} - -sub testJsonReturnInPost -{ - my @uri_param = ("contenttype" => 'application/json' ); - my $json = "{ \"key\" : \"value\" }\n"; - my %r = Http::request('POST', 'localhost', $httpTestServerPort, '/foo', - \@uri_param, $json); - is( $r{'code'}, 200, "Get request code" ); - is( $r{'status'}, 'OK', "Get request status" ); - - my $expected = <<EOS; -HTTP/1.1 200 OK -Content-Length: ## -Content-Type: application/json - -{ "key" : "value" } -EOS - is( &filterRequest($r{'all'}), $expected, 'Get json result' ); -} diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/JsonTest.pl b/vespaclient/src/perl/test/Yahoo/Vespa/JsonTest.pl deleted file mode 100644 index b0eb7962b33..00000000000 --- a/vespaclient/src/perl/test/Yahoo/Vespa/JsonTest.pl +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# -# Tests of the Json wrapper library.. -# - -use Test::More; - -use strict; - -BEGIN { - use_ok( 'Yahoo::Vespa::Json' ); - *Json:: = *Yahoo::Vespa::Json:: # Alias namespace -} -require_ok( 'Yahoo::Vespa::Json' ); - -&testSimpleJson(); - -done_testing(); - -exit(0); - -sub testSimpleJson { - my $json = <<EOS; -{ - "foo" : "bar", - "map" : { - "abc" : "def", - "num" : 13.0 - }, - "array" : [ - { "val1" : 3 }, - { "val2" : 6 } - ] -} -EOS - my $parsed = Json::parse($json); - is( $parsed->{'foo'}, 'bar', 'json test 1' ); - is( $parsed->{'map'}->{'abc'}, 'def', 'json test 2' ); - is( $parsed->{'map'}->{'num'}, 13.0, 'json test 3' ); - my $prettyPrint = <<EOS; -{ - "array" : [ - { - "val1" : 3 - }, - { - "val2" : 6 - } - ], - "map" : { - "num" : 13, - "abc" : "def" - }, - "foo" : "bar" -} -EOS - is( Json::encode($parsed), $prettyPrint, 'simple json test - encode' ); - my @keys = sort keys %{$parsed->{'map'}}; - is( scalar @keys, 2, 'simple json test - map keys' ); - is( $keys[0], 'abc', 'simple json test - map key 1' ); - is( $keys[1], 'num', 'simple json test - map key 2' ); - - @keys = @{ $parsed->{'array'} }; - is( scalar @keys, 2, 'simple json test - list keys' ); - is( $keys[0]->{'val1'}, 3, 'simple json test - list key 1' ); - is( $keys[1]->{'val2'}, 6, 'simple json test - list key 2' ); -} diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/ClusterControllerMock.pm b/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/ClusterControllerMock.pm deleted file mode 100644 index cb642dbc629..00000000000 --- a/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/ClusterControllerMock.pm +++ /dev/null @@ -1,258 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package Yahoo::Vespa::Mocks::ClusterControllerMock; - -use strict; -use warnings; -use URI::Escape; -use Yahoo::Vespa::ConsoleOutput; -use Yahoo::Vespa::Mocks::HttpClientMock; -use Yahoo::Vespa::Utils; - -BEGIN { - use base 'Exporter'; - our @EXPORT = qw( - ); -} - -our $forceInternalServerError = 0; - -# Register a handler in the Http Client mock -registerHttpClientHandler(\&handleCCRequest); - -our $clusterListJson = <<EOS; -{ - "cluster" : { - "books" : { - "link" : "/cluster/v2/books" - }, - "music" : { - "link" : "/cluster/v2/music" - } - } -} -EOS -our $musicClusterJson = <<EOS; -{ - "state" : { - "generated" : { - "state" : "up", - "reason" : "" - } - }, - "service" : { - "distributor" : { - "node" : { - "0" : { - "attributes" : { "hierarchical-group" : "top" }, - "state" : { - "generated" : { "state" : "down", "reason" : "Setting it down" }, - "unit" : { "state" : "up", "reason" : "Now reporting state U" }, - "user" : { "state" : "down", "reason" : "Setting it down" } - } - }, - "1" : { - "attributes" : { "hierarchical-group" : "top" }, - "state" : { - "generated" : { "state" : "up", "reason" : "Setting it up" }, - "unit" : { "state" : "up", "reason" : "Now reporting state U" }, - "user" : { "state" : "up", "reason" : "" - } - } - } - } - }, - "storage" : { - "node" : { - "0" : { - "attributes" : { "hierarchical-group" : "top" }, - "state" : { - "generated" : { "state" : "retired", "reason" : "Stop using" }, - "unit" : { "state" : "up", "reason" : "Now reporting state U" }, - "user" : { "state" : "retired", "reason" : "Stop using" } - }, - "partition" : { - "0" : { - "metrics" : { - "bucket-count" : 5, - "unique-document-count" : 10, - "unique-document-total-size" : 1000 - } - } - } - }, - "1" : { - "attributes" : { "hierarchical-group" : "top" }, - "state" : { - "generated" : { "state" : "up", "reason" : "Setting it up" }, - "unit" : { "state" : "up", "reason" : "Now reporting state U" }, - "user" : { "state" : "up", "reason" : "" - } - }, - "partition" : { - "0" : { - "metrics" : { - "bucket-count" : 50, - "unique-document-count" : 100, - "unique-document-total-size" : 10000 - } - } - } - } - } - } - } -} -EOS -our $booksClusterJson = <<EOS; -{ - "state" : { - "generated" : { - "state" : "up", - "reason" : "" - } - }, - "service" : { - "distributor" : { - "node" : { - "0" : { - "attributes" : { "hierarchical-group" : "top.g1" }, - "state" : { - "generated" : { "state" : "down", "reason" : "Setting it down" }, - "unit" : { "state" : "up", "reason" : "Now reporting state U" }, - "user" : { "state" : "down", "reason" : "Setting it down" } - } - }, - "1" : { - "attributes" : { "hierarchical-group" : "top.g2" }, - "state" : { - "generated" : { "state" : "up", "reason" : "Setting it up" }, - "unit" : { "state" : "up", "reason" : "Now reporting state U" }, - "user" : { "state" : "up", "reason" : "" - } - } - } - } - }, - "storage" : { - "node" : { - "0" : { - "attributes" : { "hierarchical-group" : "top.g1" }, - "state" : { - "generated" : { "state" : "down", "reason" : "Not seen" }, - "unit" : { "state" : "down", "reason" : "Not in slobrok" }, - "user" : { "state" : "down", "reason" : "default" } - } - }, - "1" : { - "attributes" : { "hierarchical-group" : "top.g2" }, - "state" : { - "generated" : { "state" : "up", "reason" : "Setting it up" }, - "unit" : { "state" : "up", "reason" : "Now reporting state U" }, - "user" : { "state" : "up", "reason" : "" - } - } - } - } - } - } -} -EOS - -return &init(); - -sub init { - #print "Verifying that cluster list json is parsable.\n"; - my $json = Json::parse($clusterListJson); - #print "Verifying that music json is parsable\n"; - $json = Json::parse($musicClusterJson); - #print "Verifying that books json is parsable\n"; - $json = Json::parse($booksClusterJson); - #print "All seems parsable.\n"; - return 1; -} - -sub setClusterDown { - $musicClusterJson =~ s/"up"/"down"/; - $musicClusterJson =~ s/""/"Not enough nodes up"/; - #print "Cluster state: $musicClusterJson\n"; - #print "Verifying that music json is parsable\n"; - my $json = Json::parse($musicClusterJson); -} - -sub handleCCRequest { # (Type, Host, Port, Path, ParameterMap, Content, Headers) - my ($type, $host, $port, $path, $params, $content, $headers) = @_; - my %paramHash; - if (defined $params) { - %paramHash = @$params; - } - if ($forceInternalServerError) { - printDebug "Forcing internal server error response\n"; - return ( - 'code' => 500, - 'status' => 'Internal Server Error (forced)' - ); - } - if ($path eq "/cluster/v2/") { - printDebug "Handling cluster list request\n"; - return ( - 'code' => 200, - 'status' => 'OK', - 'content' => $clusterListJson - ); - } - if ($path eq "/cluster/v2/music/" - && (exists $paramHash{'recursive'} - && $paramHash{'recursive'} eq 'true')) - { - printDebug "Handling cluster music state request\n"; - return ( - 'code' => 200, - 'status' => 'OK', - 'content' => $musicClusterJson - ); - } - if ($path eq "/cluster/v2/books/" - && (exists $paramHash{'recursive'} - && $paramHash{'recursive'} eq 'true')) - { - printDebug "Handling cluster books state request\n"; - return ( - 'code' => 200, - 'status' => 'OK', - 'content' => $booksClusterJson - ); - } - if ($path =~ /^\/cluster\/v2\/(books|music)\/(storage|distributor)\/(\d+)$/) - { - my ($cluster, $service, $index) = ($1, $2, $3); - my $json = Json::parse($content); - my $state = $json->{'state'}->{'user'}->{'state'}; - my $description = $json->{'state'}->{'user'}->{'reason'}; - if (!defined $description && $state eq 'up') { - $description = ""; - } - if ($state !~ /^(?:up|down|maintenance|retired)$/) { - return ( - 'code' => 500, - 'status' => "Unknown state '$state' specified" - ); - } - if (!defined $state || !defined $description) { - return ( - 'code' => 500, - 'status' => "Invalid form data or failed parsing: '$content'" - ); - } - printDebug "Handling set user state request $cluster/$service/$index"; - return ( - 'code' => 200, - 'status' => "Set user state for $cluster/$service/$index to " - . "'$state' with reason '$description'" - ); - } - printDebug "Request to '$path' not matched. Params:\n"; - foreach my $key (keys %paramHash) { - printDebug " $key => '$paramHash{$key}'\n"; - } - return; -} diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/HttpClientMock.pm b/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/HttpClientMock.pm deleted file mode 100644 index 2840da9899a..00000000000 --- a/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/HttpClientMock.pm +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# -# Switched the backend implementation of the Vespa::Http library, such that -# requests are sent here rather than onto the network. Register handlers here -# to respond to requests. -# -# Handlers are called in sequence until one of them returns a defined result. -# If none do, return a generic failure. -# - -package Yahoo::Vespa::Mocks::HttpClientMock; - -use strict; -use warnings; -use Yahoo::Vespa::ConsoleOutput; -use Yahoo::Vespa::Http; - -BEGIN { # - Define default exports for module - use base 'Exporter'; - our @EXPORT = qw( - registerHttpClientHandler - ); -} - -my @HANDLERS; - -&initialize(); - -return 1; - -#################### Default exported functions ############################# - -sub registerHttpClientHandler { # (Handler) - push @HANDLERS, $_[0]; -} - -##################### Internal utility functions ########################## - -sub initialize { # () - Yahoo::Vespa::Http::setHttpExecutor(\&clientMock); -} -sub clientMock { # (HttpRequest to forward) -> Response - foreach my $handler (@HANDLERS) { - my %result = &$handler(@_); - if (exists $result{'code'}) { - return %result; - } - } - return ( - 'code' => 500, - 'status' => 'No client handler for given request', - 'content' => '', - 'all' => '' - ); -} diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/HttpServerMock.pm b/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/HttpServerMock.pm deleted file mode 100644 index 00a6f1526b2..00000000000 --- a/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/HttpServerMock.pm +++ /dev/null @@ -1,156 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# -# A mock of an HTTP server, such that HTTP client library can be tested. -# -# Known limitations: -# - Does line by line reading of TCP data, so the content part of the HTML -# request has to end in a newline, otherwise, the server will block waiting -# for more data. -# -# Default connection handler: -# - If no special case, server returns request 200 OK, with the complete -# client request as text/plain utf8 content. -# - If request matches contenttype=\S+ (Typically due to setting a URI -# parameter), the response will contain the content of the request with the -# given content type set. -# - If request matches code=\d+ (Typically due to setting a URI parameter), -# the response will use that return code. -# - If request matches status=\S+ (Typically due to setting a URI parameter), -# the response will use that status line -# - -package Yahoo::Vespa::Mocks::HttpServerMock; - -use strict; -use warnings; -use IO::Socket::IP; -use URI::Escape; - -BEGIN { # - Set up exports for module - use base 'Exporter'; - our @EXPORT = qw( - setupTestHttpServer - ); -} - -my $HTTP_TEST_SERVER; -my $HTTP_TEST_SERVER_PORT; -my $HTTP_TEST_SERVER_PID; -my $CONNECTION_HANDLER = \&defaultConnectionHandler; - -END { # - Kill forked HTTP handler process on exit - if (defined $HTTP_TEST_SERVER_PID) { - kill(9, $HTTP_TEST_SERVER_PID); - } -} - -return 1; - -####################### Default exported functions ############################ - -sub setupTestHttpServer { # () -> HttpServerPort - my $portfile = "/tmp/vespaclient.$$.perl.httptestserverport"; - unlink($portfile); - my $pid = fork(); - if ($pid == 0) { - $HTTP_TEST_SERVER = IO::Socket::IP->new( - 'Proto' => 'tcp', - 'LocalPort' => 0, - 'Listen' => SOMAXCONN, - 'ReuseAddr' => 1, - ); - # print "Started server listening to port " . $HTTP_TEST_SERVER->sockport() - # . "\n"; - my $fh; - open ($fh, ">$portfile") or die "Failed to write port used to file."; - print $fh "<" . $HTTP_TEST_SERVER->sockport() . ">"; - close $fh; - defined $HTTP_TEST_SERVER or die "Failed to set up test HTTP server"; - while (1) { - &$CONNECTION_HANDLER(); - } - exit(0); - } else { - $HTTP_TEST_SERVER_PID = $pid; - while (1) { - if (-e $portfile) { - my $port = `cat $portfile`; - chomp $port; - if (defined $port && $port =~ /\<(\d+)\>/) { - #print "Client using port $1\n"; - $HTTP_TEST_SERVER_PORT = $1; - last; - } - } - sleep(0.01); - } - } - unlink($portfile); - return $HTTP_TEST_SERVER_PORT; -} - -####################### Internal utility functions ############################ - -sub defaultConnectionHandler { - my $client = $HTTP_TEST_SERVER->accept(); - defined $client or die "No connection to accept?"; - my $request; - my $line; - my $content_length = 0; - my $content_type; - while ($line = <$client>) { - if ($line =~ /^(.*?)\s$/) { - $line = $1; - } - if ($line =~ /Content-Length:\s(\d+)/) { - $content_length = $1; - } - if ($line =~ /contenttype=(\S+)/) { - $content_type = uri_unescape($1); - } - #print "Got line '$line'\n"; - if ($line eq '') { - last; - } - $request .= $line . "\n"; - } - if ($content_length > 0) { - $request .= "\n"; - if (defined $content_type) { - $request = ""; - } - my $read = 0; - while ($line = <$client>) { - $read += length $line; - if ($line =~ /^(.*?)\s$/) { - $line = $1; - } - $request .= $line; - if ($read >= $content_length) { - last; - } - } - } - # print "Got request '$request'.\n"; - $request =~ s/\n/\r\n/g; - my $code = 200; - my $status = "OK"; - if ($request =~ /code=(\d+)/) { - $code = $1; - } - if ($request =~ /status=([A-Za-z0-9]+)/) { - $status = $1; - } - my $response = "HTTP/1.1 $code $status\n"; - if (defined $content_type) { - $response .= "Content-Type: $content_type\n"; - } else { - $response .= "Content-Type: text/plain; charset=utf-8\n"; - } - $response .= "Content-Length: " . (length $request) . "\n" - . "\n"; - $response =~ s/\n/\r\n/g; - $response .= $request; - print $client $response; - close $client; -} diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/VespaModelMock.pm b/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/VespaModelMock.pm deleted file mode 100644 index 63ff9f767d1..00000000000 --- a/vespaclient/src/perl/test/Yahoo/Vespa/Mocks/VespaModelMock.pm +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package Yahoo::Vespa::Mocks::VespaModelMock; - -use strict; -use warnings; -use Yahoo::Vespa::VespaModel; - -Yahoo::Vespa::VespaModel::setModelRetrievalFunction(\&getModelConfig); - -our $defaultModelConfig = <<EOS; -hosts[0].name "testhost.yahoo.com" -hosts[0].services[0].name "container-clustercontroller" -hosts[0].services[0].type "container-clustercontroller" -hosts[0].services[0].configid "admin/cluster-controllers/0" -hosts[0].services[0].clustertype "" -hosts[0].services[0].clustername "cluster-controllers" -hosts[0].services[0].index 0 -hosts[0].services[0].ports[0].number 19050 -hosts[0].services[0].ports[0].tags "state external query http" -hosts[0].services[0].ports[1].number 19100 -hosts[0].services[0].ports[1].tags "external http" -hosts[0].services[0].ports[2].number 19101 -hosts[0].services[0].ports[2].tags "messaging rpc" -hosts[0].services[0].ports[3].number 19102 -hosts[0].services[0].ports[3].tags "admin rpc" -hosts[0].services[1].name "distributor2" -hosts[0].services[1].type "distributor" -hosts[0].services[1].configid "music/distributor/0" -hosts[0].services[1].clustertype "content" -hosts[0].services[1].clustername "music" -hosts[0].services[1].index 0 -hosts[0].services[1].ports[0].number 19131 -hosts[0].services[1].ports[0].tags "messaging" -hosts[0].services[1].ports[1].number 19132 -hosts[0].services[1].ports[1].tags "status rpc" -hosts[0].services[1].ports[2].number 19133 -hosts[0].services[1].ports[2].tags "status http" -hosts[0].services[2].name "storagenode3" -hosts[0].services[2].type "storagenode" -hosts[0].services[2].configid "storage/storage/0" -hosts[0].services[2].clustertype "content" -hosts[0].services[2].clustername "books" -hosts[0].services[2].index 0 -hosts[0].services[2].ports[0].number 19134 -hosts[0].services[2].ports[0].tags "messaging" -hosts[0].services[2].ports[1].number 19135 -hosts[0].services[2].ports[1].tags "status rpc" -hosts[0].services[2].ports[2].number 19136 -hosts[0].services[2].ports[2].tags "status http" -hosts[1].name "other.host.yahoo.com" -hosts[1].services[0].name "distributor2" -hosts[1].services[0].type "distributor" -hosts[1].services[0].configid "music/distributor/1" -hosts[1].services[0].clustertype "content" -hosts[1].services[0].clustername "music" -hosts[1].services[0].index 1 -hosts[1].services[0].ports[0].number 19131 -hosts[1].services[0].ports[0].tags "messaging" -hosts[1].services[0].ports[1].number 19132 -hosts[1].services[0].ports[1].tags "status rpc" -hosts[1].services[0].ports[2].number 19133 -hosts[1].services[0].ports[2].tags "status http" -hosts[1].services[1].name "storagenode3" -hosts[1].services[1].type "storagenode" -hosts[1].services[1].configid "storage/storage/1" -hosts[1].services[1].clustertype "content" -hosts[1].services[1].clustername "books" -hosts[1].services[1].index 1 -hosts[1].services[1].ports[0].number 19134 -hosts[1].services[1].ports[0].tags "messaging" -hosts[1].services[1].ports[1].number 19135 -hosts[1].services[1].ports[1].tags "status rpc" -hosts[1].services[1].ports[2].number 19136 -hosts[1].services[1].ports[2].tags "status http" -hosts[1].services[2].name "storagenode2" -hosts[1].services[2].type "storagenode" -hosts[1].services[2].configid "storage/storage/0" -hosts[1].services[2].clustertype "content" -hosts[1].services[2].clustername "music" -hosts[1].services[2].index 0 -hosts[1].services[2].ports[0].number 19134 -hosts[1].services[2].ports[0].tags "messaging" -hosts[1].services[2].ports[1].number 19135 -hosts[1].services[2].ports[1].tags "status rpc" -hosts[1].services[2].ports[2].number 19136 -hosts[1].services[2].ports[2].tags "status http" - -EOS - -sub getModelConfig { - my @output = split(/\n/, $defaultModelConfig); - return @output; -} - -1; diff --git a/vespaclient/src/perl/test/Yahoo/Vespa/VespaModelTest.pl b/vespaclient/src/perl/test/Yahoo/Vespa/VespaModelTest.pl deleted file mode 100644 index d1c1a1bb0d9..00000000000 --- a/vespaclient/src/perl/test/Yahoo/Vespa/VespaModelTest.pl +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -use Test::More; -use Yahoo::Vespa::Mocks::VespaModelMock; - -BEGIN { - use_ok( 'Yahoo::Vespa::VespaModel' ); - *VespaModel:: = *Yahoo::Vespa::VespaModel:: ; -} -require_ok( 'Yahoo::Vespa::VespaModel' ); - -&testGetSocketForService(); -&testVisitServices(); - -done_testing(); - -exit(0); - -sub testGetSocketForService { - my $sockets = VespaModel::getSocketForService( - type => 'container-clustercontroller', tag => 'state'); - my ($host, $port) = ($$sockets[0]->{'host'}, $$sockets[0]->{'port'}); - is( $host, 'testhost.yahoo.com', "Host for state API" ); - is( $port, 19050, 'Port for state API' ); - $sockets = VespaModel::getSocketForService( - type => 'container-clustercontroller', tag => 'admin'); - ($host, $port) = ($$sockets[0]->{'host'}, $$sockets[0]->{'port'}); - is( $host, 'testhost.yahoo.com', "Host for state API" ); - is( $port, 19102, 'Port for state API' ); - $sockets = VespaModel::getSocketForService( - type => 'container-clustercontroller', tag => 'http'); - ($host, $port) = ($$sockets[0]->{'host'}, $$sockets[0]->{'port'}); - is( $port, 19100, 'Port for state API' ); - - $sockets = VespaModel::getSocketForService( - type => 'distributor', index => 0); - ($host, $port) = ($$sockets[0]->{'host'}, $$sockets[0]->{'port'}); - is( $host, 'testhost.yahoo.com', 'host for distributor 0' ); -} - -my @services; - -sub serviceCallback { - my ($info) = @_; - push @services, "Name($$info{'name'}) Type($$info{'type'}) " - . "Cluster($$info{'cluster'}) Host($$info{'host'}) " - . "Index($$info{'index'})"; -} - -sub testVisitServices { - @services = (); - VespaModel::visitServices(\&serviceCallback); - my $expected = <<EOS; -Name(storagenode3) Type(storagenode) Cluster(books) Host(testhost.yahoo.com) Index(0) -Name(storagenode3) Type(storagenode) Cluster(books) Host(other.host.yahoo.com) Index(1) -Name(container-clustercontroller) Type(container-clustercontroller) Cluster(cluster-controllers) Host(testhost.yahoo.com) Index(0) -Name(distributor2) Type(distributor) Cluster(music) Host(testhost.yahoo.com) Index(0) -Name(distributor2) Type(distributor) Cluster(music) Host(other.host.yahoo.com) Index(1) -Name(storagenode2) Type(storagenode) Cluster(music) Host(other.host.yahoo.com) Index(0) -EOS - chomp $expected; - is ( join("\n", @services), $expected, "Services visited correctly" ); -} diff --git a/vespaclient/src/perl/test/testrunner.pl b/vespaclient/src/perl/test/testrunner.pl deleted file mode 100644 index 8dca35ad91d..00000000000 --- a/vespaclient/src/perl/test/testrunner.pl +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/perl -w -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -# -# Searches around in test dir to find test binaries and run them. Sadly these -# seem to return exit code 0 on some failures for unknown reasons. To counter -# that the testrunner grabs the output of the test and triggers test to fail if -# it finds unexpected data in the output. -# -# Unit tests should mostly not write as this will clutter report, but if they -# want to write some status they have to write it so it does not trigger -# failure here. Use printTest in VespaTest suite to prefix all test output to -# something we match here. -# - -use strict; -use warnings; - -$| = 1; -my @files = `find . -name \*Test.pl`; -chomp @files; - -my $tempdir = `mktemp -d /tmp/mockup-vespahome-XXXXXX`; -chomp $tempdir; -$ENV{'VESPA_HOME'} = $tempdir . "/"; -mkdir "${tempdir}/libexec"; -mkdir "${tempdir}/libexec/vespa" or die "Cannot mkdir ${tempdir}/libexec/vespa\n"; -`touch ${tempdir}/libexec/vespa/common-env.sh`; - -my $pat; -if (exists $ENV{'TEST_SUBSET'}) { - $pat = $ENV{'TEST_SUBSET'}; -} - -my $failure_pattern = qr/(?:Tests were run but no plan was declared and done_testing\(\) was not seen)/; -my $accepted_pattern = qr/^(?:\s*|\d+\.\.\d+|ok\s+\d+\s+-\s+.*|Test: .*|.*spam: .*)$/; - -my $failures = 0; -foreach my $file (@files) { - $file =~ /^(?:\.\/)?(.*)\.pl$/ or die "Strange file name '$file'."; - my $test = $1; - if (!defined $pat || $test =~ /$pat/) { - print "\nRunning test suite $test.\n\n"; - my ($code, $result) = captureCommand("PERLLIB=../lib perl -w $file"); - my @data = split(/\n/, $result); - if ($code != 0) { - ++$failures; - print "Test binary returned with non-null exitcode. Failure.\n"; - } elsif (&matchesFailurePattern(\@data)) { - ++$failures; - } elsif (¬MatchesSuccessPattern(\@data)) { - ++$failures; - } - } else { - # print "Skipping test suite '$test' not matching '$pat'.\n"; - } -} - -if ($failures > 0) { - print "\n\n$failures test suites failed.\n"; - exit(1); -} else { - print "\n\nAll tests succeeded.\n"; -} - -`rm -rv ${tempdir}`; - -exit(0); - -sub matchesFailurePattern { # (LineArrayRef) - my ($data) = @_; - foreach my $line (@$data) { - if ($line =~ $failure_pattern) { - print "Line '$line' indicates failure. Failing test suite.\n"; - return 1; - } - } - return 0; -} - -sub notMatchesSuccessPattern { # (LineArrayRef) - my ($data) = @_; - foreach my $line (@$data) { - if ($line !~ $accepted_pattern) { - print "Suspicious line '$line'.\n"; - print "Failing test due to line suspected to indicate failure.\n" - . "(Use printTest to print debug data during test to have it " - . "not been marked suspected.\n"; - return 1; - } - } - return 0; -} - -# Run a given command, giving exitcode and output back, but let command write -# directly to stdout/stderr. (Useful for long running commands or commands that -# may stall, such that you can see where it got into trouble) -sub captureCommand { # (Cmd) -> (ExitCode, Output) - my ($cmd) = @_; - my ($fh, $line); - my $data; - open ($fh, "$cmd 2>&1 |") or die "Failed to run '$cmd'."; - while ($line = <$fh>) { - print $line; - $data .= $line; - } - close $fh; - my $exitcode = $?; - return ($exitcode >> 8, $data); -} diff --git a/vespalib/src/tests/signalhandler/my_shared_library.cpp b/vespalib/src/tests/signalhandler/my_shared_library.cpp index 4b7593d863c..97c03208213 100644 --- a/vespalib/src/tests/signalhandler/my_shared_library.cpp +++ b/vespalib/src/tests/signalhandler/my_shared_library.cpp @@ -8,10 +8,12 @@ // Could have used a single std::barrier<no op functor> here, but when using explicit // phase latches it sort of feels like the semantics are more immediately obvious. -void my_cool_function(std::latch& arrival_latch, std::latch& departure_latch) { - arrival_latch.arrive_and_wait(); +void my_cool_function(vespalib::CountDownLatch& arrival_latch, vespalib::CountDownLatch& departure_latch) { + arrival_latch.countDown(); + arrival_latch.await(); // Twiddle thumbs in departure latch until main test thread has dumped our stack - departure_latch.arrive_and_wait(); + departure_latch.countDown(); + departure_latch.await(); asm(""); // Dear GCC; really, really don't inline this function. It's clobberin' time! } diff --git a/vespalib/src/tests/signalhandler/my_shared_library.h b/vespalib/src/tests/signalhandler/my_shared_library.h index e48a6d91a4f..4a1b259981b 100644 --- a/vespalib/src/tests/signalhandler/my_shared_library.h +++ b/vespalib/src/tests/signalhandler/my_shared_library.h @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/stllike/string.h> -#include <latch> +#include <vespa/vespalib/util/count_down_latch.h> -void my_cool_function(std::latch&, std::latch&) __attribute__((noinline)); +void my_cool_function(vespalib::CountDownLatch&, vespalib::CountDownLatch&) __attribute__((noinline)); vespalib::string my_totally_tubular_and_groovy_function() __attribute__((noinline)); diff --git a/vespalib/src/tests/signalhandler/signalhandler_test.cpp b/vespalib/src/tests/signalhandler/signalhandler_test.cpp index 8871a985fed..4aeffd27b89 100644 --- a/vespalib/src/tests/signalhandler/signalhandler_test.cpp +++ b/vespalib/src/tests/signalhandler/signalhandler_test.cpp @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "my_shared_library.h" +#include <vespa/vespalib/util/count_down_latch.h> #include <vespa/vespalib/util/signalhandler.h> #include <vespa/vespalib/gtest/gtest.h> #include <gmock/gmock.h> -#include <latch> #include <thread> #include <unistd.h> @@ -38,18 +38,20 @@ TEST(SignalHandlerTest, signal_handler_can_intercept_hooked_signals) TEST(SignalHandlerTest, can_dump_stack_of_another_thread) { - std::latch arrival_latch(2); - std::latch departure_latch(2); + vespalib::CountDownLatch arrival_latch(2); + vespalib::CountDownLatch departure_latch(2); std::thread t([&]{ my_cool_function(arrival_latch, departure_latch); }); - arrival_latch.arrive_and_wait(); + arrival_latch.countDown(); + arrival_latch.await(); auto trace = SignalHandler::get_cross_thread_stack_trace(t.native_handle()); EXPECT_THAT(trace, HasSubstr("my_cool_function")); - departure_latch.arrive_and_wait(); + departure_latch.countDown(); + departure_latch.await(); t.join(); } diff --git a/vespalib/src/tests/simple_thread_bundle/simple_thread_bundle_test.cpp b/vespalib/src/tests/simple_thread_bundle/simple_thread_bundle_test.cpp index d6e0e473e59..27e75315dfc 100644 --- a/vespalib/src/tests/simple_thread_bundle/simple_thread_bundle_test.cpp +++ b/vespalib/src/tests/simple_thread_bundle/simple_thread_bundle_test.cpp @@ -249,6 +249,9 @@ TEST_FF("require that various versions of run can be used to invoke targets", Si f2.check({4,4,4,4,4}); f1.run(f2.cnts); f2.check({5,5,5,5,5}); + std::initializer_list<std::reference_wrapper<Cnt>> list = {f2.cnts[0], f2.cnts[1], f2.cnts[2], f2.cnts[3], f2.cnts[4]}; + f1.run(list); + f2.check({6,6,6,6,6}); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.h b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.h index 744e00674d6..e6627eb80e6 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.h @@ -5,6 +5,7 @@ #include "i_unique_store_dictionary.h" #include "i_unique_store_dictionary_read_snapshot.h" #include <vespa/vespalib/stllike/allocator.h> +#include <cassert> namespace vespalib::datastore { diff --git a/vespalib/src/vespa/vespalib/stllike/string.h b/vespalib/src/vespa/vespalib/stllike/string.h index 7bf03895a88..73b91de883d 100644 --- a/vespalib/src/vespa/vespalib/stllike/string.h +++ b/vespalib/src/vespa/vespalib/stllike/string.h @@ -654,7 +654,7 @@ template<uint32_t StackSize> small_string<StackSize> operator + (const char * a, const small_string<StackSize> & b); -#if __cplusplus < 201709L || (!defined(__clang__) && defined(__GNUC__) && __GNUC__ < 10) +#if __cplusplus < 201709L template<typename T, uint32_t StackSize> bool operator == (const T& a, const small_string<StackSize>& b) noexcept diff --git a/vespalib/src/vespa/vespalib/testkit/test_master.hpp b/vespalib/src/vespa/vespalib/testkit/test_master.hpp index 7d9b2e1fddc..f2724731bee 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_master.hpp +++ b/vespalib/src/vespa/vespalib/testkit/test_master.hpp @@ -4,7 +4,7 @@ namespace vespalib { -#if (!defined(__clang__) && defined(__GNUC__) && __GNUC__ < 9) || (defined(__clang__) && defined(__apple_build_version__)) +#if defined(__clang__) && defined(__apple_build_version__) // cf. https://cplusplus.github.io/LWG/issue2221 template<class charT, class traits> std::basic_ostream<charT, traits>& operator<<(std::basic_ostream<charT, traits>& os, std::nullptr_t) diff --git a/vespalib/src/vespa/vespalib/util/arrayref.h b/vespalib/src/vespa/vespalib/util/arrayref.h index 88dc501370c..be4f4ef83f2 100644 --- a/vespalib/src/vespa/vespalib/util/arrayref.h +++ b/vespalib/src/vespa/vespalib/util/arrayref.h @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "small_vector.h" #include <vector> namespace vespalib { @@ -17,8 +16,6 @@ public: constexpr ArrayRef(T * v, size_t sz) noexcept : _v(v), _sz(sz) { } template<typename A=std::allocator<T>> ArrayRef(std::vector<T, A> & v) noexcept : _v(v.data()), _sz(v.size()) { } - template<size_t N> - ArrayRef(SmallVector<T, N> &v) noexcept : _v(v.data()), _sz(v.size()) { } T & operator [] (size_t i) noexcept { return _v[i]; } const T & operator [] (size_t i) const noexcept { return _v[i]; } T * data() noexcept { return _v; } @@ -38,8 +35,6 @@ public: constexpr ConstArrayRef(const T *v, size_t sz) noexcept : _v(v), _sz(sz) { } template<typename A=std::allocator<T>> ConstArrayRef(const std::vector<T, A> & v) noexcept : _v(v.data()), _sz(v.size()) { } - template<size_t N> - ConstArrayRef(const SmallVector<T, N> &v) noexcept : _v(v.data()), _sz(v.size()) { } ConstArrayRef(const ArrayRef<T> & v) noexcept : _v(v.data()), _sz(v.size()) { } constexpr ConstArrayRef() noexcept : _v(nullptr), _sz(0) {} const T & operator [] (size_t i) const noexcept { return _v[i]; } diff --git a/vespalib/src/vespa/vespalib/util/rendezvous.h b/vespalib/src/vespa/vespalib/util/rendezvous.h index 6121b3f3dd6..2880f325d96 100644 --- a/vespalib/src/vespa/vespalib/util/rendezvous.h +++ b/vespalib/src/vespa/vespalib/util/rendezvous.h @@ -93,8 +93,7 @@ public: * @return output parameter for a single thread * @param input input parameter for a single thread **/ - template <bool ext_id = external_id> - typename std::enable_if<!ext_id,OUT>::type rendezvous(IN input); + OUT rendezvous(IN input) requires (!external_id); /** * Called by individual threads to synchronize execution and share @@ -107,8 +106,7 @@ public: * @param my_id participant id for this thread (must be in range and * not conflicting with other threads) **/ - template <bool ext_id = external_id> - typename std::enable_if<ext_id,OUT>::type rendezvous(IN input, size_t my_id); + OUT rendezvous(IN input, size_t my_id) requires (external_id); }; } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/util/rendezvous.hpp b/vespalib/src/vespa/vespalib/util/rendezvous.hpp index a22e05fac3f..4b3c6138428 100644 --- a/vespalib/src/vespa/vespalib/util/rendezvous.hpp +++ b/vespalib/src/vespa/vespalib/util/rendezvous.hpp @@ -59,12 +59,12 @@ template <typename IN, typename OUT, bool external_id> Rendezvous<IN, OUT, external_id>::~Rendezvous() = default; template <typename IN, typename OUT, bool external_id> -template <bool ext_id> -typename std::enable_if<!ext_id,OUT>::type +OUT Rendezvous<IN, OUT, external_id>::rendezvous(IN input) + requires (!external_id) { OUT ret{}; - static_assert(ext_id == external_id); + static_assert(!external_id); if (_size == 1) { meet_self(input, ret); } else { @@ -75,13 +75,13 @@ Rendezvous<IN, OUT, external_id>::rendezvous(IN input) } template <typename IN, typename OUT, bool external_id> -template <bool ext_id> -typename std::enable_if<ext_id,OUT>::type +OUT Rendezvous<IN, OUT, external_id>::rendezvous(IN input, size_t my_id) + requires (external_id) { OUT ret{}; assert(my_id < _size); - static_assert(ext_id == external_id); + static_assert(external_id); if (_size == 1) { meet_self(input, ret); } else { diff --git a/vespalib/src/vespa/vespalib/util/signalhandler.cpp b/vespalib/src/vespa/vespalib/util/signalhandler.cpp index 68368269c59..d9e59ed6688 100644 --- a/vespalib/src/vespa/vespalib/util/signalhandler.cpp +++ b/vespalib/src/vespa/vespalib/util/signalhandler.cpp @@ -11,6 +11,7 @@ #include <atomic> #include <cassert> #include <chrono> +#include <mutex> #include <thread> #include <typeinfo> diff --git a/vespalib/src/vespa/vespalib/util/small_vector.h b/vespalib/src/vespa/vespalib/util/small_vector.h index cf9910e24b5..7df3913f376 100644 --- a/vespalib/src/vespa/vespalib/util/small_vector.h +++ b/vespalib/src/vespa/vespalib/util/small_vector.h @@ -4,6 +4,7 @@ #include "alloc.h" #include "traits.h" +#include "arrayref.h" #include <cstring> #include <cassert> #include <iterator> @@ -104,15 +105,15 @@ private: free(old_data); } } - template <typename InputIt> - void init(InputIt first, InputIt last, std::random_access_iterator_tag) { + template <std::random_access_iterator InputIt> + void init(InputIt first, InputIt last) { reserve(last - first); while (first != last) { small_vector::create_at((_data + _size++), *first++); } } - template <typename InputIt> - void init(InputIt first, InputIt last, std::input_iterator_tag) { + template <std::input_iterator InputIt> + void init(InputIt first, InputIt last) { while (first != last) { emplace_back(*first++); } @@ -137,10 +138,10 @@ public: small_vector::create_at((_data + _size++), value); } } - template <typename InputIt, std::enable_if_t<std::is_base_of_v<std::input_iterator_tag, typename std::iterator_traits<InputIt>::iterator_category>, bool> = true> + template <std::input_iterator InputIt> SmallVector(InputIt first, InputIt last) : SmallVector() { - init(first, last, typename std::iterator_traits<InputIt>::iterator_category()); + init(first, last); } SmallVector(SmallVector &&rhs) : SmallVector() { reserve(rhs._size); @@ -174,6 +175,7 @@ public: free(_data); } } + operator ConstArrayRef<T> () const { return ConstArrayRef<T>(data(), size()); } bool empty() const { return (_size == 0); } uint32_t size() const { return _size; } uint32_t capacity() const { return _capacity; } diff --git a/vespalib/src/vespa/vespalib/util/stash.h b/vespalib/src/vespa/vespalib/util/stash.h index 11731e69217..0d3558a26ab 100644 --- a/vespalib/src/vespa/vespalib/util/stash.h +++ b/vespalib/src/vespa/vespalib/util/stash.h @@ -6,6 +6,7 @@ #include "arrayref.h" #include "memoryusage.h" #include <cstdlib> +#include <memory> namespace vespalib { namespace stash { diff --git a/vespalib/src/vespa/vespalib/util/thread_bundle.h b/vespalib/src/vespa/vespalib/util/thread_bundle.h index 830d7c76e7c..cfdedb347c8 100644 --- a/vespalib/src/vespa/vespalib/util/thread_bundle.h +++ b/vespalib/src/vespa/vespalib/util/thread_bundle.h @@ -4,6 +4,7 @@ #include "runnable.h" #include <vector> +#include <ranges> namespace vespalib { @@ -46,10 +47,11 @@ struct ThreadBundle { } // convenience run wrapper - template <typename Item> - std::enable_if_t<!is_runnable_ptr<Item>(),void> run(std::vector<Item> &items) { + template <std::ranges::range List> + requires (!is_runnable_ptr<std::ranges::range_value_t<List>>()) + void run(List &items) { std::vector<Runnable*> targets; - targets.reserve(items.size()); + targets.reserve(std::ranges::size(items)); for (auto &item: items) { targets.push_back(resolve(item)); } diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java index 6508c154978..e5384f9abe2 100644 --- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java @@ -36,7 +36,7 @@ public class Configurator { this.zookeeperServerConfig = zookeeperServerConfig; this.configFilePath = makeAbsolutePath(zookeeperServerConfig.zooKeeperConfigFile()); System.setProperty(ZOOKEEPER_JMX_LOG4J_DISABLE, "true"); - System.setProperty("zookeeper.snapshot.trust.empty", Boolean.valueOf(zookeeperServerConfig.trustEmptySnapshot()).toString()); + System.setProperty("zookeeper.snapshot.trust.empty", String.valueOf(zookeeperServerConfig.trustEmptySnapshot())); // Max serialization length. Has effect for both client and server. // Doc says that it is max size of data in a zookeeper node, but it goes for everything that // needs to be serialized, see https://issues.apache.org/jira/browse/ZOOKEEPER-1162 for details @@ -46,6 +46,8 @@ public class Configurator { // Need to set this as a system property, otherwise it will be parsed for _every_ packet and an exception will be thrown (and handled) System.setProperty("zookeeper.globalOutstandingLimit", "1000"); System.setProperty("zookeeper.snapshot.compression.method", zookeeperServerConfig.snapshotMethod()); + System.setProperty("zookeeper.leader.closeSocketAsync", String.valueOf(zookeeperServerConfig.leaderCloseSocketAsync())); + System.setProperty("zookeeper.learner.asyncSending", String.valueOf(zookeeperServerConfig.learnerAsyncSending())); } void writeConfigToDisk() { |