diff options
| author | Øyvind Grønnesby <oyving@yahooinc.com> | 2022-05-30 12:58:57 +0200 |
|---|---|---|
| committer | Øyvind Grønnesby <oyving@yahooinc.com> | 2022-05-30 12:58:57 +0200 |
| commit | 1cce4cc2640fe7bce984b808b1571555f5df0c30 (patch) | |
| tree | 65c8a1830e8e94c368fcd418dd0f7cabd94439a6 | |
| parent | 9fa7903909043d7b855f7e3ba315050ba5a12597 (diff) | |
| parent | 970cd02608c75df65bcfee3b061f906a39566825 (diff) | |
Merge remote-tracking branch 'origin/master' into ogronnesby/contact-info-resources
Conflicts:
controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
401 files changed, 3171 insertions, 1793 deletions
diff --git a/ann_benchmark/src/vespa/ann_benchmark/vespa_ann_benchmark.cpp b/ann_benchmark/src/vespa/ann_benchmark/vespa_ann_benchmark.cpp index c62f140d944..4af30a5d9d3 100644 --- a/ann_benchmark/src/vespa/ann_benchmark/vespa_ann_benchmark.cpp +++ b/ann_benchmark/src/vespa/ann_benchmark/vespa_ann_benchmark.cpp @@ -7,6 +7,7 @@ #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/nearest_neighbor_index.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/eval/eval/value.h> #include <vespa/vespalib/test/insertion_operators.h> #include <ostream> diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java index 9f4b1a9d01c..9bd6153f159 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java @@ -7,8 +7,6 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.jdisc.http.server.jetty.RequestUtils; - -import java.util.logging.Level; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.Path; import com.yahoo.restapi.SlimeJsonResponse; @@ -16,12 +14,12 @@ import com.yahoo.security.KeyUtils; import com.yahoo.security.SubjectAlternativeName; import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; -import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceConfirmation; import com.yahoo.vespa.hosted.athenz.instanceproviderservice.InstanceValidator; +import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig; import com.yahoo.vespa.hosted.ca.Certificates; import com.yahoo.vespa.hosted.ca.instance.InstanceIdentity; import com.yahoo.vespa.hosted.ca.instance.InstanceRefresh; @@ -38,6 +36,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.function.Function; +import java.util.logging.Level; import java.util.stream.Stream; /** @@ -68,7 +67,7 @@ public class CertificateAuthorityApiHandler extends ThreadedHttpRequestHandler { this.secretStore = secretStore; this.certificates = certificates; this.caPrivateKeySecretName = athenzProviderServiceConfig.secretName(); - this.caCertificateSecretName = athenzProviderServiceConfig.domain() + ".ca.cert"; + this.caCertificateSecretName = athenzProviderServiceConfig.caCertSecretName(); this.instanceValidator = instanceValidator; } diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/TestUtils.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/TestUtils.java index 7573b5690e7..4110ad2bfa2 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/TestUtils.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/TestUtils.java @@ -18,7 +18,8 @@ public class TestUtils { .domain(domain) .certDnsSuffix(dnsSuffix) .ztsUrl("localhost/zts") - .secretName("s3cr3t"); + .secretName("s3cr3t") + .caCertSecretName(domain + ".ca.cert"); return new AthenzProviderServiceConfig( zoneConfig.athenzCaTrustStore("/dummy/path/to/athenz-ca.jks")); } diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java index 3341575afd4..ad0715cbbea 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java @@ -64,6 +64,7 @@ public class ContainerTester { " <serviceName>servicename</serviceName>\n" + " <secretName>secretname</secretName>\n" + " <secretVersion>0</secretVersion>\n" + + " <caCertSecretName>vespa.external.ca.cert</caCertSecretName>\n" + " <certDnsSuffix>suffix</certDnsSuffix>\n" + " <ztsUrl>https://localhost:123/</ztsUrl>\n" + " </config>\n" + diff --git a/build_settings.cmake b/build_settings.cmake index 1dfd55e7d0e..d0cf6f38326 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -146,6 +146,12 @@ if(VESPA_USE_SANITIZER) set(VESPA_SANITIZER_ENV "TSAN_OPTIONS=suppressions=${VESPA_SANITIZER_SUPPRESSIONS_FILE} history_size=7 detect_deadlocks=1 second_deadlock_stack=1") endif() endif() +# Dump stack when finding issues in unit tests using undefined sanitizer +if(VESPA_USE_SANITIZER) + if(VESPA_USE_SANITIZER STREQUAL "undefined") + set(VESPA_SANITIZER_ENV "UBSAN_OPTIONS=print_stacktrace=1") + endif() +endif() if(VESPA_LLVM_VERSION) else() diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java index 307509f0452..c5522c4c96a 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassVisitor.java @@ -160,7 +160,9 @@ class AnalyzeClassVisitor extends ClassVisitor implements ImportCollector { if (ExportPackage.class.getName().equals(Type.getType(desc).getClassName())) { return visitExportPackage(); } else { - addImportWithTypeDesc(desc); + if (visible) { + addImportWithTypeDesc(desc); + } return Analyze.visitAnnotationDefault(this); } } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java index 6b9b0845328..1c061ca49a2 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java @@ -3,8 +3,10 @@ package com.yahoo.container.plugin.classanalysis; import com.yahoo.container.plugin.classanalysis.sampleclasses.Base; import com.yahoo.container.plugin.classanalysis.sampleclasses.ClassAnnotation; +import com.yahoo.container.plugin.classanalysis.sampleclasses.InvisibleAnnotation; import com.yahoo.container.plugin.classanalysis.sampleclasses.Derived; import com.yahoo.container.plugin.classanalysis.sampleclasses.DummyAnnotation; +import com.yahoo.container.plugin.classanalysis.sampleclasses.InvisibleDummyAnnotation; import com.yahoo.container.plugin.classanalysis.sampleclasses.Fields; import com.yahoo.container.plugin.classanalysis.sampleclasses.Interface1; import com.yahoo.container.plugin.classanalysis.sampleclasses.Interface2; @@ -109,6 +111,11 @@ public class AnalyzeClassTest { } @Test + public void invisible_annotation_not_included() { + assertFalse(analyzeClass(InvisibleAnnotation.class).getReferencedClasses().contains(name(InvisibleDummyAnnotation.class))); + } + + @Test public void method_annotation_is_included() { assertTrue(analyzeClass(MethodAnnotation.class).getReferencedClasses().contains(name(DummyAnnotation.class))); } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/DummyAnnotation.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/DummyAnnotation.java index a07b2917bd4..42335da3d62 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/DummyAnnotation.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/DummyAnnotation.java @@ -1,9 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.plugin.classanalysis.sampleclasses; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + /** * Input for class analysis tests. * @author Tony Vaagenes */ +@Retention(RetentionPolicy.RUNTIME) public @interface DummyAnnotation { } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/InvisibleAnnotation.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/InvisibleAnnotation.java new file mode 100644 index 00000000000..ced7c3305b0 --- /dev/null +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/InvisibleAnnotation.java @@ -0,0 +1,10 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.classanalysis.sampleclasses; + +/** + * Input for class analysis tests.* + * @author arnej + */ +@InvisibleDummyAnnotation +public class InvisibleAnnotation { +} diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/InvisibleDummyAnnotation.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/InvisibleDummyAnnotation.java new file mode 100644 index 00000000000..b3cb75df354 --- /dev/null +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/sampleclasses/InvisibleDummyAnnotation.java @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.plugin.classanalysis.sampleclasses; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * Input for class analysis tests. + * @author arnej + */ +@Retention(RetentionPolicy.CLASS) +public @interface InvisibleDummyAnnotation { +} diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index c44915c7ade..da7dce45e4d 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -20,7 +20,7 @@ <junit5.platform.version>1.8.1</junit5.platform.version> <!-- MUST BE KEPT IN SYNC WITH parent/pom.xml --> - <athenz.version>1.10.14</athenz.version> + <athenz.version>1.10.54</athenz.version> <httpclient5.version>5.1.2</httpclient5.version> <junit5.version>5.8.1</junit5.version> <!-- TODO: in parent this is named 'junit.version' --> <onnxruntime.version>1.8.0</onnxruntime.version> diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java index 0dd04a409ae..251ef48c9f7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java @@ -21,6 +21,7 @@ public class LogserverContainer extends Container { public LogserverContainer(AbstractConfigProducer<?> parent, DeployState deployState) { super(parent, "" + 0, 0, deployState); + if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester()) useDynamicPorts(); LogserverContainerCluster cluster = (LogserverContainerCluster) parent; addComponent(new AccessLogComponent( cluster, AccessLogType.jsonAccessLog, CompressionType.GZIP, Optional.of(cluster.getName()), true)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index c4d115c61d6..95307456f80 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -241,6 +241,7 @@ public class VespaMetricSet { metrics.add(new Metric("jdisc.http.ssl.handshake.failure.incompatible_protocols.rate")); metrics.add(new Metric("jdisc.http.ssl.handshake.failure.incompatible_ciphers.rate")); metrics.add(new Metric("jdisc.http.ssl.handshake.failure.unknown.rate")); + metrics.add(new Metric("jdisc.http.ssl.handshake.failure.connection_closed.rate")); metrics.add(new Metric("jdisc.http.handler.unhandled_exceptions.rate")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index a91a7949ad8..dae2f7e8cb8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -72,29 +72,30 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { } private void assignLogserver(DeployState deployState, NodesSpecification nodesSpecification, Admin admin) { - if (nodesSpecification.minResources().nodes() > 1) throw new IllegalArgumentException("You can only request a single log server"); - if (deployState.getProperties().applicationId().instance().isTester()) return; // No logserver is needed on tester applications - if (nodesSpecification.isDedicated()) { - Collection<HostResource> hosts = allocateHosts(admin.hostSystem(), "logserver", nodesSpecification); - if (hosts.isEmpty()) return; // No log server can be created (and none is needed) + if (nodesSpecification.minResources().nodes() > 1) + throw new IllegalArgumentException("You can only request a single log server"); + + Collection<HostResource> hosts = List.of(); + if (nodesSpecification.isDedicated()) + hosts = allocateHosts(admin.hostSystem(), "logserver", nodesSpecification); + else if (containerModels.iterator().hasNext()) + hosts = sortedContainerHostsFrom(containerModels.iterator().next(), nodesSpecification.minResources().nodes(), false); + else + context.getDeployLogger().logApplicationPackage(Level.INFO, "No container host available to use for running logserver"); - Logserver logserver = createLogserver(deployState, admin, hosts); - createContainerOnLogserverHost(deployState, admin, logserver.getHostResource()); - } else if (containerModels.iterator().hasNext()) { - List<HostResource> hosts = sortedContainerHostsFrom(containerModels.iterator().next(), nodesSpecification.minResources().nodes(), false); - if (hosts.isEmpty()) return; // No log server can be created (and none is needed) + if (hosts.isEmpty()) return; // No log server can be created (and none is needed) - createLogserver(deployState, admin, hosts); - } else { - context.getDeployLogger().logApplicationPackage(Level.INFO, "No container host available to use for running logserver"); - } + Logserver logserver = createLogserver(deployState, admin, hosts); + if (nodesSpecification.isDedicated() || deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester()) + createContainerOnLogserverHost(deployState, admin, logserver.getHostResource()); } private NodesSpecification createNodesSpecificationForLogserver() { DeployState deployState = context.getDeployState(); - if (deployState.getProperties().useDedicatedNodeForLogserver() && - context.getApplicationType() == ConfigModelContext.ApplicationType.DEFAULT && - deployState.isHosted()) + if ( deployState.getProperties().useDedicatedNodeForLogserver() + && context.getApplicationType() == ConfigModelContext.ApplicationType.DEFAULT + && deployState.isHosted() + && ! deployState.getProperties().applicationId().instance().isTester()) return NodesSpecification.dedicated(1, context); else return NodesSpecification.nonDedicated(1, context); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 0bd93c6d0df..951317b68f3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -86,6 +86,7 @@ import com.yahoo.vespa.model.container.search.GUIHandler; import com.yahoo.vespa.model.container.search.PageTemplates; import com.yahoo.vespa.model.container.search.searchchain.SearchChains; import com.yahoo.vespa.model.container.xml.document.DocumentFactoryBuilder; +import com.yahoo.vespa.model.container.xml.embedder.EmbedderConfig; import com.yahoo.vespa.model.content.StorageGroup; import org.w3c.dom.Element; import org.w3c.dom.Node; @@ -197,9 +198,11 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addClusterContent(ApplicationContainerCluster cluster, Element spec, ConfigModelContext context) { DeployState deployState = context.getDeployState(); DocumentFactoryBuilder.buildDocumentFactories(cluster, spec); + addConfiguredComponents(deployState, cluster, spec); addSecretStore(cluster, spec, deployState); + addEmbedderComponents(deployState, cluster, spec); addModelEvaluation(spec, cluster, context); addModelEvaluationBundles(cluster); @@ -382,6 +385,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } } + private static void addEmbedderComponents(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { + for (Element node : XML.getChildren(spec, "embedder")) { + Element transformed = EmbedderConfig.transform(deployState, node); + cluster.addComponent(new DomComponentBuilder().build(deployState, cluster, transformed)); + } + } + private void addConfiguredComponents(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { for (Element components : XML.getChildren(spec, "components")) { addIncludes(components); @@ -422,13 +432,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { protected void addAccessLogs(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { List<Element> accessLogElements = getAccessLogElements(spec); - if (cluster.isHostedVespa() && !accessLogElements.isEmpty()) { - log.logApplicationPackage( - Level.WARNING, "Applications are not allowed to override the 'accesslog' element"); - } else { - for (Element accessLog : accessLogElements) { - AccessLogBuilder.buildIfNotDisabled(deployState, cluster, accessLog).ifPresent(cluster::addComponent); - } + for (Element accessLog : accessLogElements) { + AccessLogBuilder.buildIfNotDisabled(deployState, cluster, accessLog).ifPresent(cluster::addComponent); } if (accessLogElements.isEmpty() && deployState.getAccessLoggingEnabledByDefault()) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/embedder/EmbedderConfig.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/embedder/EmbedderConfig.java new file mode 100644 index 00000000000..a2286647cdd --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/embedder/EmbedderConfig.java @@ -0,0 +1,79 @@ +package com.yahoo.vespa.model.container.xml.embedder; + +import com.yahoo.config.model.deploy.DeployState; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +/** + * Translates config in services.xml of the form + * + * <embedder id="..." class="..." bundle="..." def="..."> + * <!-- options --> + * </embedder> + * + * to component configuration of the form + * + * <component id="..." class="..." bundle="..."> + * <config name=def> + * <!-- options --> + * </config> + * </component> + * + * with some added interpretations based on recognizing the class. + * + * @author lesters + */ +public class EmbedderConfig { + + static EmbedderConfigTransformer getEmbedderTransformer(Element spec, boolean hosted) { + String classId = getEmbedderClass(spec); + switch (classId) { + case "ai.vespa.embedding.BertBaseEmbedder": return new EmbedderConfigBertBaseTransformer(spec, hosted); + } + return new EmbedderConfigTransformer(spec, hosted); + } + + static String modelIdToUrl(String id) { + switch (id) { + case "test-model-id": + return "test-model-url"; + case "minilm-l6-v2": + return "https://data.vespa.oath.cloud/onnx_models/sentence_all_MiniLM_L6_v2.onnx"; + case "bert-base-uncased": + return "https://data.vespa.oath.cloud/onnx_models/bert-base-uncased-vocab.txt"; + } + throw new IllegalArgumentException("Unknown model id: '" + id + "'"); + } + + /** + * Transforms the <embedder ...> element to component configuration. + * + * @param deployState the deploy state - as config generation can depend on context + * @param embedderSpec the XML element containing the <embedder ...> + * @return a new XML element containting the <component ...> configuration + */ + public static Element transform(DeployState deployState, Element embedderSpec) { + EmbedderConfigTransformer transformer = getEmbedderTransformer(embedderSpec, deployState.isHosted()); + NodeList children = embedderSpec.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + Node child = children.item(i); + if (child instanceof Element) { + transformer.addOption((Element) child); + } + } + return transformer.createComponentConfig(deployState); + } + + private static String getEmbedderClass(Element spec) { + if (spec.hasAttribute("class")) { + return spec.getAttribute("class"); + } + if (spec.hasAttribute("id")) { + return spec.getAttribute("id"); + } + throw new IllegalArgumentException("Embedder specification does not have a required class attribute"); + } + + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/embedder/EmbedderConfigBertBaseTransformer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/embedder/EmbedderConfigBertBaseTransformer.java new file mode 100644 index 00000000000..9431926d088 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/embedder/EmbedderConfigBertBaseTransformer.java @@ -0,0 +1,40 @@ +package com.yahoo.vespa.model.container.xml.embedder; + +import org.w3c.dom.Element; + +import java.util.Map; + +/** + * Transforms embedding configuration to component configuration for the + * BertBaseEmbedder using embedder.bert-base-embedder.def + * + * @author lesters + */ +public class EmbedderConfigBertBaseTransformer extends EmbedderConfigTransformer { + + private static final String BUNDLE = "model-integration"; + private static final String DEF = "embedding.bert-base-embedder"; + + public EmbedderConfigBertBaseTransformer(Element spec, boolean hosted) { + super(spec, hosted, BUNDLE, DEF); + + EmbedderOption.Builder modelOption = new EmbedderOption.Builder() + .name("model") + .required(true) + .optionTransformer(new EmbedderOption.ModelOptionTransformer("transformerModelPath", "transformerModelUrl")); + EmbedderOption.Builder vocabOption = new EmbedderOption.Builder() + .name("vocab") + .required(true) + .optionTransformer(new EmbedderOption.ModelOptionTransformer("tokenizerVocabPath", "tokenizerVocabUrl")); + + // Defaults + if (hosted) { + modelOption.attributes(Map.of("id", "minilm-l6-v2")).value(""); + vocabOption.attributes(Map.of("id", "bert-base-uncased")).value(""); + } + + addOption(modelOption.build()); + addOption(vocabOption.build()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/embedder/EmbedderConfigTransformer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/embedder/EmbedderConfigTransformer.java new file mode 100644 index 00000000000..30327fdc8af --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/embedder/EmbedderConfigTransformer.java @@ -0,0 +1,102 @@ +package com.yahoo.vespa.model.container.xml.embedder; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.text.XML; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +/** + * A specific embedder to component configuration transformer. + * + * @author lesters + */ +public class EmbedderConfigTransformer { + + private final Document doc = XML.getDocumentBuilder().newDocument(); + + private final String id; + private final String className; + private final String bundle; + private final String def; + private final Map<String, EmbedderOption> options = new HashMap<>(); + + public EmbedderConfigTransformer(Element spec, boolean hosted) { + this(spec, hosted, null, null); + } + + public EmbedderConfigTransformer(Element spec, boolean hosted, String defaultBundle, String defaultDef) { + id = spec.getAttribute("id"); + className = spec.hasAttribute("class") ? spec.getAttribute("class") : id; + bundle = spec.hasAttribute("bundle") ? spec.getAttribute("bundle") : defaultBundle; + def = spec.hasAttribute("def") ? spec.getAttribute("def") : defaultDef; + + if (className == null || className.length() == 0) { + throw new IllegalArgumentException("Embedder class is empty"); + } + if (this.bundle == null || this.bundle.length() == 0) { + throw new IllegalArgumentException("Embedder configuration requires a bundle name"); + } + if (this.def == null || this.def.length() == 0) { + throw new IllegalArgumentException("Embedder configuration requires a config definition name"); + } + } + + Element createComponentConfig(DeployState deployState) { + checkRequiredOptions(); + + Element component = doc.createElement("component"); + component.setAttribute("id", id); + component.setAttribute("class", className); + component.setAttribute("bundle", bundle); + + if (options.size() > 0) { + Element config = doc.createElement("config"); + config.setAttribute("name", def); + for (Map.Entry<String, EmbedderOption> entry : options.entrySet()) { + entry.getValue().toElement(deployState, config); + } + component.appendChild(config); + } + + return component; + } + + // TODO: support nested options + void addOption(Element elem) { + String name = elem.getTagName(); + + EmbedderOption.Builder builder = new EmbedderOption.Builder(); + builder.name(name); + builder.value(elem.getTextContent()); + builder.attributes(elem); + + if (options.containsKey(name)) { + builder.required(options.get(name).required()); + builder.optionTransformer(options.get(name).optionTransformer()); + } + options.put(name, builder.build()); + } + + void addOption(EmbedderOption option) { + options.put(option.name(), option); + } + + private void checkRequiredOptions() { + List<String> missingOptions = new ArrayList<>(); + for (EmbedderOption option : options.values()) { + if ( ! option.isSet()) { + missingOptions.add(option.name()); + } + } + if (missingOptions.size() > 0) { + throw new IllegalArgumentException("Embedder '" + className + "' requires options for " + missingOptions); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/embedder/EmbedderOption.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/embedder/EmbedderOption.java new file mode 100644 index 00000000000..206745887d1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/embedder/EmbedderOption.java @@ -0,0 +1,184 @@ +package com.yahoo.vespa.model.container.xml.embedder; + +import com.yahoo.config.model.deploy.DeployState; +import org.w3c.dom.Element; +import org.w3c.dom.NamedNodeMap; + +import java.util.HashMap; +import java.util.Map; + + +/** + * Holds options for embedder configuration. This includes code for handling special + * options such as model specifiers. + * + * @author lesters + */ +public class EmbedderOption { + + public static final OptionTransformer defaultOptionTransformer = new OptionTransformer(); + + private final String name; + private final boolean required; + private final String value; + private final Map<String, String> attributes; + private final OptionTransformer optionTransformer; + private final boolean set; + + private EmbedderOption(Builder builder) { + this.name = builder.name; + this.required = builder.required; + this.value = builder.value; + this.attributes = builder.attributes; + this.optionTransformer = builder.optionTransformer; + this.set = builder.set; + } + + public void toElement(DeployState deployState, Element parent) { + optionTransformer.transform(deployState, parent, this); + } + + public String name() { + return name; + } + + public String value() { + return value; + } + + public boolean required() { + return required; + } + + public OptionTransformer optionTransformer() { + return optionTransformer; + } + + public boolean isSet() { + return set; + } + + /** + * Basic option transformer. No special handling of options. + */ + public static class OptionTransformer { + public void transform(DeployState deployState, Element parent, EmbedderOption option) { + createElement(parent, option.name(), option.value()); + } + + public static Element createElement(Element parent, String name, String value) { + Element element = parent.getOwnerDocument().createElement(name); + element.setTextContent(value); + parent.appendChild(element); + return element; + } + } + + /** + * Transforms model options of type <x id="..." url="..." path="..." /> to the + * required fields in the config definition. + */ + public static class ModelOptionTransformer extends OptionTransformer { + + private final String pathField; + private final String urlField; + + public ModelOptionTransformer(String pathField, String urlField) { + super(); + this.pathField = pathField; + this.urlField = urlField; + } + + @Override + public void transform(DeployState deployState, Element parent, EmbedderOption option) { + String id = option.attributes.get("id"); + String url = option.attributes.get("url"); + String path = option.attributes.get("path"); + + // Always use path if it is set + if (path != null && path.length() > 0) { + createElement(parent, pathField, path); + createElement(parent, urlField, ""); + return; + } + + // Only use the id if we're on cloud + if (deployState.isHosted() && id != null && id.length() > 0) { + createElement(parent, urlField, EmbedderConfig.modelIdToUrl(id)); + createElement(parent, pathField, createDummyPath(deployState)); + return; + } + + // Otherwise, use url + if (url != null && url.length() > 0) { + createElement(parent, urlField, url); + createElement(parent, pathField, createDummyPath(deployState)); + return; + } + + if ( ! deployState.isHosted() && id != null && id.length() > 0) { + throw new IllegalArgumentException("Model option 'id' is not valid here"); + } + throw new IllegalArgumentException("Model option requires either a 'path' or a 'url' attribute"); + } + + private String createDummyPath(DeployState deployState) { + // For now, until we have optional config parameters, return services.xml as it is guaranteed to exist + return "services.xml"; + } + + } + + public static class Builder { + private String name = ""; + private boolean required = false; + private String value = ""; + private Map<String, String> attributes = Map.of(); + private OptionTransformer optionTransformer = defaultOptionTransformer; + private boolean set = false; + + public Builder name(String name) { + this.name = name; + return this; + } + + public Builder required(boolean required) { + this.required = required; + return this; + } + + public Builder value(String value) { + this.value = value; + this.set = true; + return this; + } + + public Builder attributes(Map<String, String> attributes) { + this.attributes = attributes; + return this; + } + + public Builder attributes(Element element) { + NamedNodeMap map = element.getAttributes(); + if (map.getLength() > 0) { + this.attributes = new HashMap<>(map.getLength()); + for (int i = 0; i < map.getLength(); ++i) { + String attr = map.item(i).getNodeName(); + attributes.put(attr, element.getAttribute(attr)); + } + } + return this; + } + + public Builder optionTransformer(OptionTransformer optionTransformer) { + this.optionTransformer = optionTransformer; + return this; + } + + public EmbedderOption build() { + return new EmbedderOption(this); + } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java index d2205fb64b3..495dc1bf2d5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelProbe.java @@ -49,9 +49,7 @@ public class OnnxModelProbe { } } - } catch (IllegalArgumentException | IOException | InterruptedException e) { - System.err.println(e.getMessage()); - } + } catch (IllegalArgumentException | IOException | InterruptedException ignored) { } return outputType; } diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index e8ff1721397..3fdbff84f6d 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -19,6 +19,7 @@ ContainerServices = DocumentApi? & Components* & Component* & + Embedder* & Handler* & Client* & Server* & @@ -103,6 +104,14 @@ ZooKeeper = element zookeeper { empty } +Embedder = element embedder { + attribute id { string }? & + attribute class { xsd:Name | JavaId }? & + attribute bundle { xsd:Name }? & + attribute def { xsd:Name }? & + anyElement* +} + ModelEvaluation = element model-evaluation { element onnx { element models { diff --git a/config-model/src/test/cfg/application/embed/services.xml b/config-model/src/test/cfg/application/embed/services.xml new file mode 100644 index 00000000000..f319d875ed9 --- /dev/null +++ b/config-model/src/test/cfg/application/embed/services.xml @@ -0,0 +1,26 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + + <container version="1.0"> + + <embedder id="test" class="ai.vespa.embedding.UndefinedEmbedder" bundle="dummy" def="test.dummy"> + <num>12</num> + <str>some text</str> + </embedder> + + <embedder id="transformer" class="ai.vespa.embedding.BertBaseEmbedder"> + <!-- model specifics --> + <model id="test-model-id" url="test-model-url"/> + <vocab path="files/vocab.txt"/> + + <!-- tunable parameters: number of threads etc --> + <onnxIntraOpThreads>4</onnxIntraOpThreads> + </embedder> + + <nodes> + <node hostalias="node1" /> + </nodes> + </container> + +</services> diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 81649375c8a..16b4a8bed9b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -1095,39 +1095,6 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { logger.msgs.get(1).getSecond()); } - @Test - public void logs_accesslog_not_overidable_in_hosted() { - String containerService = joinLines("<container id='foo' version='1.0'>", - " <accesslog type='json' fileNamePattern='logs/vespa/qrs/access.%Y%m%d%H%M%S' symlinkName='json_access' />", - " <nodes count=\"2\">", - " </nodes>", - "</container>"); - - String deploymentXml = joinLines("<deployment version='1.0'>", - " <prod>", - " <region>us-east-1</region>", - " </prod>", - "</deployment>"); - - ApplicationPackage applicationPackage = new MockApplicationPackage.Builder() - .withServices(containerService) - .withDeploymentSpec(deploymentXml) - .build(); - - TestLogger logger = new TestLogger(); - DeployState deployState = new DeployState.Builder() - .applicationPackage(applicationPackage) - .zone(new Zone(Environment.prod, RegionName.from("us-east-1"))) - .properties(new TestProperties().setHostedVespa(true)) - .deployLogger(logger) - .build(); - createModel(root, deployState, null, DomBuilderTest.parse(containerService)); - assertFalse(logger.msgs.isEmpty()); - assertEquals(Level.WARNING, logger.msgs.get(0).getFirst()); - assertEquals("Applications are not allowed to override the 'accesslog' element", - logger.msgs.get(0).getSecond()); - } - private void assertComponentConfigured(ApplicationContainerCluster cluster, String componentId) { Component<?, ?> component = cluster.getComponentsMap().get(ComponentId.fromString(componentId)); assertNotNull(component); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/EmbedderTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/EmbedderTestCase.java new file mode 100644 index 00000000000..0dae86473c8 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/EmbedderTestCase.java @@ -0,0 +1,223 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.component.ComponentId; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.path.Path; +import com.yahoo.text.XML; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.ConfigPayloadBuilder; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.xml.embedder.EmbedderConfig; +import org.junit.Test; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NamedNodeMap; +import org.xml.sax.SAXException; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class EmbedderTestCase { + + private static final String PREDEFINED_EMBEDDER_CLASS = "ai.vespa.embedding.BertBaseEmbedder"; + private static final String PREDEFINED_EMBEDDER_CONFIG = "embedding.bert-base-embedder"; + + @Test + public void testGenericEmbedConfig() throws IOException, SAXException { + String embedder = "<embedder id=\"test\" class=\"ai.vespa.test\" bundle=\"bundle\" def=\"def.name\">" + + " <val>123</val>" + + "</embedder>"; + String component = "<component id=\"test\" class=\"ai.vespa.test\" bundle=\"bundle\">" + + " <config name=\"def.name\">" + + " <val>123</val>" + + " </config>" + + "</component>"; + assertTransform(embedder, component); + } + + @Test + public void testGenericEmbedConfigRequiresBundleAndDef() throws IOException, SAXException { + assertTransformThrows("<embedder id=\"test\" class=\"ai.vespa.test\"></embedder>", + "Embedder configuration requires a bundle name"); + assertTransformThrows("<embedder id=\"test\" class=\"ai.vespa.test\" bundle=\"bundle\"></embedder>", + "Embedder configuration requires a config definition name"); + } + + @Test + public void testPredefinedEmbedConfigSelfHosted() throws IOException, SAXException { + assertTransformThrows("<embedder id=\"test\" class=\"" + PREDEFINED_EMBEDDER_CLASS + "\"></embedder>", + "Embedder '" + PREDEFINED_EMBEDDER_CLASS + "' requires options for [vocab, model]"); + assertTransformThrows("<embedder id=\"test\" class=\"" + PREDEFINED_EMBEDDER_CLASS + "\">" + + " <model />" + + " <vocab />" + + "</embedder>", + "Model option requires either a 'path' or a 'url' attribute"); + assertTransformThrows("<embedder id=\"test\" class=\"" + PREDEFINED_EMBEDDER_CLASS + "\">" + + " <model id=\"my_model_id\" />" + + " <vocab id=\"my_vocab_id\" />" + + "</embedder>", + "Model option 'id' is not valid here"); + + String embedder = "<embedder id=\"test\" class=\"" + PREDEFINED_EMBEDDER_CLASS + "\">" + + " <model id=\"my_model_id\" url=\"my-model-url\" />" + + " <vocab id=\"my_vocab_id\" url=\"my-vocab-url\" />" + + "</embedder>"; + String component = "<component id=\"test\" class=\"" + PREDEFINED_EMBEDDER_CLASS + "\" bundle=\"model-integration\">" + + " <config name=\"" + PREDEFINED_EMBEDDER_CONFIG + "\">" + + " <tokenizerVocabUrl>my-vocab-url</tokenizerVocabUrl>" + + " <tokenizerVocabPath></tokenizerVocabPath>" + + " <transformerModelUrl>my-model-url</transformerModelUrl>" + + " <transformerModelPath></transformerModelPath>" + + " </config>" + + "</component>"; + assertTransform(embedder, component, false); + + // Path has priority: + embedder = "<embedder id=\"test\" class=\"" + PREDEFINED_EMBEDDER_CLASS + "\">" + + " <model id=\"my_model_id\" url=\"my-model-url\" path=\"files/model.onnx\" />" + + " <vocab id=\"my_vocab_id\" url=\"my-vocab-url\" path=\"files/vocab.txt\" />" + + "</embedder>"; + component = "<component id=\"test\" class=\"" + PREDEFINED_EMBEDDER_CLASS + "\" bundle=\"model-integration\">" + + " <config name=\"" + PREDEFINED_EMBEDDER_CONFIG + "\">" + + " <tokenizerVocabPath>files/vocab.txt</tokenizerVocabPath>" + + " <tokenizerVocabUrl></tokenizerVocabUrl>" + + " <transformerModelPath>files/model.onnx</transformerModelPath>" + + " <transformerModelUrl></transformerModelUrl>" + + " </config>" + + "</component>"; + assertTransform(embedder, component, false); + } + + @Test + public void testPredefinedEmbedConfigCloud() throws IOException, SAXException { + String embedder = "<embedder id=\"test\" class=\"" + PREDEFINED_EMBEDDER_CLASS + "\" />"; + String component = "<component id=\"test\" class=\"" + PREDEFINED_EMBEDDER_CLASS + "\" bundle=\"model-integration\">" + + " <config name=\"" + PREDEFINED_EMBEDDER_CONFIG + "\">" + + " <tokenizerVocabUrl>some url</tokenizerVocabUrl>" + + " <tokenizerVocabPath></tokenizerVocabPath>" + + " <transformerModelUrl>some url</transformerModelUrl>" + + " <transformerModelPath></transformerModelPath>" + + " </config>" + + "</component>"; + assertTransform(embedder, component, true); + + embedder = "<embedder id=\"test\" class=\"" + PREDEFINED_EMBEDDER_CLASS + "\">" + + " <model id=\"my_model_id\" />" + + " <vocab id=\"my_vocab_id\" />" + + "</embedder>"; + assertTransformThrows(embedder, "Unknown model id: 'my_vocab_id'", true); + + embedder = "<embedder id=\"test\" class=\"" + PREDEFINED_EMBEDDER_CLASS + "\">" + + " <model id=\"test-model-id\" />" + + " <vocab id=\"test-model-id\" />" + + "</embedder>"; + component = "<component id=\"test\" class=\"" + PREDEFINED_EMBEDDER_CLASS + "\" bundle=\"model-integration\">" + + " <config name=\"" + PREDEFINED_EMBEDDER_CONFIG + "\">" + + " <tokenizerVocabUrl>test-model-url</tokenizerVocabUrl>" + + " <tokenizerVocabPath></tokenizerVocabPath>" + + " <transformerModelUrl>test-model-url</transformerModelUrl>" + + " <transformerModelPath></transformerModelPath>" + + " </config>" + + "</component>"; + assertTransform(embedder, component, true); + } + + @Test + public void testEmbedConfig() throws Exception { + final String emptyPathFileName = "services.xml"; + + Path applicationDir = Path.fromString("src/test/cfg/application/embed/"); + VespaModel model = loadModel(applicationDir, false); + ApplicationContainerCluster containerCluster = model.getContainerClusters().get("container"); + + Component<?, ?> testComponent = containerCluster.getComponentsMap().get(new ComponentId("test")); + ConfigPayloadBuilder testConfig = testComponent.getUserConfigs().get(new ConfigDefinitionKey("dummy", "test")); + assertEquals(testConfig.getObject("num").getValue(), "12"); + assertEquals(testConfig.getObject("str").getValue(), "some text"); + + Component<?, ?> transformer = containerCluster.getComponentsMap().get(new ComponentId("transformer")); + ConfigPayloadBuilder transformerConfig = transformer.getUserConfigs().get(new ConfigDefinitionKey("bert-base-embedder", "embedding")); + assertEquals(transformerConfig.getObject("transformerModelUrl").getValue(), "test-model-url"); + assertEquals(transformerConfig.getObject("transformerModelPath").getValue(), emptyPathFileName); + assertEquals(transformerConfig.getObject("tokenizerVocabUrl").getValue(), ""); + assertEquals(transformerConfig.getObject("tokenizerVocabPath").getValue(), "files/vocab.txt"); + } + + private VespaModel loadModel(Path path, boolean hosted) throws Exception { + FilesApplicationPackage applicationPackage = FilesApplicationPackage.fromFile(path.toFile()); + TestProperties properties = new TestProperties().setHostedVespa(hosted); + DeployState state = new DeployState.Builder().properties(properties).applicationPackage(applicationPackage).build(); + return new VespaModel(state); + } + + private void assertTransform(String embedder, String component) throws IOException, SAXException { + assertTransform(embedder, component, false); + } + + private void assertTransform(String embedder, String component, boolean hosted) throws IOException, SAXException { + Element emb = createElement(embedder); + Element cmp = createElement(component); + Element trans = EmbedderConfig.transform(createEmptyDeployState(hosted), emb); + assertSpec(cmp, trans); + } + + private void assertSpec(Element e1, Element e2) { + assertEquals(e1.getTagName(), e2.getTagName()); + assertAttributes(e1, e2); + assertAttributes(e2, e1); + assertChildren(e1, e2); + } + + private void assertAttributes(Element e1, Element e2) { + NamedNodeMap map = e1.getAttributes(); + for (int i = 0; i < map.getLength(); ++i) { + String attr = map.item(i).getNodeName(); + assertEquals(e1.getAttribute(attr), e2.getAttribute(attr)); + } + } + + private void assertChildren(Element e1, Element e2) { + List<Element> list1 = XML.getChildren(e1); + List<Element> list2 = XML.getChildren(e2); + assertEquals(list1.size(), list2.size()); + for (int i = 0; i < list1.size(); ++i) { + Element child1 = list1.get(i); + Element child2 = list2.get(i); + assertSpec(child1, child2); + } + } + + private void assertTransformThrows(String embedder, String msg) throws IOException, SAXException { + assertTransformThrows(embedder, msg, false); + } + + private void assertTransformThrows(String embedder, String msg, boolean hosted) throws IOException, SAXException { + try { + EmbedderConfig.transform(createEmptyDeployState(hosted), createElement(embedder)); + fail("Expected exception was not thrown: " + msg); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), msg); + } + } + + private Element createElement(String xml) throws IOException, SAXException { + Document doc = XML.getDocumentBuilder().parse(new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))); + return (Element) doc.getFirstChild(); + } + + private DeployState createEmptyDeployState(boolean hosted) { + TestProperties properties = new TestProperties().setHostedVespa(hosted); + return new DeployState.Builder().properties(properties).build(); + } + +} diff --git a/configdefinitions/src/vespa/athenz-provider-service.def b/configdefinitions/src/vespa/athenz-provider-service.def index 0363f299cd6..2131aa88d30 100644 --- a/configdefinitions/src/vespa/athenz-provider-service.def +++ b/configdefinitions/src/vespa/athenz-provider-service.def @@ -13,14 +13,17 @@ secretName string # Secret version secretVersion int +# Secret name of CA certificate +caCertSecretName string + # Certificate DNS suffix -certDnsSuffix string +certDnsSuffix string default="" # Athenz ZTS server url -ztsUrl string +ztsUrl string default="" # Path to Athenz CA JKS trust store -athenzCaTrustStore string +athenzCaTrustStore string default="" # Period between certificate updates updatePeriodDays int default=1 diff --git a/configdefinitions/src/vespa/embedding.bert-base-embedder.def b/configdefinitions/src/vespa/embedding.bert-base-embedder.def index a37599de411..115e021972c 100644 --- a/configdefinitions/src/vespa/embedding.bert-base-embedder.def +++ b/configdefinitions/src/vespa/embedding.bert-base-embedder.def @@ -1,8 +1,13 @@ namespace=embedding +# Settings for wordpiece tokenizer +tokenizerVocabUrl url +tokenizerVocabPath path + # Transformer model settings -transformerModelUrl url default=https://data.vespa.oath.cloud/onnx_models/sentence_all_MiniLM_L6_v2.onnx +transformerModelUrl url +transformerModelPath path # Max length of token sequence model can handle transformerMaxTokens int default=384 @@ -23,5 +28,3 @@ onnxExecutionMode enum { parallel, sequential } default=sequential onnxInterOpThreads int default=1 onnxIntraOpThreads int default=-4 # n=number of threads -> n<0: CPUs/(-n), n==0: CPUs, n>0: n -# Settings for wordpiece tokenizer -tokenizerVocabUrl url default=https://data.vespa.oath.cloud/onnx_models/bert-base-uncased-vocab.txt diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index b13d4542eba..fd4f1824885 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -1135,9 +1135,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye .anyMatch(serviceInfo -> serviceInfo.getServiceType().equalsIgnoreCase("logserver"))) .findFirst().orElseThrow(() -> new IllegalArgumentException("Could not find host info for logserver")); - ServiceInfo serviceInfo = logServerHostInfo.getServices().stream().filter(service -> List.of(LOGSERVER_CONTAINER.serviceName, CONTAINER.serviceName).contains(service.getServiceType())) - .findFirst().orElseThrow(() -> new IllegalArgumentException("No container running on logserver host")); - int port = servicePort(serviceInfo); + ServiceInfo logService = logServerHostInfo.getServices().stream() + .filter(service -> LOGSERVER_CONTAINER.serviceName.equals(service.getServiceType())) + .findFirst() + .or(() -> logServerHostInfo.getServices().stream() + .filter(service -> CONTAINER.serviceName.equals(service.getServiceType())) + .findFirst()) + .orElseThrow(() -> new IllegalArgumentException("No container running on logserver host")); + int port = servicePort(logService); return "http://" + logServerHostInfo.getHostname() + ":" + port + "/logs"; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 328f903d095..15bd898c053 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -13,7 +13,7 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; import com.yahoo.vespa.config.ConnectionPool; import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.filedistribution.CompressedFileReference; +import com.yahoo.vespa.filedistribution.FileReferenceCompressor; import com.yahoo.vespa.filedistribution.EmptyFileReferenceData; import com.yahoo.vespa.filedistribution.FileDistributionConnectionPool; import com.yahoo.vespa.filedistribution.FileDownloader; @@ -36,6 +36,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getOtherConfigServersInCluster; +import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.compressed; public class FileServer { @@ -123,21 +124,15 @@ public class FileServer { private void serveFile(FileReference reference, Receiver target) { File file = root.getFile(reference); log.log(Level.FINE, () -> "Start serving " + reference + " with file '" + file.getAbsolutePath() + "'"); - boolean success = false; - String errorDescription = "OK"; FileReferenceData fileData = EmptyFileReferenceData.empty(reference, file.getName()); try { fileData = readFileReferenceData(reference); - success = true; - } catch (IOException e) { - errorDescription = "For" + reference.value() + ": failed reading file '" + file.getAbsolutePath() + "'"; - log.warning(errorDescription + " for sending to '" + target.toString() + "'. " + e.toString()); - fileData.close(); - } - - try { - target.receive(fileData, new ReplayStatus(success ? 0 : 1, success ? "OK" : errorDescription)); + target.receive(fileData, new ReplayStatus(0, "OK")); log.log(Level.FINE, () -> "Done serving " + reference.value() + " with file '" + file.getAbsolutePath() + "'"); + } catch (IOException e) { + String errorDescription = "For" + reference.value() + ": failed reading file '" + file.getAbsolutePath() + "'"; + log.warning(errorDescription + " for sending to '" + target.toString() + "'. " + e.getMessage()); + target.receive(fileData, new ReplayStatus(1, errorDescription)); } catch (Exception e) { log.log(Level.WARNING, "Failed serving " + reference + ": " + Exceptions.toMessageString(e)); } finally { @@ -150,8 +145,8 @@ public class FileServer { if (file.isDirectory()) { Path tempFile = Files.createTempFile("filereferencedata", reference.value()); - File compressedFile = CompressedFileReference.compress(file.getParentFile(), tempFile.toFile()); - return new LazyTemporaryStorageFileReferenceData(reference, file.getName(), FileReferenceData.Type.compressed, compressedFile); + File compressedFile = new FileReferenceCompressor(compressed).compress(file.getParentFile(), tempFile.toFile()); + return new LazyTemporaryStorageFileReferenceData(reference, file.getName(), compressed, compressedFile); } else { return new LazyFileReferenceData(reference, file.getName(), FileReferenceData.Type.file, file); } @@ -197,7 +192,6 @@ public class FileServer { request.returnRequest(); } - boolean hasFileDownloadIfNeeded(FileReferenceDownload fileReferenceDownload) { FileReference fileReference = fileReferenceDownload.fileReference(); if (hasFile(fileReference)) return true; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java index 5fa8edb7803..14586e2289e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/LogRetriever.java @@ -16,8 +16,8 @@ public class LogRetriever { private final CloseableHttpClient httpClient = VespaHttpClientBuilder.create().build(); - public HttpResponse getLogs(String logServerHostname) { - HttpGet get = new HttpGet(logServerHostname); + public HttpResponse getLogs(String logServerUri) { + HttpGet get = new HttpGet(logServerUri); try { return new ProxyResponse(httpClient.execute(get)); } catch (IOException e) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java index bb7dedac6d1..881d3de4eaa 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java @@ -10,6 +10,7 @@ import com.yahoo.config.model.api.TenantSecretStore; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.TenantName; import com.yahoo.path.Path; @@ -142,6 +143,10 @@ public abstract class Session implements Comparable<Session> { sessionZooKeeperClient.writeOperatorCertificates(operatorCertificates); } + public void setCloudAccount(Optional<CloudAccount> cloudAccount) { + sessionZooKeeperClient.writeCloudAccount(cloudAccount); + } + /** Returns application id read from ZooKeeper. Will throw RuntimeException if not found */ public ApplicationId getApplicationId() { return sessionZooKeeperClient.readApplicationId() @@ -181,6 +186,10 @@ public abstract class Session implements Comparable<Session> { return sessionZooKeeperClient.readOperatorCertificates(); } + public Optional<CloudAccount> getCloudAccount() { + return sessionZooKeeperClient.readCloudAccount(); + } + private Transaction createSetStatusTransaction(Status status) { return sessionZooKeeperClient.createWriteStatusTransaction(status); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index a5305f8f886..f572f241ce0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -12,13 +12,11 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.config.model.application.provider.DeployData; import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.io.IOUtils; -import com.yahoo.lang.SettableOptional; import com.yahoo.path.Path; import com.yahoo.transaction.AbstractTransaction; import com.yahoo.transaction.NestedTransaction; @@ -52,6 +50,7 @@ import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.zookeeper.KeeperException; + import java.io.File; import java.io.FilenameFilter; import java.io.IOException; @@ -279,6 +278,7 @@ public class SessionRepository { session.setAthenzDomain(existingSession.getAthenzDomain()); session.setTenantSecretStores(existingSession.getTenantSecretStores()); session.setOperatorCertificates(existingSession.getOperatorCertificates()); + session.setCloudAccount(existingSession.getCloudAccount()); return session; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockLogRetriever.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockLogRetriever.java index d7014c74d76..6a95c059fcc 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockLogRetriever.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockLogRetriever.java @@ -14,7 +14,7 @@ import java.nio.charset.StandardCharsets; public class MockLogRetriever extends LogRetriever { @Override - public HttpResponse getLogs(String logServerHostname) { + public HttpResponse getLogs(String logServerUri) { return new MockHttpResponse(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index 800cf7bad60..c58b550d660 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -15,6 +15,7 @@ import com.yahoo.config.model.provision.Hosts; import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.test.HostedConfigModelRegistry; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; @@ -453,6 +454,21 @@ public class HostedDeployTest { tester.tenant().getApplicationRepo().database().readReindexingStatus(tester.applicationId())); } + @Test + public void testRedeployWithCloudAccount() { + CloudAccount cloudAccount = new CloudAccount("012345678912"); + DeployTester tester = new DeployTester.Builder(temporaryFolder) + .modelFactory(createHostedModelFactory(Version.fromString("4.5.6"), Clock.systemUTC())) + .build(); + tester.deployApp("src/test/apps/hosted/", new PrepareParams.Builder() + .vespaVersion("4.5.6") + .cloudAccount(cloudAccount)); + Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(tester.applicationId()); + assertTrue(deployment.isPresent()); + deployment.get().activate(); + assertEquals(cloudAccount, ((Deployment) deployment.get()).session().getCloudAccount().get()); + } + /** Create the given number of hosts using the supplied versions--the last version is repeated as needed. */ private List<Host> createHosts(int count, String ... versions) { return IntStream.rangeClosed(1, count) diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java index b3a7ebc761a..686d23c4c99 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java @@ -64,6 +64,7 @@ class MetricDefinitions { static final String SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_PROTOCOLS = "jdisc.http.ssl.handshake.failure.incompatible_protocols"; static final String SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_CIPHERS = "jdisc.http.ssl.handshake.failure.incompatible_ciphers"; static final String SSL_HANDSHAKE_FAILURE_UNKNOWN = "jdisc.http.ssl.handshake.failure.unknown"; + static final String SSL_HANDSHAKE_FAILURE_CONNECTION_CLOSED = "jdisc.http.ssl.handshake.failure.connection_closed"; static final String JETTY_THREADPOOL_MAX_THREADS = "jdisc.http.jetty.threadpool.thread.max"; static final String JETTY_THREADPOOL_MIN_THREADS = "jdisc.http.jetty.threadpool.thread.min"; diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailure.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailure.java index 331d8837bfa..20ce15d683f 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailure.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailure.java @@ -1,7 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.server.jetty; +import org.eclipse.jetty.io.EofException; + import javax.net.ssl.SSLHandshakeException; +import java.io.IOException; import java.util.Optional; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -33,29 +36,49 @@ enum SslHandshakeFailure { INVALID_CLIENT_CERT( MetricDefinitions.SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT, // Includes mismatch of client certificate and private key "INVALID_CLIENT_CERTIFICATE", - "(PKIX path (building|validation) failed: .+)|(Invalid CertificateVerify signature)"); + "(PKIX path (building|validation) failed: .+)|(Invalid CertificateVerify signature)"), + CONNECTION_CLOSED( + MetricDefinitions.SSL_HANDSHAKE_FAILURE_CONNECTION_CLOSED, + "CONNECTION_CLOSED", + e -> e.getCause() instanceof EofException + && e.getCause().getCause() instanceof IOException + && e.getCause().getCause().getMessage().equals("Broken pipe")); private final String metricName; private final String failureType; - private final Predicate<String> messageMatcher; + private final Predicate<SSLHandshakeException> predicate; SslHandshakeFailure(String metricName, String failureType, String messagePattern) { + this(metricName, failureType, new MessagePatternPredicate(messagePattern)); + } + + SslHandshakeFailure(String metricName, String failureType, Predicate<SSLHandshakeException> predicate) { this.metricName = metricName; this.failureType = failureType; - this.messageMatcher = Pattern.compile(messagePattern).asMatchPredicate(); + this.predicate = predicate; } String metricName() { return metricName; } String failureType() { return failureType; } static Optional<SslHandshakeFailure> fromSslHandshakeException(SSLHandshakeException exception) { - String message = exception.getMessage(); - if (message == null || message.isBlank()) return Optional.empty(); - for (SslHandshakeFailure failure : values()) { - if (failure.messageMatcher.test(message)) { - return Optional.of(failure); - } + for (SslHandshakeFailure type : values()) { + if (type.predicate.test(exception)) return Optional.of(type); } return Optional.empty(); } + + private static class MessagePatternPredicate implements Predicate<SSLHandshakeException> { + final Pattern pattern; + + MessagePatternPredicate(String pattern) { this.pattern = Pattern.compile(pattern); } + + @Override + public boolean test(SSLHandshakeException e) { + String message = e.getMessage(); + if (message == null || message.isBlank()) return false; + return pattern.matcher(message).matches(); + } + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java index 69eda662692..46e7fb48553 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveService.java @@ -3,7 +3,9 @@ package com.yahoo.vespa.hosted.controller.api.integration.archive; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; +import java.net.URI; import java.util.Map; import java.util.Set; @@ -17,7 +19,9 @@ public interface ArchiveService { ArchiveBucket createArchiveBucketFor(ZoneId zoneId); - void updateBucketPolicy(ZoneId zoneId, ArchiveBucket bucket, Map<TenantName, String> authorizeIamRoleByTenantName); + void updatePolicies(ZoneId zoneId, Set<ArchiveBucket> buckets, Map<TenantName,ArchiveAccess> authorizeAccessByTenantName); - void updateKeyPolicy(ZoneId zoneId, String keyArn, Set<String> tenantAuthorizedIamRoles); + boolean canAddTenantToBucket(ZoneId zoneId, ArchiveBucket bucket); + + URI bucketURI(ZoneId zoneId, String bucketName, TenantName tenantName); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java index ce7b56ad1f6..a2847439ce7 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/MockArchiveService.java @@ -3,8 +3,11 @@ package com.yahoo.vespa.hosted.controller.api.integration.archive; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; +import java.net.URI; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.TreeMap; @@ -15,8 +18,10 @@ import java.util.TreeMap; */ public class MockArchiveService implements ArchiveService { - public Map<ArchiveBucket, Map<TenantName, String>> authorizedIamRolesForBucket = new HashMap<>(); - public Map<String, Set<String>> authorizedIamRolesForKey = new TreeMap<>(); + + public Set<ArchiveBucket> archiveBuckets = new HashSet<>(); + public Map<TenantName, ArchiveAccess> authorizeAccessByTenantName = new HashMap<>(); + @Override public ArchiveBucket createArchiveBucketFor(ZoneId zoneId) { @@ -24,12 +29,18 @@ public class MockArchiveService implements ArchiveService { } @Override - public void updateBucketPolicy(ZoneId zoneId, ArchiveBucket bucket, Map<TenantName, String> authorizeIamRoleByTenantName) { - authorizedIamRolesForBucket.put(bucket, authorizeIamRoleByTenantName); + public void updatePolicies(ZoneId zoneId, Set<ArchiveBucket> buckets, Map<TenantName, ArchiveAccess> authorizeAccessByTenantName) { + this.archiveBuckets = new HashSet<>(buckets); + this.authorizeAccessByTenantName = new HashMap<>(authorizeAccessByTenantName); + } + + @Override + public boolean canAddTenantToBucket(ZoneId zoneId, ArchiveBucket bucket) { + return bucket.tenants().size() < 5; } @Override - public void updateKeyPolicy(ZoneId zoneId, String keyArn, Set<String> tenantAuthorizedIamRoles) { - authorizedIamRolesForKey.put(keyArn, tenantAuthorizedIamRoles); + public URI bucketURI(ZoneId zoneId, String bucketName, TenantName tenantName) { + return URI.create(String.format("s3://%s/%s/", bucketName, tenantName.value())); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index bff3fa14b89..1c6642f2dc6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -64,7 +64,10 @@ enum PathGroup { "/application/v4/tenant/{tenant}/key/"), tenantArchiveAccess(Matcher.tenant, - "/application/v4/tenant/{tenant}/archive-access"), + "/application/v4/tenant/{tenant}/archive-access", + "/application/v4/tenant/{tenant}/archive-access/aws", + "/application/v4/tenant/{tenant}/archive-access/gcp"), + billingToken(Matcher.tenant, "/billing/v1/tenant/{tenant}/token"), diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java index a739a8e2b01..60950341a42 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java @@ -177,7 +177,7 @@ public class SystemFlagsDataArchive { if (!filenamesForSystem.isEmpty() && !filenamesForSystem.contains(filename)) { if (systemDefinition != null && filename.startsWith(systemDefinition.system().value() + '.')) { throw new IllegalArgumentException(String.format( - "Environment or zone in filename '%s' is does not exist", filename)); + "Environment or zone in filename '%s' does not exist", filename)); } return; // Ignore files irrelevant for system } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/ArchiveAccess.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/ArchiveAccess.java new file mode 100644 index 00000000000..fba361f9223 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/ArchiveAccess.java @@ -0,0 +1,105 @@ +package com.yahoo.vespa.hosted.controller.tenant; + +import com.yahoo.text.Text; + +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class ArchiveAccess { + + private static final Pattern VALID_AWS_ARCHIVE_ACCESS_ROLE_PATTERN = Pattern.compile("arn:aws:iam::\\d{12}:.+"); + private static final Pattern VALID_GCP_ARCHIVE_ACCESS_MEMBER_PATTERN = Pattern.compile("(?<prefix>[a-zA-Z]+):.+"); + + private static final Set<String> gcpMemberPrefixes = Set.of("user", "serviceAccount", "group", "domain"); + + // AWS IAM Role + private final Optional<String> awsRole; + // GCP Member + private final Optional<String> gcpMember; + + public ArchiveAccess() { + this(Optional.empty(), Optional.empty()); + } + + private ArchiveAccess(Optional<String> awsRole, Optional<String> gcpMember) { + this.awsRole = awsRole; + this.gcpMember = gcpMember; + + awsRole.ifPresent(role -> validateAWSIAMRole(role)); + gcpMember.ifPresent(member -> validateGCPMember(member)); + } + + public ArchiveAccess withAWSRole(String role) { + return new ArchiveAccess(Optional.of(role), gcpMember()); + } + + public ArchiveAccess withGCPMember(String member) { + return new ArchiveAccess(awsRole(), Optional.of(member)); + } + + public ArchiveAccess withAWSRole(Optional<String> role) { + return new ArchiveAccess(role, gcpMember()); + } + + public ArchiveAccess withGCPMember(Optional<String> member) { + return new ArchiveAccess(awsRole(), member); + } + + public ArchiveAccess removeAWSRole() { + return new ArchiveAccess(Optional.empty(), gcpMember()); + } + + public ArchiveAccess removeGCPMember() { + return new ArchiveAccess(awsRole(), Optional.empty()); + } + + private void validateAWSIAMRole(String role) { + if (!VALID_AWS_ARCHIVE_ACCESS_ROLE_PATTERN.matcher(role).matches()) { + throw new IllegalArgumentException(Text.format("Invalid archive access role '%s': Must match expected pattern: '%s'", + awsRole.get(), VALID_AWS_ARCHIVE_ACCESS_ROLE_PATTERN.pattern())); + } + if (role.length() > 100) { + throw new IllegalArgumentException("Invalid archive access role too long, must be 100 or less characters"); + } + } + + private void validateGCPMember(String member) { + var matcher = VALID_GCP_ARCHIVE_ACCESS_MEMBER_PATTERN.matcher(member); + if (!matcher.matches()) { + throw new IllegalArgumentException(Text.format("Invalid GCP archive access member '%s': Must match expected pattern: '%s'", + gcpMember.get(), VALID_GCP_ARCHIVE_ACCESS_MEMBER_PATTERN.pattern())); + } + var prefix = matcher.group("prefix"); + if (!gcpMemberPrefixes.contains(prefix)) { + throw new IllegalArgumentException(Text.format("Invalid GCP member prefix '%s', must be one of '%s'", + prefix, gcpMemberPrefixes.stream().collect(Collectors.joining(", ")))); + } + if (!"domain".equals(prefix) && !member.contains("@")) { + throw new IllegalArgumentException(Text.format("Invalid GCP member '%s', prefix '%s' must be followed by an email id", member, prefix)); + } + } + + public Optional<String> awsRole() { + return awsRole; + } + + public Optional<String> gcpMember() { + return gcpMember; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ArchiveAccess that = (ArchiveAccess) o; + return awsRole.equals(that.awsRole) && gcpMember.equals(that.gcpMember); + } + + @Override + public int hashCode() { + return Objects.hash(awsRole, gcpMember); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java index 9a4e04ebb3a..54924b9c456 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.tenant; import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableBiMap; import com.yahoo.config.provision.TenantName; -import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import java.security.Principal; @@ -13,7 +12,6 @@ import java.time.Instant; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.regex.Pattern; /** * A paying tenant in a Vespa cloud service. @@ -22,29 +20,22 @@ import java.util.regex.Pattern; */ public class CloudTenant extends Tenant { - private static final Pattern VALID_ARCHIVE_ACCESS_ROLE_PATTERN = Pattern.compile("arn:aws:iam::\\d{12}:.+"); - private final Optional<Principal> creator; private final BiMap<PublicKey, Principal> developerKeys; private final TenantInfo info; private final List<TenantSecretStore> tenantSecretStores; - private final Optional<String> archiveAccessRole; + private final ArchiveAccess archiveAccess; /** Public for the serialization layer — do not use! */ public CloudTenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<Principal> creator, BiMap<PublicKey, Principal> developerKeys, TenantInfo info, - List<TenantSecretStore> tenantSecretStores, Optional<String> archiveAccessRole) { + List<TenantSecretStore> tenantSecretStores, ArchiveAccess archiveAccess) { super(name, createdAt, lastLoginInfo, Optional.empty()); this.creator = creator; this.developerKeys = developerKeys; this.info = Objects.requireNonNull(info); this.tenantSecretStores = tenantSecretStores; - this.archiveAccessRole = archiveAccessRole; - if (!archiveAccessRole.map(role -> VALID_ARCHIVE_ACCESS_ROLE_PATTERN.matcher(role).matches()).orElse(true)) - throw new IllegalArgumentException(Text.format("Invalid archive access role '%s': Must match expected pattern: '%s'", - archiveAccessRole.get(), VALID_ARCHIVE_ACCESS_ROLE_PATTERN.pattern())); - if (archiveAccessRole.map(role -> role.length() > 100).orElse(false)) - throw new IllegalArgumentException("Invalid archive access role too long, must be 100 or less characters"); + this.archiveAccess = Objects.requireNonNull(archiveAccess); } /** Creates a tenant with the given name, provided it passes validation. */ @@ -53,7 +44,7 @@ public class CloudTenant extends Tenant { createdAt, LastLoginInfo.EMPTY, Optional.ofNullable(creator), - ImmutableBiMap.of(), TenantInfo.empty(), List.of(), Optional.empty()); + ImmutableBiMap.of(), TenantInfo.empty(), List.of(), new ArchiveAccess()); } /** The user that created the tenant */ @@ -66,11 +57,6 @@ public class CloudTenant extends Tenant { return info; } - /** An iam role which is allowed to access the S3 (log, dump) archive) */ - public Optional<String> archiveAccessRole() { - return archiveAccessRole; - } - /** Returns the set of developer keys and their corresponding developers for this tenant. */ public BiMap<PublicKey, Principal> developerKeys() { return developerKeys; } @@ -79,6 +65,16 @@ public class CloudTenant extends Tenant { return tenantSecretStores; } + /** + * Role or member that is allowed to access archive bucket (log, dump) + * + * For AWS is this the IAM role + * For GCP it is a GCP member + */ + public ArchiveAccess archiveAccess() { + return archiveAccess; + } + @Override public Type type() { return Type.cloud; diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java index e4c46cfed1a..ce2746f92e0 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java @@ -135,7 +135,7 @@ public class SystemFlagsDataArchiveTest { Path directory = Paths.get("src/test/resources/system-flags-with-unknown-file-name/"); expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage( - "Environment or zone in filename 'main.prod.unknown-region.json' is does not exist"); + "Environment or zone in filename 'main.prod.unknown-region.json' does not exist"); SystemFlagsDataArchive.fromDirectoryAndSystem(directory, createZoneRegistryMock()); } diff --git a/controller-server/pom.xml b/controller-server/pom.xml index 5cf53929a98..ff7367e03b2 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -102,12 +102,6 @@ </dependency> <dependency> - <groupId>javax.ws.rs</groupId> - <artifactId>javax.ws.rs-api</artifactId> - <scope>provided</scope> - </dependency> - - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>flags</artifactId> <version>${project.version}</version> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java index 7a0e60aacb4..4f58e87035b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java @@ -8,11 +8,11 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.security.KeyUtils; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.athenz.api.AthenzDomain; -import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; +import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; @@ -128,26 +128,26 @@ public abstract class LockedTenant { private final BiMap<PublicKey, Principal> developerKeys; private final TenantInfo info; private final List<TenantSecretStore> tenantSecretStores; - private final Optional<String> archiveAccessRole; + private final ArchiveAccess archiveAccess; private Cloud(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<Principal> creator, BiMap<PublicKey, Principal> developerKeys, TenantInfo info, - List<TenantSecretStore> tenantSecretStores, Optional<String> archiveAccessRole) { + List<TenantSecretStore> tenantSecretStores, ArchiveAccess archiveAccess) { super(name, createdAt, lastLoginInfo); this.developerKeys = ImmutableBiMap.copyOf(developerKeys); this.creator = creator; this.info = info; this.tenantSecretStores = tenantSecretStores; - this.archiveAccessRole = archiveAccessRole; + this.archiveAccess = archiveAccess; } private Cloud(CloudTenant tenant) { - this(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.creator(), tenant.developerKeys(), tenant.info(), tenant.tenantSecretStores(), tenant.archiveAccessRole()); + this(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), tenant.creator(), tenant.developerKeys(), tenant.info(), tenant.tenantSecretStores(), tenant.archiveAccess()); } @Override public CloudTenant get() { - return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccessRole); + return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess); } public Cloud withDeveloperKey(PublicKey key, Principal principal) { @@ -155,38 +155,38 @@ public abstract class LockedTenant { if (keys.containsKey(key)) throw new IllegalArgumentException("Key " + KeyUtils.toPem(key) + " is already owned by " + keys.get(key)); keys.put(key, principal); - return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccessRole); + return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess); } public Cloud withoutDeveloperKey(PublicKey key) { BiMap<PublicKey, Principal> keys = HashBiMap.create(developerKeys); keys.remove(key); - return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccessRole); + return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess); } public Cloud withInfo(TenantInfo newInfo) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, newInfo, tenantSecretStores, archiveAccessRole); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, newInfo, tenantSecretStores, archiveAccess); } @Override public LockedTenant with(LastLoginInfo lastLoginInfo) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccessRole); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess); } public Cloud withSecretStore(TenantSecretStore tenantSecretStore) { ArrayList<TenantSecretStore> secretStores = new ArrayList<>(tenantSecretStores); secretStores.add(tenantSecretStore); - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccessRole); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess); } public Cloud withoutSecretStore(TenantSecretStore tenantSecretStore) { ArrayList<TenantSecretStore> secretStores = new ArrayList<>(tenantSecretStores); secretStores.remove(tenantSecretStore); - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccessRole); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores, archiveAccess); } - public Cloud withArchiveAccessRole(Optional<String> role) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, role); + public Cloud withArchiveAccess(ArchiveAccess archiveAccess) { + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java index 21914b87818..0a0adcfc252 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDb.java @@ -46,7 +46,7 @@ public class CuratorArchiveBucketDb { return getBucketNameFromCache(zoneId, tenant) .or(() -> findAndUpdateArchiveUriCache(zoneId, tenant, buckets(zoneId))) .or(() -> createIfMissing ? Optional.of(assignToBucket(zoneId, tenant)) : Optional.empty()) - .map(bucketName -> URI.create(Text.format("s3://%s/%s/", bucketName, tenant.value()))); + .map(bucketName -> archiveService.bucketURI(zoneId, bucketName, tenant)); } private String assignToBucket(ZoneId zoneId, TenantName tenant) { @@ -57,7 +57,7 @@ public class CuratorArchiveBucketDb { .orElseGet(() -> { // If not, find an existing bucket with space Optional<ArchiveBucket> unfilledBucket = zoneBuckets.stream() - .filter(bucket -> bucket.tenants().size() < tenantsPerBucket().orElse(Integer.MAX_VALUE)) + .filter(bucket -> archiveService.canAddTenantToBucket(zoneId, bucket)) .findAny(); // And place the tenant in that bucket. @@ -94,23 +94,6 @@ public class CuratorArchiveBucketDb { return bucketName; } - private OptionalInt tenantsPerBucket() { - if (system.isPublic()) { - /* - * Due to policy limits, we can't put data for more than this many tenants in a bucket. - * Policy size limit is 20kb, about 550 bytes for non-tenant related policies. Each tenant - * needs about 500 + len(role_arn) bytes, we limit role_arn to 100 characters, so we can - * fit about (20k - 550) / 600 ~ 32 tenants per bucket. - */ - return OptionalInt.of(30); - } else { - /* - * The S3 policies in main/cd have a fixed size. - */ - return OptionalInt.empty(); - } - } - private Optional<String> getBucketNameFromCache(ZoneId zoneId, TenantName tenantName) { return Optional.ofNullable(archiveUriCache.get(zoneId)).map(map -> map.get(tenantName)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index e4869cc9bf4..3bc2aec95ff 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.restapi.RestApiException; import com.yahoo.text.Text; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; @@ -33,7 +34,6 @@ import com.yahoo.vespa.hosted.controller.security.TenantSpec; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import javax.ws.rs.ForbiddenException; import java.time.Instant; import java.util.Arrays; import java.util.List; @@ -197,7 +197,7 @@ public class AthenzFacade implements AccessControl { } catch (ZmsClientException e) { if (e.getErrorCode() == com.yahoo.jdisc.Response.Status.FORBIDDEN) - throw new ForbiddenException("Not authorized to create application", e); + throw new RestApiException.Forbidden("Not authorized to create application", e); else throw e; } @@ -289,7 +289,7 @@ public class AthenzFacade implements AccessControl { private void verifyIsDomainAdmin(AthenzIdentity identity, AthenzDomain domain) { log("getMembership(domain=%s, role=%s, principal=%s)", domain, "admin", identity); if ( ! zmsClient.getMembership(new AthenzRole(domain, "admin"), identity)) - throw new ForbiddenException( + throw new RestApiException.Forbidden( Text.format("The user '%s' is not admin in Athenz domain '%s'", identity.getFullName(), domain.getName())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index be07a2b0cb1..f7452323955 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -7,9 +7,7 @@ import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.InstanceName; -import com.yahoo.text.Text; import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; @@ -148,9 +146,11 @@ public class DeploymentTrigger { return; } - applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> + applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { + if (application.get().deploymentSpec().instance(id.instance()).isPresent()) applications().store(application.with(id.instance(), - instance -> withRemainingChange(instance, instance.change(), jobs.deploymentStatus(application.get()))))); + instance -> withRemainingChange(instance, instance.change(), jobs.deploymentStatus(application.get())))); + }); } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index fb7a3fc4ba5..632eb556444 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -662,7 +662,10 @@ public class InternalStepRunner implements StepRunner { "or a Java test bundle under 'components/' with at least one test with the annotation " + "for this suite. See docs.vespa.ai/en/testing.html for details."); controller.jobController().updateTestReport(id); - return Optional.of(noTests); + + DeploymentSpec spec = controller.applications().requireApplication(TenantAndApplicationId.from(id.application())).deploymentSpec(); + boolean requireTests = spec.steps().stream().anyMatch(step -> step.concerns(id.type().environment())); + return Optional.of(requireTests ? testFailure : noTests); } case SUCCESS: logger.log("Tests completed successfully."); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 30f16acf77d..9331c9942b6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -69,8 +69,12 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.endStagingSetup; import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal; +import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; import static java.time.temporal.ChronoUnit.SECONDS; +import static java.util.Comparator.comparing; import static java.util.Comparator.naturalOrder; import static java.util.function.Predicate.not; import static java.util.logging.Level.INFO; @@ -179,11 +183,27 @@ public class JobController { if (deployment.isEmpty() || deployment.get().at().isBefore(run.start())) return run; - Instant from = run.lastVespaLogTimestamp().isAfter(deployment.get().at()) ? run.lastVespaLogTimestamp() : deployment.get().at(); + Instant deployedAt = run.stepInfo(installInitialReal).or(() -> run.stepInfo(installReal)).flatMap(StepInfo::startTime).orElseThrow(); + Instant from = run.lastVespaLogTimestamp().isAfter(run.start()) ? run.lastVespaLogTimestamp() : deployedAt.minusSeconds(10); List<LogEntry> log = LogEntry.parseVespaLog(controller.serviceRegistry().configServer() .getLogs(new DeploymentId(id.application(), zone), Map.of("from", Long.toString(from.toEpochMilli()))), from); + + if (run.hasStep(installTester) && run.versions().targetPlatform().isAfter(new Version("7.590"))) { // todo jonmv: remove + deployedAt = run.stepInfo(installTester).flatMap(StepInfo::startTime).orElseThrow(); + from = run.lastVespaLogTimestamp().isAfter(run.start()) ? run.lastVespaLogTimestamp() : deployedAt.minusSeconds(10); + List<LogEntry> testerLog = LogEntry.parseVespaLog(controller.serviceRegistry().configServer() + .getLogs(new DeploymentId(id.tester().id(), zone), + Map.of("from", Long.toString(from.toEpochMilli()))), + from); + + Instant justNow = controller.clock().instant().minusSeconds(2); + log = Stream.concat(log.stream(), testerLog.stream()) + .filter(entry -> entry.at().isBefore(justNow)) + .sorted(comparing(LogEntry::at)) + .collect(toUnmodifiableList()); + } if (log.isEmpty()) return run; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java index 03cc6c6ba8d..7ffaaabb1a7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -45,8 +45,9 @@ public class Run { // For deserialisation only -- do not use! public Run(RunId id, Map<Step, StepInfo> steps, Versions versions, boolean isRedeployment, Instant start, Optional<Instant> end, - Optional<Instant> sleepUntil, RunStatus status, long lastTestRecord, Instant lastVespaLogTimestamp, Optional<Instant> noNodesDownSince, - Optional<ConvergenceSummary> convergenceSummary, Optional<X509Certificate> testerCertificate, boolean dryRun, Optional<String> reason) { + Optional<Instant> sleepUntil, RunStatus status, long lastTestRecord, Instant lastVespaLogTimestamp, + Optional<Instant> noNodesDownSince, Optional<ConvergenceSummary> convergenceSummary, + Optional<X509Certificate> testerCertificate, boolean dryRun, Optional<String> reason) { this.id = id; this.steps = Collections.unmodifiableMap(new EnumMap<>(steps)); this.versions = versions; @@ -67,8 +68,9 @@ public class Run { public static Run initial(RunId id, Versions versions, boolean isRedeployment, Instant now, JobProfile profile, Optional<String> triggeredBy) { EnumMap<Step, StepInfo> steps = new EnumMap<>(Step.class); profile.steps().forEach(step -> steps.put(step, StepInfo.initial(step))); - return new Run(id, steps, requireNonNull(versions), isRedeployment, requireNonNull(now), Optional.empty(), Optional.empty(), running, - -1, Instant.EPOCH, Optional.empty(), Optional.empty(), Optional.empty(), profile == JobProfile.developmentDryRun, triggeredBy); + return new Run(id, steps, requireNonNull(versions), isRedeployment, requireNonNull(now), Optional.empty(), + Optional.empty(), running, -1, Instant.EPOCH, Optional.empty(), Optional.empty(), + Optional.empty(), profile == JobProfile.developmentDryRun, triggeredBy); } /** Returns a new Run with the status of the given completed step set accordingly. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index 379cc9c4f0a..ec4d138c44f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -65,7 +65,7 @@ public enum Step { deactivateReal(true, deployInitialReal, deployReal, endTests, copyVespaLogs), /** Deactivate the tester. */ - deactivateTester(true, deployTester, endTests), + deactivateTester(true, deployTester, endTests, copyVespaLogs), /** Report completion to the deployment orchestration machinery. */ report(true, installReal, deactivateReal, deactivateTester); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java index bd69ea41b05..788360996ff 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainer.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.archive.CuratorArchiveBucketDb; +import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; @@ -45,47 +46,34 @@ public class ArchiveAccessMaintainer extends ControllerMaintainer { @Override protected double maintain() { - // Count buckets - so we can alert if we get close to the account limit of 1000 - zoneRegistry.zonesIncludingSystem().all().ids().forEach(zoneId -> - metric.set(bucketCountMetricName, archiveBucketDb.buckets(zoneId).size(), - metric.createContext(Map.of("zone", zoneId.value())))); + // Count buckets - so we can alert if we get close to the AWS account limit of 1000 + zoneRegistry.zonesIncludingSystem().all().zones().forEach(z -> + metric.set(bucketCountMetricName, archiveBucketDb.buckets(z.getVirtualId()).size(), + metric.createContext(Map.of( + "zone", z.getVirtualId().value(), + "cloud", z.getCloudName().value())))); zoneRegistry.zonesIncludingSystem().controllerUpgraded().zones().forEach(z -> { - ZoneId zoneId = z.getVirtualId(); - try { - var tenantArchiveAccessRoles = cloudTenantArchiveExternalAccessRoles(); - archiveBucketDb.buckets(zoneId).forEach(archiveBucket -> - archiveService.updateBucketPolicy(zoneId, archiveBucket, - Maps.filterEntries(tenantArchiveAccessRoles, - entry -> archiveBucket.tenants().contains(entry.getKey()))) - ); - Map<String, List<ArchiveBucket>> bucketsPerKey = archiveBucketDb.buckets(zoneId).stream() - .collect(groupingBy(ArchiveBucket::keyArn)); - bucketsPerKey.forEach((keyArn, buckets) -> { - Set<String> authorizedIamRolesForKey = buckets.stream() - .flatMap(b -> b.tenants().stream()) - .filter(tenantArchiveAccessRoles::containsKey) - .map(tenantArchiveAccessRoles::get) - .collect(Collectors.toSet()); - archiveService.updateKeyPolicy(zoneId, keyArn, authorizedIamRolesForKey); - }); - } catch (Exception e) { - throw new RuntimeException("Failed to maintain archive access in " + zoneId.value(), e); - } - } - ); + ZoneId zoneId = z.getVirtualId(); + try { + var tenantArchiveAccessRoles = cloudTenantArchiveExternalAccessRoles(); + var buckets = archiveBucketDb.buckets(zoneId); + archiveService.updatePolicies(zoneId, buckets, tenantArchiveAccessRoles); + } catch (Exception e) { + throw new RuntimeException("Failed to maintain archive access in " + zoneId.value(), e); + } + }); return 1.0; } - private Map<TenantName, String> cloudTenantArchiveExternalAccessRoles() { + private Map<TenantName, ArchiveAccess> cloudTenantArchiveExternalAccessRoles() { List<Tenant> tenants = controller().tenants().asList(); return tenants.stream() .filter(t -> t instanceof CloudTenant) .map(t -> (CloudTenant) t) - .filter(t -> t.archiveAccessRole().isPresent()) .collect(Collectors.toUnmodifiableMap( - Tenant::name, cloudTenant -> cloudTenant.archiveAccessRole().orElseThrow())); + Tenant::name, cloudTenant -> cloudTenant.archiveAccess())); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java index 36ab2e6f384..42821ea8fe2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java @@ -40,15 +40,13 @@ public class ArchiveUriUpdater extends ControllerMaintainer { protected double maintain() { Map<ZoneId, Set<TenantName>> tenantsByZone = new HashMap<>(); - tenantsByZone.put(controller().zoneRegistry().systemZone().getVirtualId(), - new HashSet<>(INFRASTRUCTURE_TENANTS)); + controller().zoneRegistry().zonesIncludingSystem().reachable().zones().forEach( + z -> tenantsByZone.put(z.getVirtualId(), new HashSet<>(INFRASTRUCTURE_TENANTS))); for (var application : applications.asList()) { for (var instance : application.instances().values()) { for (var deployment : instance.deployments().values()) { - tenantsByZone - .computeIfAbsent(deployment.zone(), zone -> new HashSet<>(INFRASTRUCTURE_TENANTS)) - .add(instance.id().tenant()); + tenantsByZone.get(deployment.zone()).add(instance.id().tenant()); } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index 369699eb3a3..b4fac8074aa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -83,7 +83,6 @@ public class JobRunner extends ControllerMaintainer { jobs.abort(run.id(), "job timeout of " + jobTimeout + " reached"); advance(jobs.run(run.id()).get()); }); - else if (run.readySteps().isEmpty()) executors.execute(() -> finish(run.id())); else if (run.hasFailed() || run.sleepUntil().map(sleepUntil -> ! sleepUntil.isAfter(controller().clock().instant())).orElse(true)) @@ -93,7 +92,8 @@ public class JobRunner extends ControllerMaintainer { private void finish(RunId id) { try { jobs.finish(id); - controller().applications().deploymentTrigger().notifyOfCompletion(id.application()); + if ( ! id.type().environment().isManuallyDeployed()) + controller().applications().deploymentTrigger().notifyOfCompletion(id.application()); } catch (TimeoutException e) { // One of the steps are still being run — that's ok, we'll try to finish the run again later. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializer.java index 269864ad641..a4c7c50085c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ArchiveBucketsSerializer.java @@ -13,7 +13,6 @@ import java.util.stream.Collectors; /** * (de)serializes tenant/bucket mappings for a zone - * <p> * * @author andreer */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index dd28978d948..fcc6d99aec2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.SystemName; import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; @@ -214,7 +213,7 @@ class RunSerializer { run.sleepUntil().ifPresent(end -> runObject.setLong(sleepingUntilField, end.toEpochMilli())); runObject.setString(statusField, valueOf(run.status())); runObject.setLong(lastTestRecordField, run.lastTestLogEntry()); - runObject.setLong(lastVespaLogTimestampField, Instant.EPOCH.until(run.lastVespaLogTimestamp(), ChronoUnit.MICROS)); + if (run.lastVespaLogTimestamp().isAfter(Instant.EPOCH)) runObject.setLong(lastVespaLogTimestampField, Instant.EPOCH.until(run.lastVespaLogTimestamp(), ChronoUnit.MICROS)); run.noNodesDownSince().ifPresent(noNodesDownSince -> runObject.setLong(noNodesDownSinceField, noNodesDownSince.toEpochMilli())); run.convergenceSummary().ifPresent(convergenceSummary -> toSlime(convergenceSummary, runObject.setArray(convergenceSummaryField))); run.testerCertificate().ifPresent(certificate -> runObject.setString(testerCertificateField, X509CertificateUtils.toPem(certificate))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index 00e38abcba7..e7cf0c34511 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInf import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; +import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; @@ -77,6 +78,10 @@ public class TenantSerializer { private static final String lastLoginInfoField = "lastLoginInfo"; private static final String secretStoresField = "secretStores"; private static final String archiveAccessRoleField = "archiveAccessRole"; + private static final String archiveAccessField = "archiveAccess"; + private static final String awsArchiveAccessRoleField = "awsArchiveAccessRole"; + private static final String gcpArchiveAccessMemberField = "gcpArchiveAccessMember"; + private static final String awsIdField = "awsId"; private static final String roleField = "role"; @@ -117,7 +122,13 @@ public class TenantSerializer { toSlime(legacyBillingInfo, root.setObject(billingInfoField)); toSlime(tenant.info(), root); toSlime(tenant.tenantSecretStores(), root); - tenant.archiveAccessRole().ifPresent(role -> root.setString(archiveAccessRoleField, role)); + toSlime(tenant.archiveAccess(), root); + } + + private void toSlime(ArchiveAccess archiveAccess, Cursor root) { + Cursor object = root.setObject(archiveAccessField); + archiveAccess.awsRole().ifPresent(role -> object.setString(awsArchiveAccessRoleField, role)); + archiveAccess.gcpMember().ifPresent(member -> object.setString(gcpArchiveAccessMemberField, member)); } private void toSlime(DeletedTenant tenant, Cursor root) { @@ -175,8 +186,8 @@ public class TenantSerializer { BiMap<PublicKey, Principal> developerKeys = developerKeysFromSlime(tenantObject.field(pemDeveloperKeysField)); TenantInfo info = tenantInfoFromSlime(tenantObject.field(tenantInfoField)); List<TenantSecretStore> tenantSecretStores = secretStoresFromSlime(tenantObject.field(secretStoresField)); - Optional<String> archiveAccessRole = SlimeUtils.optionalString(tenantObject.field(archiveAccessRoleField)); - return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccessRole); + ArchiveAccess archiveAccess = archiveAccessFromSlime(tenantObject); + return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores, archiveAccess); } private DeletedTenant deletedTenantFrom(Inspector tenantObject) { @@ -195,6 +206,22 @@ public class TenantSerializer { return keys.build(); } + ArchiveAccess archiveAccessFromSlime(Inspector tenantObject) { + // TODO(enygaard, 2022-05-24): Remove when all tenants have been rewritten to use ArchiveAccess object + Optional<String> archiveAccessRole = SlimeUtils.optionalString(tenantObject.field(archiveAccessRoleField)); + if (archiveAccessRole.isPresent()) { + return new ArchiveAccess().withAWSRole(archiveAccessRole.get()); + } + Inspector object = tenantObject.field(archiveAccessField); + if (!object.valid()) { + return new ArchiveAccess(); + } + Optional<String> awsArchiveAccessRole = SlimeUtils.optionalString(object.field(awsArchiveAccessRoleField)); + Optional<String> gcpArchiveAccessMember = SlimeUtils.optionalString(object.field(gcpArchiveAccessMemberField)); + return new ArchiveAccess() + .withAWSRole(awsArchiveAccessRole) + .withGCPMember(gcpArchiveAccessMember); + } TenantInfo tenantInfoFromSlime(Inspector infoObject) { if (!infoObject.valid()) return TenantInfo.empty(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java index 886dc27b404..c57133d8efd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponse.java @@ -29,7 +29,9 @@ public class ProxyResponse extends HttpResponse { super(statusResponse); this.contentType = contentType; - String configServerPrefix = HttpURL.from(configServer).withPath(Path.empty()).asURI().toString(); + // Configserver always serves from 4443, therefore all responses will have port 4443 in them, + // but the request URI (loadbalancer/VIP) is not always 4443 + String configServerPrefix = HttpURL.from(configServer).withPort(4443).withPath(Path.empty()).asURI().toString(); String controllerRequestPrefix = controllerRequest.getControllerPrefixUri().toString(); bodyResponseRewritten = bodyResponse.replace(configServerPrefix, controllerRequestPrefix); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index d781ea96b47..7af0f962b33 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -35,6 +35,7 @@ import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; import com.yahoo.restapi.ResourceResponse; +import com.yahoo.restapi.RestApiException; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.security.KeyUtils; import com.yahoo.slime.Cursor; @@ -108,6 +109,7 @@ import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus; import com.yahoo.vespa.hosted.controller.security.AccessControlRequests; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.support.access.SupportAccess; +import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; @@ -122,9 +124,6 @@ import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; -import javax.ws.rs.ForbiddenException; -import javax.ws.rs.InternalServerErrorException; -import javax.ws.rs.NotAuthorizedException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -209,10 +208,10 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); } } - catch (ForbiddenException e) { + catch (RestApiException.Forbidden e) { return ErrorResponse.forbidden(Exceptions.toMessageString(e)); } - catch (NotAuthorizedException e) { + catch (RestApiException.Unauthorized e) { return ErrorResponse.unauthorized(Exceptions.toMessageString(e)); } catch (NotExistsException e) { @@ -305,7 +304,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/info/profile")) return putTenantInfo(path.get("tenant"), request, this::putTenantInfoProfile); if (path.matches("/application/v4/tenant/{tenant}/info/billing")) return putTenantInfo(path.get("tenant"), request, this::putTenantInfoBilling); if (path.matches("/application/v4/tenant/{tenant}/info/contacts")) return putTenantInfo(path.get("tenant"), request, this::putTenantInfoContacts); - if (path.matches("/application/v4/tenant/{tenant}/archive-access")) return allowArchiveAccess(path.get("tenant"), request); + if (path.matches("/application/v4/tenant/{tenant}/archive-access")) return allowAwsArchiveAccess(path.get("tenant"), request); // TODO(enygaard, 2022-05-25) Remove when no longer used by console + if (path.matches("/application/v4/tenant/{tenant}/archive-access/aws")) return allowAwsArchiveAccess(path.get("tenant"), request); + if (path.matches("/application/v4/tenant/{tenant}/archive-access/gcp")) return allowGcpArchiveAccess(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/secret-store/{name}")) return addSecretStore(path.get("tenant"), path.get("name"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); @@ -353,7 +354,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}")) return deleteTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/access/managed/operator")) return removeManagedAccess(path.get("tenant")); if (path.matches("/application/v4/tenant/{tenant}/key")) return removeDeveloperKey(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/archive-access")) return removeArchiveAccess(path.get("tenant")); + if (path.matches("/application/v4/tenant/{tenant}/archive-access")) return removeAwsArchiveAccess(path.get("tenant")); // TODO(enygaard, 2022-05-25) Remove when no longer used by console + if (path.matches("/application/v4/tenant/{tenant}/archive-access/aws")) return removeAwsArchiveAccess(path.get("tenant")); + if (path.matches("/application/v4/tenant/{tenant}/archive-access/gcp")) return removeGcpArchiveAccess(path.get("tenant")); if (path.matches("/application/v4/tenant/{tenant}/secret-store/{name}")) return deleteSecretStore(path.get("tenant"), path.get("name"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return deleteApplication(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deployment")) return removeAllProdDeployments(path.get("tenant"), path.get("application")); @@ -1174,7 +1177,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return new SlimeJsonResponse(slime); } - private HttpResponse allowArchiveAccess(String tenantName, HttpRequest request) { + private HttpResponse allowAwsArchiveAccess(String tenantName, HttpRequest request) { if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud) throw new IllegalArgumentException("Tenant '" + tenantName + "' is not a cloud tenant"); @@ -1182,27 +1185,62 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { var role = mandatory("role", data).asString(); if (role.isBlank()) { - return ErrorResponse.badRequest("Archive access role can't be whitespace only"); + return ErrorResponse.badRequest("AWS archive access role can't be whitespace only"); } controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, lockedTenant -> { - lockedTenant = lockedTenant.withArchiveAccessRole(Optional.of(role)); + var access = lockedTenant.get().archiveAccess(); + lockedTenant = lockedTenant.withArchiveAccess(access.withAWSRole(role)); controller.tenants().store(lockedTenant); }); - return new MessageResponse("Archive access role set to '" + role + "' for tenant " + tenantName + "."); + return new MessageResponse("AWS archive access role set to '" + role + "' for tenant " + tenantName + "."); } - private HttpResponse removeArchiveAccess(String tenantName) { + private HttpResponse removeAwsArchiveAccess(String tenantName) { if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud) throw new IllegalArgumentException("Tenant '" + tenantName + "' is not a cloud tenant"); controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, lockedTenant -> { - lockedTenant = lockedTenant.withArchiveAccessRole(Optional.empty()); + var access = lockedTenant.get().archiveAccess(); + lockedTenant = lockedTenant.withArchiveAccess(access.removeAWSRole()); controller.tenants().store(lockedTenant); }); - return new MessageResponse("Archive access role removed for tenant " + tenantName + "."); + return new MessageResponse("AWS archive access role removed for tenant " + tenantName + "."); + } + + private HttpResponse allowGcpArchiveAccess(String tenantName, HttpRequest request) { + if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud) + throw new IllegalArgumentException("Tenant '" + tenantName + "' is not a cloud tenant"); + + var data = toSlime(request.getData()).get(); + var member = mandatory("member", data).asString(); + + if (member.isBlank()) { + return ErrorResponse.badRequest("GCP archive access role can't be whitespace only"); + } + + controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, lockedTenant -> { + var access = lockedTenant.get().archiveAccess(); + lockedTenant = lockedTenant.withArchiveAccess(access.withGCPMember(member)); + controller.tenants().store(lockedTenant); + }); + + return new MessageResponse("GCP archive access member set to '" + member + "' for tenant " + tenantName + "."); + } + + private HttpResponse removeGcpArchiveAccess(String tenantName) { + if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud) + throw new IllegalArgumentException("Tenant '" + tenantName + "' is not a cloud tenant"); + + controller.tenants().lockOrThrow(TenantName.from(tenantName), LockedTenant.Cloud.class, lockedTenant -> { + var access = lockedTenant.get().archiveAccess(); + lockedTenant = lockedTenant.withArchiveAccess(access.removeGCPMember()); + controller.tenants().store(lockedTenant); + }); + + return new MessageResponse("GCP archive access member removed for tenant " + tenantName + "."); } private HttpResponse patchApplication(String tenantName, String applicationName, HttpRequest request) { @@ -2500,7 +2538,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { log.warning(String.format("Failed to get quota for tenant %s: %s", tenant.name(), Exceptions.toMessageString(e))); } - cloudTenant.archiveAccessRole().ifPresent(role -> object.setString("archiveAccessRole", role)); + // TODO(enygaard, 2022-05-25) Remove when console is using new archive access structure + cloudTenant.archiveAccess().awsRole().ifPresent(role -> object.setString("archiveAccessRole", role)); + toSlime(cloudTenant.archiveAccess(), object.setObject("archiveAccess")); break; } @@ -2531,6 +2571,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { tenantMetaDataToSlime(tenant, applications, object.setObject("metaData")); } + private void toSlime(ArchiveAccess archiveAccess, Cursor object) { + archiveAccess.awsRole().ifPresent(role -> object.setString("awsRole", role)); + archiveAccess.gcpMember().ifPresent(member -> object.setString("gcpMember", member)); + } + private void toSlime(Quota quota, QuotaUsage usage, Cursor object) { quota.budget().ifPresentOrElse( budget -> object.setDouble("budget", budget.doubleValue()), @@ -2676,7 +2721,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private static Principal requireUserPrincipal(HttpRequest request) { Principal principal = request.getJDiscRequest().getUserPrincipal(); - if (principal == null) throw new InternalServerErrorException("Expected a user principal"); + if (principal == null) throw new RestApiException.InternalServerError("Expected a user principal"); return principal; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java index c72d8ceb089..a3b77e22f1d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/changemanagement/ChangeManagementApiHandler.java @@ -9,6 +9,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.Path; +import com.yahoo.restapi.RestApiException; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; @@ -24,7 +25,6 @@ import com.yahoo.vespa.hosted.controller.maintenance.ChangeManagementAssessor; import com.yahoo.vespa.hosted.controller.persistence.ChangeRequestSerializer; import com.yahoo.yolean.Exceptions; -import javax.ws.rs.BadRequestException; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -96,13 +96,13 @@ public class ChangeManagementApiHandler extends AuditLoggingRequestHandler { try { return SlimeUtils.jsonToSlime(request.getData().readAllBytes()).get(); } catch (IOException e) { - throw new BadRequestException("Failed to parse request body"); + throw new RestApiException.BadRequest("Failed to parse request body"); } } private static Inspector getInspectorFieldOrThrow(Inspector inspector, String field) { if (!inspector.field(field).valid()) - throw new BadRequestException("Field " + field + " cannot be null"); + throw new RestApiException.BadRequest("Field " + field + " cannot be null"); return inspector.field(field); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java index 346e61c907c..3ca6335dcb7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java @@ -12,6 +12,7 @@ import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; import com.yahoo.restapi.ResourceResponse; +import com.yahoo.restapi.RestApiException; import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.Inspector; import com.yahoo.slime.SlimeUtils; @@ -27,7 +28,6 @@ import com.yahoo.vespa.hosted.controller.support.access.SupportAccess; import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; import com.yahoo.yolean.Exceptions; -import javax.ws.rs.InternalServerErrorException; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; @@ -203,7 +203,7 @@ public class ControllerApiHandler extends AuditLoggingRequestHandler { private static Principal requireUserPrincipal(HttpRequest request) { Principal principal = request.getJDiscRequest().getUserPrincipal(); - if (principal == null) throw new InternalServerErrorException("Expected a user principal"); + if (principal == null) throw new RestApiException.InternalServerError("Expected a user principal"); return principal; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java index 87691d2927a..81e9b1972b9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/security/CloudAccessControl.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.security; import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.TenantName; +import com.yahoo.restapi.RestApiException; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; @@ -22,7 +23,6 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import javax.ws.rs.ForbiddenException; import java.time.Instant; import java.util.List; import java.util.stream.Collectors; @@ -76,7 +76,7 @@ public class CloudAccessControl implements AccessControl { var trialTenants = billingController.tenantsWithPlan(tenantNames, trialPlanId).size(); if (maxTrialTenants.value() >= 0 && maxTrialTenants.value() <= trialTenants) { - throw new ForbiddenException("Too many tenants with trial plans, please contact the Vespa support team"); + throw new RestApiException.Forbidden("Too many tenants with trial plans, please contact the Vespa support team"); } } @@ -84,11 +84,11 @@ public class CloudAccessControl implements AccessControl { if (allowedByPrivilegedRole(auth0Credentials)) return; if (!allowedByFeatureFlag(auth0Credentials)) { - throw new ForbiddenException("You are not currently permitted to create tenants. Please contact the Vespa team to request access."); + throw new RestApiException.Forbidden("You are not currently permitted to create tenants. Please contact the Vespa team to request access."); } if(administeredTenants(auth0Credentials) >= 3) { - throw new ForbiddenException("You are already administering 3 tenants. If you need more, please contact the Vespa team."); + throw new RestApiException.Forbidden("You are already administering 3 tenants. If you need more, please contact the Vespa team."); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDbTest.java index 1a052b6a578..4c3d76b1b17 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDbTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/archive/CuratorArchiveBucketDbTest.java @@ -32,7 +32,7 @@ public class CuratorArchiveBucketDbTest { assertEquals(Optional.of(URI.create("s3://existingBucket/default/")), bucketDb.archiveUriFor(ZoneId.defaultId(), TenantName.defaultName(), true)); // Assigns to existing bucket while there is space - IntStream.range(0, 29).forEach(i -> + IntStream.range(0, 4).forEach(i -> assertEquals( Optional.of(URI.create("s3://existingBucket/tenant" + i + "/")), bucketDb .archiveUriFor(ZoneId.defaultId(), TenantName.from("tenant" + i), true))); @@ -47,7 +47,7 @@ public class CuratorArchiveBucketDbTest { assertEquals(Optional.empty(), bucketDb.archiveUriFor(ZoneId.from("prod.us-east-3"), TenantName.from("newTenant"), false)); // Lists all buckets by zone - Set<TenantName> existingBucketTenants = Streams.concat(Stream.of(TenantName.defaultName()), IntStream.range(0, 29).mapToObj(i -> TenantName.from("tenant" + i))).collect(Collectors.toUnmodifiableSet()); + Set<TenantName> existingBucketTenants = Streams.concat(Stream.of(TenantName.defaultName()), IntStream.range(0, 4).mapToObj(i -> TenantName.from("tenant" + i))).collect(Collectors.toUnmodifiableSet()); assertEquals( Set.of( new ArchiveBucket("existingBucket", "keyArn").withTenants(existingBucketTenants), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index ae4b6259da1..27951d95689 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -445,7 +445,6 @@ public class DeploymentContext { tester.cloud().set(noTests ? Status.NO_TESTS : Status.FAILURE); runner.advance(currentRun(job)); assertTrue(jobs.run(id).get().hasEnded()); - assertEquals(noTests, jobs.run(id).get().hasSucceeded()); assertTrue(configServer().nodeRepository().list(job.type().zone(), NodeFilter.all().applications(TesterId.of(instanceId).id())).isEmpty()); return this; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index beda9bd551d..d602ee8cde3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -2267,4 +2267,16 @@ public class DeploymentTriggerTest { assertEquals(Set.of(), tests.deploymentStatus().jobsToRun().keySet()); } + @Test + public void testNoTests() { + DeploymentContext app = tester.newDeploymentContext(); + app.submit(new ApplicationPackageBuilder().systemTest().region("us-east-3").build()); + + // Declared tests must have run actual tests to succeed. + app.failTests(systemTest, true); + assertFalse(tester.jobs().last(app.instanceId(), systemTest).get().hasSucceeded()); + app.failTests(stagingTest, true); + assertTrue(tester.jobs().last(app.instanceId(), stagingTest).get().hasSucceeded()); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 031cdaa84ae..ae08d1b3a22 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -6,7 +6,6 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; @@ -24,23 +23,14 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud. import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.pkg.TestPackage; -import com.yahoo.vespa.hosted.controller.config.ControllerConfig; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import org.junit.Before; import org.junit.Test; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; -import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -60,7 +50,6 @@ import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; -import static java.nio.charset.StandardCharsets.UTF_8; import static java.time.temporal.ChronoUnit.SECONDS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -455,6 +444,7 @@ public class InternalStepRunnerTest { @Test public void vespaLogIsCopied() { // Tests fail. We should get logs. This fails too, on the first attempt. + tester.controllerTester().computeVersionStatus(); RunId id = app.startSystemTestTests(); tester.cloud().set(TesterCloud.Status.ERROR); tester.configServer().setLogStream(() -> { throw new ConfigServerException(ConfigServerException.ErrorCode.NOT_FOUND, "404", "context"); }); @@ -464,31 +454,43 @@ public class InternalStepRunnerTest { assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.copyVespaLogs)); assertTestLogEntries(id, Step.copyVespaLogs, new LogEntry(lastId + 2, tester.clock().instant(), info, - "Found no logs, but will retry"), - new LogEntry(lastId + 4, tester.clock().instant(), info, "Found no logs, but will retry")); // Config servers now provide the log, and we get it. - tester.configServer().setLogStream(() -> vespaLog); + tester.configServer().setLogStream(() -> vespaLog(tester.clock().instant())); tester.runner().run(); assertEquals(failed, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); assertTestLogEntries(id, Step.copyVespaLogs, new LogEntry(lastId + 2, tester.clock().instant(), info, "Found no logs, but will retry"), - new LogEntry(lastId + 4, tester.clock().instant(), info, - "Found no logs, but will retry"), - new LogEntry(lastId + 5, Instant.EPOCH.plus(3554970337935104L, ChronoUnit.MICROS), info, + new LogEntry(lastId + 3, tester.clock().instant().minusSeconds(4), info, + "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), + new LogEntry(lastId + 4, tester.clock().instant().minusSeconds(4), info, + "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), + /* + new LogEntry(lastId + 5, tester.clock().instant().minusSeconds(4), info, + "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), + new LogEntry(lastId + 6, tester.clock().instant().minusSeconds(4), info, "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), - new LogEntry(lastId + 6, Instant.EPOCH.plus(3554970337947777L, ChronoUnit.MICROS), info, + */ + new LogEntry(lastId + 5, tester.clock().instant().minusSeconds(3), info, "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), - new LogEntry(lastId + 7, Instant.EPOCH.plus(3554970337947820L, ChronoUnit.MICROS), info, + new LogEntry(lastId + 6, tester.clock().instant().minusSeconds(3), warning, + "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstderr\n" + + "java.lang.NullPointerException\n\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\n\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)")); + /* + new LogEntry(lastId + 9, tester.clock().instant().minusSeconds(3), info, "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), - new LogEntry(lastId + 8, Instant.EPOCH.plus(3554970337947845L, ChronoUnit.MICROS), warning, + new LogEntry(lastId + 10, tester.clock().instant().minusSeconds(3), warning, "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstderr\n" + "java.lang.NullPointerException\n\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\n\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)")); + */ } @Test @@ -548,10 +550,12 @@ public class InternalStepRunnerTest { assertEquals(List.of(entries), tester.jobs().details(id).get().get(step)); } - private static final String vespaLog = "-1554970337.084804\t17480180-v6-3.ostk.bm2.prod.ne1.yahoo.com\t5549/832\tcontainer\tContainer.com.yahoo.container.jdisc.ConfiguredApplication\tinfo\tSwitching to the latest deployed set of configurations and components. Application switch number: 2\n" + - "3554970337.935104\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n" + - "3554970337.947777\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n" + - "3554970337.947820\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n" + - "3554970337.947845\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)"; + private static String vespaLog(Instant now) { + return "-1\t17480180-v6-3.ostk.bm2.prod.ne1.yahoo.com\t5549/832\tcontainer\tContainer.com.yahoo.container.jdisc.ConfiguredApplication\tinfo\tSwitching to the latest deployed set of configurations and components. Application switch number: 2\n" + + (now.getEpochSecond() - 4) + "." + now.getNano() / 1000 + "\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n" + + (now.getEpochSecond() - 4) + "." + now.getNano() / 1000 + "\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n" + + (now.getEpochSecond() - 3) + "." + now.getNano() / 1000 + "\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n" + + (now.getEpochSecond() - 3) + "." + now.getNano() / 1000 + "\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)"; + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java index 5571f957e83..3535417c586 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveAccessMaintainerTest.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; import com.yahoo.vespa.hosted.controller.api.integration.archive.MockArchiveService; +import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import org.junit.Test; @@ -33,24 +34,24 @@ public class ArchiveAccessMaintainerTest { String tenant1role = "arn:aws:iam::123456789012:role/my-role"; String tenant2role = "arn:aws:iam::210987654321:role/my-role"; var tenant1 = createTenantWithAccessRole(tester, "tenant1", tenant1role); - createTenantWithAccessRole(tester, "tenant2", tenant2role); + var tenant2 = createTenantWithAccessRole(tester, "tenant2", tenant2role); ZoneId testZone = ZoneId.from("prod.aws-us-east-1c"); tester.controller().archiveBucketDb().archiveUriFor(testZone, tenant1, true); var testBucket = new ArchiveBucket("bucketName", "keyArn").withTenant(tenant1); MockArchiveService archiveService = (MockArchiveService) tester.controller().serviceRegistry().archiveService(); - assertNull(archiveService.authorizedIamRolesForBucket.get(testBucket)); - assertNull(archiveService.authorizedIamRolesForKey.get(testBucket.keyArn())); + + assertEquals(0, archiveService.authorizeAccessByTenantName.size()); MockMetric metric = new MockMetric(); new ArchiveAccessMaintainer(tester.controller(), metric, Duration.ofMinutes(10)).maintain(); - assertEquals(Map.of(tenant1, tenant1role), archiveService.authorizedIamRolesForBucket.get(testBucket)); - assertEquals(Set.of(tenant1role), archiveService.authorizedIamRolesForKey.get(testBucket.keyArn())); + assertEquals(new ArchiveAccess().withAWSRole(tenant1role), archiveService.authorizeAccessByTenantName.get(tenant1)); + assertEquals(new ArchiveAccess().withAWSRole(tenant2role), archiveService.authorizeAccessByTenantName.get(tenant2)); var expected = Map.of("archive.bucketCount", tester.controller().zoneRegistry().zonesIncludingSystem().all().ids().stream() .collect(Collectors.toMap( - zone -> Map.of("zone", zone.value()), + zone -> Map.of("zone", zone.value(), "cloud", "default"), zone -> zone.equals(testZone) ? 1d : 0d))); assertEquals(expected, metric.metrics()); @@ -59,7 +60,7 @@ public class ArchiveAccessMaintainerTest { private TenantName createTenantWithAccessRole(ControllerTester tester, String tenantName, String role) { var tenant = tester.createTenant(tenantName, Tenant.Type.cloud); tester.controller().tenants().lockOrThrow(tenant, LockedTenant.Cloud.class, lockedTenant -> { - lockedTenant = lockedTenant.withArchiveAccessRole(Optional.of(role)); + lockedTenant = lockedTenant.withArchiveAccess(new ArchiveAccess().withAWSRole(role)); tester.controller().tenants().store(lockedTenant); }); return tenant; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java index c1d9c03819d..d208657c1c4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java @@ -40,27 +40,24 @@ public class ArchiveUriUpdaterTest { var application = tester.newDeploymentContext(tenant1.value(), "app1", "instance1"); ZoneId zone = ZoneId.from("prod", "aws-us-east-1c"); - // Initially we should not set any archive URIs as the archive service does not return any + // Initially we should only is the bucket for hosted-vespa tenant updater.maintain(); - assertArchiveUris(Map.of(), zone); - // but the controller zone is always present - assertArchiveUris(Map.of(TenantName.from("hosted-vespa"), "s3://bucketName/hosted-vespa/"), - ZoneId.from("prod", "controller")); + assertArchiveUris(Map.of(TenantName.from("hosted-vespa"), "s3://bucketName/hosted-vespa/"), zone); + assertArchiveUris(Map.of(TenantName.from("hosted-vespa"), "s3://bucketName/hosted-vespa/"), ZoneId.from("prod", "controller")); // Archive service now has URI for tenant1, but tenant1 is not deployed in zone setBucketNameInService(Map.of(tenant1, "uri-1"), zone); - setBucketNameInService(Map.of(tenantInfra, "uri-3"), zone); updater.maintain(); - assertArchiveUris(Map.of(), zone); + assertArchiveUris(Map.of(TenantName.from("hosted-vespa"), "s3://bucketName/hosted-vespa/"), zone); deploy(application, zone); updater.maintain(); - assertArchiveUris(Map.of(tenant1, "s3://uri-1/tenant1/", tenantInfra, "s3://uri-3/hosted-vespa/"), zone); + assertArchiveUris(Map.of(tenant1, "s3://uri-1/tenant1/", tenantInfra, "s3://bucketName/hosted-vespa/"), zone); // URI for tenant1 should be updated and removed for tenant2 setArchiveUriInNodeRepo(Map.of(tenant1, "wrong-uri", tenant2, "uri-2"), zone); updater.maintain(); - assertArchiveUris(Map.of(tenant1, "s3://uri-1/tenant1/", tenantInfra, "s3://uri-3/hosted-vespa/"), zone); + assertArchiveUris(Map.of(tenant1, "s3://uri-1/tenant1/", tenantInfra, "s3://bucketName/hosted-vespa/"), zone); } private void assertArchiveUris(Map<TenantName, String> expectedUris, ZoneId zone) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 5fd1e8347ef..b9ced334e5a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -93,8 +93,7 @@ public class JobRunnerTest { ApplicationId id = appId.defaultInstance(); byte[] testPackageBytes = new byte[0]; jobs.submit(appId, Submission.basic(applicationPackage, testPackageBytes), 2); - - start(jobs, id, systemTest); + start(jobs, id, systemTest); try { start(jobs, id, systemTest); fail("Job is already running, so this should not be allowed!"); @@ -106,12 +105,16 @@ public class JobRunnerTest { assertFalse(jobs.last(id, systemTest).get().hasEnded()); assertTrue(jobs.last(id, stagingTest).get().stepStatuses().values().stream().allMatch(unfinished::equals)); assertFalse(jobs.last(id, stagingTest).get().hasEnded()); - runner.maintain(); + runner.maintain(); phaser.arriveAndAwaitAdvance(); assertTrue(jobs.last(id, systemTest).get().stepStatuses().values().stream().allMatch(succeeded::equals)); - assertTrue(jobs.last(id, stagingTest).get().hasEnded()); assertTrue(jobs.last(id, stagingTest).get().hasFailed()); + + runner.maintain(); + phaser.arriveAndAwaitAdvance(); + assertTrue(jobs.last(id, systemTest).get().hasEnded()); + assertTrue(jobs.last(id, stagingTest).get().hasEnded()); } @Test @@ -165,8 +168,8 @@ public class JobRunnerTest { outcomes.put(endTests, testFailure); runner.maintain(); assertTrue(run.get().hasFailed()); - assertEquals(List.of(copyVespaLogs, deactivateTester), run.get().readySteps()); - assertStepsWithStartTime(run.get(), deployTester, deployReal, installTester, installReal, startTests, endTests, copyVespaLogs, deactivateTester); + assertEquals(List.of(copyVespaLogs), run.get().readySteps()); + assertStepsWithStartTime(run.get(), deployTester, deployReal, installTester, installReal, startTests, endTests, copyVespaLogs); outcomes.put(copyVespaLogs, running); runner.maintain(); @@ -442,8 +445,8 @@ public class JobRunnerTest { @Override public void execute(Runnable command) { phaser.register(); delegate.execute(() -> { - command.run(); - phaser.arriveAndDeregister(); + try { command.run(); } + finally { phaser.arriveAndDeregister(); } }); } }; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java index 75dbebe96ff..5666f8bafd8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; @@ -60,7 +61,7 @@ public class NotificationsDbTest { List.of(TenantContacts.Audience.NOTIFICATIONS), email)))), List.of(), - Optional.empty()); + new ArchiveAccess()); private static final List<Notification> notifications = List.of( notification(1001, Type.deployment, Level.error, NotificationSource.from(tenant), "tenant msg"), notification(1101, Type.applicationPackage, Level.warning, NotificationSource.from(TenantAndApplicationId.from(tenant.value(), "app1")), "app msg"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index fd0ea50e50b..6a01a70eb98 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -25,7 +25,6 @@ import java.time.Instant; import java.util.List; import java.util.Optional; -import static com.yahoo.config.provision.SystemName.main; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; @@ -132,6 +131,7 @@ public class RunSerializerTest { assertEquals(run.end(), phoenix.end()); assertEquals(run.status(), phoenix.status()); assertEquals(run.lastTestLogEntry(), phoenix.lastTestLogEntry()); + assertEquals(run.lastVespaLogTimestamp(), phoenix.lastVespaLogTimestamp()); assertEquals(run.noNodesDownSince(), phoenix.noNodesDownSince()); assertEquals(run.testerCertificate(), phoenix.testerCertificate()); assertEquals(run.versions(), phoenix.versions()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java index e0d14f19f21..a9e633a78d6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java @@ -6,12 +6,14 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.security.KeyUtils; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; +import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.DeletedTenant; @@ -101,7 +103,7 @@ public class TenantSerializerTest { otherPublicKey, new SimplePrincipal("jane")), TenantInfo.empty(), List.of(), - Optional.empty() + new ArchiveAccess() ); CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); assertEquals(tenant.name(), serialized.name()); @@ -123,13 +125,61 @@ public class TenantSerializerTest { new TenantSecretStore("ss1", "123", "role1"), new TenantSecretStore("ss2", "124", "role2") ), - Optional.of("arn:aws:iam::123456789012:role/my-role") + new ArchiveAccess().withAWSRole("arn:aws:iam::123456789012:role/my-role") ); CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); assertEquals(tenant.info(), serialized.info()); assertEquals(tenant.tenantSecretStores(), serialized.tenantSecretStores()); } + @Test + public void cloud_tenant_with_old_archive_access_serialization() { + var json = "{\n" + + " \"name\": \"elderly-lady\",\n" + + " \"type\": \"cloud\",\n" + + " \"createdAt\": 1234,\n" + + " \"lastLoginInfo\": {\n" + + " \"user\": 123,\n" + + " \"developer\": 456\n" + + " },\n" + + " \"creator\": \"foobar-user\",\n" + + " \"pemDeveloperKeys\": [\n" + + " {\n" + + " \"key\": \"-----BEGIN PUBLIC KEY-----\\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuKVFA8dXk43kVfYKzkUqhEY2rDT9\\nz/4jKSTHwbYR8wdsOSrJGVEUPbS2nguIJ64OJH7gFnxM6sxUVj+Nm2HlXw==\\n-----END PUBLIC KEY-----\\n\",\n" + + " \"user\": \"joe\"\n" + + " },\n" + + " {\n" + + " \"key\": \"-----BEGIN PUBLIC KEY-----\\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFELzPyinTfQ/sZnTmRp5E4Ve/sbE\\npDhJeqczkyFcT2PysJ5sZwm7rKPEeXDOhzTPCyRvbUqc2SGdWbKUGGa/Yw==\\n-----END PUBLIC KEY-----\\n\",\n" + + " \"user\": \"jane\"\n" + + " }\n" + + " ],\n" + + " \"billingInfo\": {\n" + + " \"customerId\": \"customer\",\n" + + " \"productCode\": \"Vespa\"\n" + + " },\n" + + " \"archiveAccessRole\": \"arn:aws:iam::123456789012:role/my-role\"\n" + + "}"; + var tenant = (CloudTenant) serializer.tenantFrom(SlimeUtils.jsonToSlime(json)); + assertEquals("arn:aws:iam::123456789012:role/my-role", tenant.archiveAccess().awsRole().get()); + assertFalse(tenant.archiveAccess().gcpMember().isPresent()); + } + + @Test + public void cloud_tenant_with_archive_access() { + CloudTenant tenant = new CloudTenant(TenantName.from("elderly-lady"), + Instant.ofEpochMilli(1234L), + lastLoginInfo(123L, 456L, null), + Optional.of(new SimplePrincipal("foobar-user")), + ImmutableBiMap.of(publicKey, new SimplePrincipal("joe"), + otherPublicKey, new SimplePrincipal("jane")), + TenantInfo.empty(), + List.of(), + new ArchiveAccess().withAWSRole("arn:aws:iam::123456789012:role/my-role").withGCPMember("user:foo@example.com") + ); + CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); + assertEquals(serialized.archiveAccess().awsRole().get(), "arn:aws:iam::123456789012:role/my-role"); + assertEquals(serialized.archiveAccess().gcpMember().get(), "user:foo@example.com"); + } @Test public void cloud_tenant_with_tenant_info_partial() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java index 32fe8ddecff..845d007c154 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ProxyResponseTest.java @@ -24,7 +24,7 @@ public class ProxyResponseTest { Map.of(), null, List.of(URI.create("http://example.com")), Path.parse("configserver")); ProxyResponse proxyResponse = new ProxyResponse( request, - "response link is http://configserver:1234/bla/bla/", + "response link is http://configserver:4443/bla/bla/", 200, URI.create("http://configserver:1234"), "application/json"); @@ -42,7 +42,7 @@ public class ProxyResponseTest { Map.of(), null, List.of(URI.create("http://example.com")), Path.parse("configserver")); ProxyResponse proxyResponse = new ProxyResponse( request, - "response link is http://configserver:1234/bla/bla/", + "response link is http://configserver:4443/bla/bla/", 200, URI.create("http://configserver:1234"), "application/json"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index 065492e47ec..26dd6335ab8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; +import com.yahoo.restapi.RestApiException; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.ControllerTester; @@ -29,7 +30,6 @@ import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import org.junit.Before; import org.junit.Test; -import javax.ws.rs.ForbiddenException; import java.io.File; import java.util.Collections; import java.util.Optional; @@ -267,7 +267,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { try { tester.controller().tenants().create(tenantSpec("tenant2"), credentials("administrator")); fail("Should not be allowed to create tenant that exceed trial limit"); - } catch (ForbiddenException e) { + } catch (RestApiException.Forbidden e) { assertEquals("Too many tenants with trial plans, please contact the Vespa support team", e.getMessage()); } } @@ -361,9 +361,40 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { .data("{\"role\":\"dummy\"}").roles(Role.administrator(tenantName)), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Invalid archive access role 'dummy': Must match expected pattern: 'arn:aws:iam::\\\\d{12}:.+'\"}", 400); + tester.assertResponse(request("/application/v4/tenant/scoober/archive-access/aws", PUT) + .data("{\"role\":\"arn:aws:iam::123456789012:role/my-role\"}").roles(Role.administrator(tenantName)), + "{\"message\":\"AWS archive access role set to 'arn:aws:iam::123456789012:role/my-role' for tenant scoober.\"}", 200); + tester.assertResponse(request("/application/v4/tenant/scoober", GET).roles(Role.reader(tenantName)), + (response) -> assertTrue(response.getBodyAsString().contains("\"archiveAccessRole\":\"arn:aws:iam::123456789012:role/my-role\"")), + 200); + tester.assertResponse(request("/application/v4/tenant/scoober/archive-access/aws", DELETE).roles(Role.administrator(tenantName)), + "{\"message\":\"AWS archive access role removed for tenant scoober.\"}", 200); + tester.assertResponse(request("/application/v4/tenant/scoober", GET).roles(Role.reader(tenantName)), + (response) -> assertFalse(response.getBodyAsString().contains("\"archiveAccessRole\":\"arn:aws:iam::123456789012:role/my-role\"")), + 200); + + tester.assertResponse(request("/application/v4/tenant/scoober/archive-access/gcp", PUT) + .data("{\"member\":\"user:test@example.com\"}").roles(Role.administrator(tenantName)), + "{\"message\":\"GCP archive access member set to 'user:test@example.com' for tenant scoober.\"}", 200); + tester.assertResponse(request("/application/v4/tenant/scoober", GET).roles(Role.reader(tenantName)), + (response) -> assertTrue(response.getBodyAsString().contains("\"gcpMember\":\"user:test@example.com\"")), + 200); + tester.assertResponse(request("/application/v4/tenant/scoober/archive-access/gcp", DELETE).roles(Role.administrator(tenantName)), + "{\"message\":\"GCP archive access member removed for tenant scoober.\"}", 200); + tester.assertResponse(request("/application/v4/tenant/scoober", GET).roles(Role.reader(tenantName)), + (response) -> assertFalse(response.getBodyAsString().contains("\"gcpMember\":\"user:test@example.com\"")), + 200); + + tester.assertResponse(request("/application/v4/tenant/scoober/archive-access", PUT) + .data("{\"role\":\"arn:aws:iam::123456789012:role/my-role\"}").roles(Role.administrator(tenantName)), + "{\"message\":\"AWS archive access role set to 'arn:aws:iam::123456789012:role/my-role' for tenant scoober.\"}", 200); + tester.assertResponse(request("/application/v4/tenant/scoober", GET).roles(Role.reader(tenantName)), + (response) -> assertTrue(response.getBodyAsString().contains("\"archiveAccessRole\":\"arn:aws:iam::123456789012:role/my-role\"")), + 200); + tester.assertResponse(request("/application/v4/tenant/scoober/archive-access", PUT) .data("{\"role\":\"arn:aws:iam::123456789012:role/my-role\"}").roles(Role.administrator(tenantName)), - "{\"message\":\"Archive access role set to 'arn:aws:iam::123456789012:role/my-role' for tenant scoober.\"}", 200); + "{\"message\":\"AWS archive access role set to 'arn:aws:iam::123456789012:role/my-role' for tenant scoober.\"}", 200); tester.assertResponse(request("/application/v4/tenant/scoober", GET).roles(Role.reader(tenantName)), (response) -> assertTrue(response.getBodyAsString().contains("\"archiveAccessRole\":\"arn:aws:iam::123456789012:role/my-role\"")), 200); @@ -373,7 +404,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { new File("deployment-cloud.json")); tester.assertResponse(request("/application/v4/tenant/scoober/archive-access", DELETE).roles(Role.administrator(tenantName)), - "{\"message\":\"Archive access role removed for tenant scoober.\"}", 200); + "{\"message\":\"AWS archive access role removed for tenant scoober.\"}", 200); tester.assertResponse(request("/application/v4/tenant/scoober", GET).roles(Role.reader(tenantName)), (response) -> assertFalse(response.getBodyAsString().contains("archiveAccessRole")), 200); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java index 8db6bdf9a4a..279bd289c00 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java @@ -26,8 +26,7 @@ public class BadgeApiTest extends ControllerContainerTest { public void testBadgeApi() throws IOException { ContainerTester tester = new ContainerTester(container, responseFiles); var application = new DeploymentTester(new ControllerTester(tester)).newDeploymentContext("tenant", "application", "default"); - ApplicationPackage applicationPackage = new ApplicationPackageBuilder().systemTest() - .parallel("us-west-1", "aws-us-east-1a") + ApplicationPackage applicationPackage = new ApplicationPackageBuilder().parallel("us-west-1", "aws-us-east-1a") .test("us-west-1") .region("ap-southeast-1") .test("ap-southeast-1") @@ -59,6 +58,8 @@ public class BadgeApiTest extends ControllerContainerTest { Files.readString(Paths.get(responseFiles + "overview.svg")), 200); tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/production-us-west-1?historyLength=0"), Files.readString(Paths.get(responseFiles + "single-running.svg")), 200); + tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/system-test"), + Files.readString(Paths.get(responseFiles + "running-test.svg")), 200); tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/production-us-west-1?historyLength=32"), Files.readString(Paths.get(responseFiles + "history.svg")), 200); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/overview.svg b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/overview.svg index a0005ed6d76..46e4acaace6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/overview.svg +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/overview.svg @@ -1,4 +1,4 @@ -<svg xmlns='http://www.w3.org/2000/svg' width='763.7809900000001' height='20' role='img' aria-label='Deployment Status'> +<svg xmlns='http://www.w3.org/2000/svg' width='689.25265' height='20' role='img' aria-label='Deployment Status'> <title>Deployment Status</title> <linearGradient id='light' x2='0' y2='100%'> <stop offset='0' stop-color='#fff' stop-opacity='.5'/> @@ -46,32 +46,29 @@ <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> </linearGradient> <clipPath id='rounded'> - <rect width='763.7809900000001' height='20' rx='3' fill='#fff'/> + <rect width='689.25265' height='20' rx='3' fill='#fff'/> </clipPath> <g clip-path='url(#rounded)'> - <rect x='757.7809900000001' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='725.59036' rx='3' width='38.19063' height='20' fill='url(#run-on-success)'/> - <polygon points='635.8470950000001 0 635.8470950000001 20 734.59036 20 742.59036 0' fill='#00f844'/> - <rect x='635.8470950000001' rx='3' width='131.74345499999998' height='20' fill='url(#shade)'/> - <rect x='635.8470950000001' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='603.656465' rx='3' width='38.19063' height='20' fill='#bf103c'/> - <polygon points='486.981225 0 486.981225 20 612.656465 20 620.656465 0' fill='#00f844'/> - <rect x='486.981225' rx='3' width='158.67543' height='20' fill='url(#shade)'/> - <rect x='486.981225' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='348.865175' rx='3' width='144.11604999999997' height='20' fill='url(#run-on-success)'/> - <rect x='358.865175' rx='3' width='134.11604999999997' height='20' fill='url(#shade)'/> - <rect x='358.865175' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='326.674545' rx='3' width='38.19063' height='20' fill='#00f844'/> - <polygon points='237.71563000000003 0 237.71563000000003 20 335.674545 20 343.674545 0' fill='url(#run-on-failure)'/> - <rect x='237.71563000000003' rx='3' width='130.959105' height='20' fill='url(#shade)'/> - <rect x='237.71563000000003' rx='3' width='8' height='20' fill='url(#shadow)'/> - <rect x='153.18729000000002' rx='3' width='90.52834000000001' height='20' fill='url(#run-on-warning)'/> - <rect x='163.18729000000002' rx='3' width='80.52834000000001' height='20' fill='url(#shade)'/> + <rect x='683.25265' rx='3' width='8' height='20' fill='url(#shadow)'/> + <rect x='651.06202' rx='3' width='38.19063' height='20' fill='url(#run-on-success)'/> + <polygon points='561.318755 0 561.318755 20 660.06202 20 668.06202 0' fill='#00f844'/> + <rect x='561.318755' rx='3' width='131.74345499999998' height='20' fill='url(#shade)'/> + <rect x='561.318755' rx='3' width='8' height='20' fill='url(#shadow)'/> + <rect x='529.128125' rx='3' width='38.19063' height='20' fill='#bf103c'/> + <polygon points='412.452885 0 412.452885 20 538.128125 20 546.128125 0' fill='#00f844'/> + <rect x='412.452885' rx='3' width='158.67543' height='20' fill='url(#shade)'/> + <rect x='412.452885' rx='3' width='8' height='20' fill='url(#shadow)'/> + <rect x='274.336835' rx='3' width='144.11604999999997' height='20' fill='url(#run-on-success)'/> + <rect x='284.336835' rx='3' width='134.11604999999997' height='20' fill='url(#shade)'/> + <rect x='284.336835' rx='3' width='8' height='20' fill='url(#shadow)'/> + <rect x='252.146205' rx='3' width='38.19063' height='20' fill='#00f844'/> + <polygon points='163.18729000000002 0 163.18729000000002 20 261.146205 20 269.146205 0' fill='url(#run-on-failure)'/> + <rect x='163.18729000000002' rx='3' width='130.959105' height='20' fill='url(#shade)'/> <rect width='169.18729000000002' height='20' fill='#404040'/> <rect x='-6.0' rx='3' width='175.18729000000002' height='20' fill='url(#shade)'/> <rect width='2' height='20' fill='url(#left-light)'/> - <rect x='761.7809900000001' width='2' height='20' fill='url(#right-shadow)'/> - <rect width='763.7809900000001' height='20' fill='url(#light)'/> + <rect x='687.25265' width='2' height='20' fill='url(#right-shadow)'/> + <rect width='689.25265' height='20' fill='url(#light)'/> </g> <g fill='#fff' text-anchor='middle' font-family='Verdana,Geneva,DejaVu Sans,sans-serif' text-rendering='geometricPrecision' font-size='11'> <svg x='6.5' y='3.0' width='16.0' height='16.0' viewBox='0 0 150 150'> @@ -96,29 +93,27 @@ </svg> <text font-size='11' x='96.09364500000001' y='15' fill='#000' fill-opacity='.4' textLength='135.18729000000002'>tenant.application.default</text> <text font-size='11' x='95.59364500000001' y='14' fill='#fff' textLength='135.18729000000002'>tenant.application.default</text> - <text font-size='11' x='206.95146000000003' y='15' fill='#000' fill-opacity='.4' textLength='62.52834000000001'>system-test</text> - <text font-size='11' x='206.45146000000003' y='14' fill='#fff' textLength='62.52834000000001'>system-test</text> - <text font-size='11' x='276.60659250000003' y='15' fill='#000' fill-opacity='.4' textLength='52.781925'>us-west-1</text> - <text font-size='11' x='276.10659250000003' y='14' fill='#fff' textLength='52.781925'>us-west-1</text> - <text font-size='9' x='323.08605000000006' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text> - <text font-size='9' x='322.58605000000006' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text> - <text font-size='9' x='351.26986000000005' y='15' fill='#000' fill-opacity='.4' textLength='16.190630000000002'>test</text> - <text font-size='9' x='350.76986000000005' y='14' fill='#fff' textLength='16.190630000000002'>test</text> - <text font-size='11' x='412.334705' y='15' fill='#000' fill-opacity='.4' textLength='81.93905999999998'>aws-us-east-1a</text> - <text font-size='11' x='411.834705' y='14' fill='#fff' textLength='81.93905999999998'>aws-us-east-1a</text> - <text font-size='9' x='473.39273000000003' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text> - <text font-size='9' x='472.89273000000003' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text> - <text font-size='11' x='539.73035' y='15' fill='#000' fill-opacity='.4' textLength='80.49825'>ap-southeast-1</text> - <text font-size='11' x='539.23035' y='14' fill='#fff' textLength='80.49825'>ap-southeast-1</text> - <text font-size='9' x='600.06797' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text> - <text font-size='9' x='599.56797' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text> - <text font-size='9' x='628.25178' y='15' fill='#000' fill-opacity='.4' textLength='16.190630000000002'>test</text> - <text font-size='9' x='627.75178' y='14' fill='#fff' textLength='16.190630000000002'>test</text> - <text font-size='11' x='675.1302325' y='15' fill='#000' fill-opacity='.4' textLength='53.566275'>eu-west-1</text> - <text font-size='11' x='674.6302325' y='14' fill='#fff' textLength='53.566275'>eu-west-1</text> - <text font-size='9' x='722.0018650000001' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text> - <text font-size='9' x='721.5018650000001' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text> - <text font-size='9' x='750.1856750000001' y='15' fill='#000' fill-opacity='.4' textLength='16.190630000000002'>test</text> - <text font-size='9' x='749.6856750000001' y='14' fill='#fff' textLength='16.190630000000002'>test</text> + <text font-size='11' x='202.07825250000002' y='15' fill='#000' fill-opacity='.4' textLength='52.781925'>us-west-1</text> + <text font-size='11' x='201.57825250000002' y='14' fill='#fff' textLength='52.781925'>us-west-1</text> + <text font-size='9' x='248.55771000000001' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text> + <text font-size='9' x='248.05771000000001' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text> + <text font-size='9' x='276.74152000000004' y='15' fill='#000' fill-opacity='.4' textLength='16.190630000000002'>test</text> + <text font-size='9' x='276.24152000000004' y='14' fill='#fff' textLength='16.190630000000002'>test</text> + <text font-size='11' x='337.806365' y='15' fill='#000' fill-opacity='.4' textLength='81.93905999999998'>aws-us-east-1a</text> + <text font-size='11' x='337.306365' y='14' fill='#fff' textLength='81.93905999999998'>aws-us-east-1a</text> + <text font-size='9' x='398.86439' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text> + <text font-size='9' x='398.36439' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text> + <text font-size='11' x='465.20201' y='15' fill='#000' fill-opacity='.4' textLength='80.49825'>ap-southeast-1</text> + <text font-size='11' x='464.70201' y='14' fill='#fff' textLength='80.49825'>ap-southeast-1</text> + <text font-size='9' x='525.53963' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text> + <text font-size='9' x='525.03963' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text> + <text font-size='9' x='553.72344' y='15' fill='#000' fill-opacity='.4' textLength='16.190630000000002'>test</text> + <text font-size='9' x='553.22344' y='14' fill='#fff' textLength='16.190630000000002'>test</text> + <text font-size='11' x='600.6018925' y='15' fill='#000' fill-opacity='.4' textLength='53.566275'>eu-west-1</text> + <text font-size='11' x='600.1018925' y='14' fill='#fff' textLength='53.566275'>eu-west-1</text> + <text font-size='9' x='647.473525' y='15' fill='#000' fill-opacity='.4' textLength='28.176989999999996'>deploy</text> + <text font-size='9' x='646.973525' y='14' fill='#fff' textLength='28.176989999999996'>deploy</text> + <text font-size='9' x='675.657335' y='15' fill='#000' fill-opacity='.4' textLength='16.190630000000002'>test</text> + <text font-size='9' x='675.157335' y='14' fill='#fff' textLength='16.190630000000002'>test</text> </g> </svg> diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/running-test.svg b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/running-test.svg new file mode 100644 index 00000000000..9463c01e8ad --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/running-test.svg @@ -0,0 +1,93 @@ +<svg xmlns='http://www.w3.org/2000/svg' width='300.38627894289516' height='20' role='img' aria-label='Deployment Status'> + <title>Deployment Status</title> + <linearGradient id='light' x2='0' y2='100%'> + <stop offset='0' stop-color='#fff' stop-opacity='.5'/> + <stop offset='.1' stop-color='#fff' stop-opacity='.15'/> + <stop offset='.9' stop-color='#000' stop-opacity='.15'/> + <stop offset='1' stop-color='#000' stop-opacity='.5'/> + </linearGradient> + <linearGradient id='left-light' x2='100%' y2='0'> + <stop offset='0' stop-color='#fff' stop-opacity='.3'/> + <stop offset='.5' stop-color='#fff' stop-opacity='.1'/> + <stop offset='1' stop-color='#fff' stop-opacity='.0'/> + </linearGradient> + <linearGradient id='right-shadow' x2='100%' y2='0'> + <stop offset='0' stop-color='#000' stop-opacity='.0'/> + <stop offset='.5' stop-color='#000' stop-opacity='.1'/> + <stop offset='1' stop-color='#000' stop-opacity='.3'/> + </linearGradient> + <linearGradient id='shadow' x2='100%' y2='0'> + <stop offset='0' stop-color='#222' stop-opacity='.3'/> + <stop offset='.625' stop-color='#555' stop-opacity='.3'/> + <stop offset='.9' stop-color='#555' stop-opacity='.05'/> + <stop offset='1' stop-color='#555' stop-opacity='.0'/> + </linearGradient> + <linearGradient id='shade' x2='100%' y2='0'> + <stop offset='0' stop-color='#000' stop-opacity='.20'/> + <stop offset='0.05' stop-color='#000' stop-opacity='.10'/> + <stop offset='1' stop-color='#000' stop-opacity='.0'/> + </linearGradient> + <linearGradient id='run-on-failure' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#bf103c' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> + <linearGradient id='run-on-warning' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#bd890b' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> + <linearGradient id='run-on-success' x1='40%' x2='80%' y2='0%'> + <stop offset='0' stop-color='#ab83ff' /> + <stop offset='1' stop-color='#00f844' /> + <animate attributeName='x1' values='-110%;150%;20%;-110%' dur='6s' repeatCount='indefinite' /> + <animate attributeName='x2' values='-10%;250%;120%;-10%' dur='6s' repeatCount='indefinite' /> + </linearGradient> + <clipPath id='rounded'> + <rect width='300.38627894289516' height='20' rx='3' fill='#fff'/> + </clipPath> + <g clip-path='url(#rounded)'> + <rect x='297.7936599677133' rx='3' width='8' height='20' fill='url(#shadow)'/> + <rect x='267.7546449838567' rx='3' width='36.03901498385664' height='20' fill='#00f844'/> + <rect x='267.7546449838567' rx='3' width='36.03901498385664' height='20' fill='url(#shade)'/> + <rect x='271.5979829411765' rx='3' width='8' height='20' fill='url(#shadow)'/> + <rect x='237.71563000000003' rx='3' width='39.88235294117647' height='20' fill='#00f844'/> + <rect x='237.71563000000003' rx='3' width='39.88235294117647' height='20' fill='url(#shade)'/> + <rect x='237.71563000000003' rx='3' width='8' height='20' fill='url(#shadow)'/> + <rect x='163.18729000000002' rx='3' width='80.52834000000001' height='20' fill='url(#run-on-warning)'/> + <rect x='163.18729000000002' rx='3' width='80.52834000000001' height='20' fill='url(#shade)'/> + <rect width='169.18729000000002' height='20' fill='#404040'/> + <rect x='-6.0' rx='3' width='175.18729000000002' height='20' fill='url(#shade)'/> + <rect width='2' height='20' fill='url(#left-light)'/> + <rect x='298.38627894289516' width='2' height='20' fill='url(#right-shadow)'/> + <rect width='300.38627894289516' height='20' fill='url(#light)'/> + </g> + <g fill='#fff' text-anchor='middle' font-family='Verdana,Geneva,DejaVu Sans,sans-serif' text-rendering='geometricPrecision' font-size='11'> + <svg x='6.5' y='3.0' width='16.0' height='16.0' viewBox='0 0 150 150'> + <polygon fill='#402a14' fill-opacity='0.5' points='84.84 10 34.1 44.46 34.1 103.78 84.84 68.02 135.57 103.78 135.57 44.46 84.84 10'/> + <polygon fill='#402a14' fill-opacity='0.5' points='84.84 68.02 84.84 10 135.57 44.46 135.57 103.78 84.84 68.02'/> + <polygon fill='#061a29' fill-opacity='0.5' points='65.07 81.99 14.34 46.22 14.34 105.54 65.07 140 115.81 105.54 115.81 46.22 65.07 81.99'/> + <polygon fill='#061a29' fill-opacity='0.5' points='65.07 81.99 65.07 140 14.34 105.54 14.34 46.22 65.07 81.99'/> + </svg> + <svg x='6.0' y='2.0' width='16.0' height='16.0' viewBox='0 0 150 150'> + <linearGradient id='yellow-shaded' x1='91.17' y1='44.83' x2='136.24' y2='73.4' gradientUnits='userSpaceOnUse'> + <stop offset='0.01' stop-color='#c6783e'/> + <stop offset='0.54' stop-color='#ff9750'/> + </linearGradient> + <linearGradient id='blue-shaded' x1='60.71' y1='104.56' x2='-15.54' y2='63' gradientUnits='userSpaceOnUse'> + <stop offset='0' stop-color='#005a8e'/> + <stop offset='0.54' stop-color='#1a7db6'/> + </linearGradient> + <polygon fill='#ff9d4b' points='84.84 10 34.1 44.46 34.1 103.78 84.84 68.02 135.57 103.78 135.57 44.46 84.84 10'/> + <polygon fill='url(#yellow-shaded)' points='84.84 68.02 84.84 10 135.57 44.46 135.57 103.78 84.84 68.02'/> + <polygon fill='#1a7db6' points='65.07 81.99 14.34 46.22 14.34 105.54 65.07 140 115.81 105.54 115.81 46.22 65.07 81.99'/> + <polygon fill='url(#blue-shaded)' points='65.07 81.99 65.07 140 14.34 105.54 14.34 46.22 65.07 81.99'/> + </svg> + <text font-size='11' x='96.09364500000001' y='15' fill='#000' fill-opacity='.4' textLength='135.18729000000002'>tenant.application.default</text> + <text font-size='11' x='95.59364500000001' y='14' fill='#fff' textLength='135.18729000000002'>tenant.application.default</text> + <text font-size='11' x='206.95146000000003' y='15' fill='#000' fill-opacity='.4' textLength='62.52834000000001'>system-test</text> + <text font-size='11' x='206.45146000000003' y='14' fill='#fff' textLength='62.52834000000001'>system-test</text> + </g> +</svg> diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java index 15c7dbf73ab..9024d7c8e7e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.restapi.ApplicationRequestToDiscFilterRequestWrapper; +import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; @@ -76,7 +77,7 @@ public class SignatureFilterTest { ImmutableBiMap.of(), TenantInfo.empty(), List.of(), - Optional.empty())); + new ArchiveAccess())); tester.curator().writeApplication(new Application(appId, tester.clock().instant())); } @@ -122,7 +123,7 @@ public class SignatureFilterTest { ImmutableBiMap.of(publicKey, () -> "user"), TenantInfo.empty(), List.of(), - Optional.empty())); + new ArchiveAccess())); verifySecurityContext(requestOf(signer.signed(request.copy(), Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes), new SecurityContext(new SimplePrincipal("user"), Set.of(Role.reader(id.tenant()), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json index 3237e99783d..f980f9231f3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-keys.json @@ -24,6 +24,7 @@ "budgetUsed": 0.0, "clusterSize": 5 }, + "archiveAccess": { }, "applications": [ { "tenant": "my-tenant", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json index 0cc8ba2cd9e..1152033791b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-with-secrets.json @@ -32,6 +32,7 @@ "budgetUsed": 0.0, "clusterSize": 5 }, + "archiveAccess": { }, "applications": [ { "tenant": "my-tenant", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json index 3153c6e218a..631346181a1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/tenant-without-applications.json @@ -15,6 +15,7 @@ "budgetUsed": 0.0, "clusterSize": 5 }, + "archiveAccess": { }, "applications": [ ], "metaData": { "createdAtMillis": 1600000000000 diff --git a/default_build_settings.cmake b/default_build_settings.cmake index 0b8cf873a27..87345f58002 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -56,6 +56,12 @@ function(setup_vespa_default_build_settings_almalinux_8_6) set(DEFAULT_VESPA_LLVM_VERSION "13" PARENT_SCOPE) endfunction() +function(setup_vespa_default_build_settings_almalinux_9_0) + message("-- Setting up default build settings for almalinux 9.0") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "13" PARENT_SCOPE) +endfunction() + function(setup_vespa_default_build_settings_darwin) message("-- Setting up default build settings for darwin") set(DEFAULT_VESPA_LLVM_VERSION "13" PARENT_SCOPE) @@ -210,6 +216,8 @@ function(vespa_use_default_build_settings) setup_vespa_default_build_settings_rocky_8_6() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "almalinux 8.6") setup_vespa_default_build_settings_almalinux_8_6() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "almalinux 9.0") + setup_vespa_default_build_settings_almalinux_9_0() elseif(VESPA_OS_DISTRO STREQUAL "darwin") setup_vespa_default_build_settings_darwin() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 34") diff --git a/eval/src/tests/eval/compile_cache/compile_cache_test.cpp b/eval/src/tests/eval/compile_cache/compile_cache_test.cpp index c9d56de897c..2688dc7b4f5 100644 --- a/eval/src/tests/eval/compile_cache/compile_cache_test.cpp +++ b/eval/src/tests/eval/compile_cache/compile_cache_test.cpp @@ -270,9 +270,9 @@ struct CompileCheck : test::EvalSpec::EvalTest { for (const Entry &entry: list) { auto fun = entry.fun->get().get_function(); if (std::isnan(entry.expect)) { - EXPECT_TRUE(std::isnan(fun(&entry.params[0]))); + EXPECT_TRUE(std::isnan(fun(entry.params.data()))); } else { - EXPECT_EQUAL(fun(&entry.params[0]), entry.expect); + EXPECT_EQUAL(fun(entry.params.data()), entry.expect); } } } diff --git a/eval/src/tests/eval/compiled_function/compiled_function_test.cpp b/eval/src/tests/eval/compiled_function/compiled_function_test.cpp index 45e0c6d08fc..1d612826ab2 100644 --- a/eval/src/tests/eval/compiled_function/compiled_function_test.cpp +++ b/eval/src/tests/eval/compiled_function/compiled_function_test.cpp @@ -108,7 +108,7 @@ struct MyEvalTest : test::EvalSpec::EvalTest { CompiledFunction cfun(*function, PassParams::ARRAY); auto fun = cfun.get_function(); ASSERT_EQUAL(cfun.num_params(), param_values.size()); - double result = fun(¶m_values[0]); + double result = fun(param_values.data()); if (is_same(expected_result, result)) { print_pass && fprintf(stderr, "verifying: %s -> %g ... PASS\n", as_string(param_names, param_values, expression).c_str(), diff --git a/eval/src/tests/eval/gbdt/gbdt_test.cpp b/eval/src/tests/eval/gbdt/gbdt_test.cpp index 09d4a081ac4..582bb484f0e 100644 --- a/eval/src/tests/eval/gbdt/gbdt_test.cpp +++ b/eval/src/tests/eval/gbdt/gbdt_test.cpp @@ -188,7 +188,7 @@ struct DummyForest2 : public Forest { size_t num_trees; explicit DummyForest2(size_t num_trees_in) : num_trees(num_trees_in) {} static double eval(const Forest *forest, const double *) { - const DummyForest1 &self = *((const DummyForest1 *)forest); + const DummyForest2 &self = *((const DummyForest2 *)forest); return double(self.num_trees); } static Optimize::Result optimize(const ForestStats &stats, diff --git a/eval/src/vespa/eval/eval/array_array_map.h b/eval/src/vespa/eval/eval/array_array_map.h index e3601644f3e..4deb5e17a9f 100644 --- a/eval/src/vespa/eval/eval/array_array_map.h +++ b/eval/src/vespa/eval/eval/array_array_map.h @@ -42,7 +42,7 @@ public: bool valid() const { return (id != npos()); } }; - ConstArrayRef<K> get_keys(Tag tag) const { return {&_keys[tag.id * _keys_per_entry], _keys_per_entry}; } + ConstArrayRef<K> get_keys(Tag tag) const { return {_keys.data() + (tag.id * _keys_per_entry), _keys_per_entry}; } ArrayRef<V> get_values(Tag tag) { return {&_values[tag.id * _values_per_entry], _values_per_entry}; } ConstArrayRef<V> get_values(Tag tag) const { return {&_values[tag.id * _values_per_entry], _values_per_entry}; } diff --git a/eval/src/vespa/eval/eval/fast_addr_map.cpp b/eval/src/vespa/eval/eval/fast_addr_map.cpp index 97c2de64ed9..08c3cdaf870 100644 --- a/eval/src/vespa/eval/eval/fast_addr_map.cpp +++ b/eval/src/vespa/eval/eval/fast_addr_map.cpp @@ -5,6 +5,10 @@ namespace vespalib::eval { +FastAddrMap::FastAddrMap(size_t num_mapped_dims, const StringIdVector &labels_in, size_t expected_subspaces) + : _labels(num_mapped_dims, labels_in), + _map(expected_subspaces * 2, Hash(), Equal(_labels)) +{} FastAddrMap::~FastAddrMap() = default; } diff --git a/eval/src/vespa/eval/eval/fast_addr_map.h b/eval/src/vespa/eval/eval/fast_addr_map.h index b9bc39ad619..bda708b0945 100644 --- a/eval/src/vespa/eval/eval/fast_addr_map.h +++ b/eval/src/vespa/eval/eval/fast_addr_map.h @@ -65,7 +65,7 @@ public: LabelView(size_t num_mapped_dims, const StringIdVector &labels_in) : addr_size(num_mapped_dims), labels(labels_in) {} ConstArrayRef<string_id> get_addr(size_t idx) const { - return {&labels[idx * addr_size], addr_size}; + return {labels.data() + (idx * addr_size), addr_size}; } }; @@ -104,9 +104,7 @@ private: HashType _map; public: - FastAddrMap(size_t num_mapped_dims, const StringIdVector &labels_in, size_t expected_subspaces) - : _labels(num_mapped_dims, labels_in), - _map(expected_subspaces * 2, Hash(), Equal(_labels)) {} + FastAddrMap(size_t num_mapped_dims, const StringIdVector &labels_in, size_t expected_subspaces); ~FastAddrMap(); FastAddrMap(const FastAddrMap &) = delete; FastAddrMap &operator=(const FastAddrMap &) = delete; diff --git a/eval/src/vespa/eval/eval/fast_value.hpp b/eval/src/vespa/eval/eval/fast_value.hpp index a40314430c7..42bf459ae3e 100644 --- a/eval/src/vespa/eval/eval/fast_value.hpp +++ b/eval/src/vespa/eval/eval/fast_value.hpp @@ -214,7 +214,9 @@ void FastCells<T>::reallocate(size_t need) { capacity = roundUp2inN(size + need); alloc::Alloc new_memory = alloc::Alloc::alloc(elem_size * capacity); - memcpy(new_memory.get(), memory.get(), elem_size * size); + if (memory.get()) { + memcpy(new_memory.get(), memory.get(), elem_size * size); + } memory = std::move(new_memory); } diff --git a/eval/src/vespa/eval/eval/vm_forest.cpp b/eval/src/vespa/eval/eval/vm_forest.cpp index 9b9e6ee1124..9a6f4730a5e 100644 --- a/eval/src/vespa/eval/eval/vm_forest.cpp +++ b/eval/src/vespa/eval/eval/vm_forest.cpp @@ -36,17 +36,19 @@ constexpr uint32_t INVERTED = 3; // Note: We need to use double for set membership checks (IN) due to // string hashing. -const double *as_double_ptr(const uint32_t *pos) { - return reinterpret_cast<const double*>(pos); +double read_double(const uint32_t *pos) { + double value; + memcpy(&value, pos, sizeof(value)); + return value; } const float *as_float_ptr(const uint32_t *pos) { return reinterpret_cast<const float*>(pos); } -bool find_in(double value, const double *set, const double *end) { - for (; set < end; ++set) { - if (value == *set) { +bool find_in(double value, const uint32_t *set, const uint32_t *end) { + for (; set < end; set += 2) { + if (value == read_double(set)) { return true; } } @@ -82,8 +84,8 @@ double general_find_leaf(const double *input, const uint32_t *pos, uint32_t node return *as_float_ptr(pos); } } else if (node_type == IN) { - if (find_in(input[pos[0] >> 12], as_double_ptr(pos + 2), - as_double_ptr(pos + 2 + (2 * (pos[1] & 0xff))))) + if (find_in(input[pos[0] >> 12], pos + 2, + pos + 2 + (2 * (pos[1] & 0xff)))) { node_type = (pos[0] & 0xf0) >> 4; pos += 2 + (2 * (pos[1] & 0xff)); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index 748d9eb1003..81a8944149c 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -9,7 +9,6 @@ import com.yahoo.jrt.Request; import com.yahoo.jrt.Supervisor; import net.jpountz.xxhash.StreamingXXHash64; import net.jpountz.xxhash.XXHashFactory; - import java.io.File; import java.io.IOException; import java.nio.file.FileAlreadyExistsException; @@ -23,6 +22,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; +import static com.yahoo.vespa.filedistribution.FileReferenceData.Type; + /** * When asking for a file reference, this handles RPC callbacks from config server with file data and metadata. * Uses the same Supervisor as the original caller that requests files, so communication uses the same @@ -105,18 +106,12 @@ public class FileReceiver { } File close(long hash) { - if (hasher.getValue() != hash) - throw new RuntimeException("xxhash from content (" + currentHash + ") is not equal to xxhash in request (" + hash + ")"); + verifyHash(hash); File file = new File(fileReferenceDir, fileName); File decompressedDir = null; try { - // Unpack if necessary - if (fileType == FileReferenceData.Type.compressed) { - decompressedDir = Files.createTempDirectory(tmpDir.toPath(), "archive").toFile(); - CompressedFileReference.decompress(inprogressFile, decompressedDir); - moveFileToDestination(decompressedDir, fileReferenceDir); - } else { + if (fileType == Type.file) { try { Files.createDirectories(fileReferenceDir.toPath()); } catch (IOException e) { @@ -125,6 +120,10 @@ public class FileReceiver { } log.log(Level.FINE, () -> "Uncompressed file, moving to " + file.getAbsolutePath()); moveFileToDestination(inprogressFile, file); + } else { + decompressedDir = Files.createTempDirectory(tmpDir.toPath(), "archive").toFile(); + new FileReferenceCompressor(fileType).decompress(inprogressFile, decompressedDir); + moveFileToDestination(decompressedDir, fileReferenceDir); } } catch (IOException e) { log.log(Level.SEVERE, "Failed writing file: " + e.getMessage(), e); @@ -139,6 +138,12 @@ public class FileReceiver { double percentageReceived() { return (double)currentFileSize/(double)fileSize; } + + void verifyHash(long hash) { + if (hasher.getValue() != hash) + throw new RuntimeException("xxhash from content (" + currentHash + ") is not equal to xxhash in request (" + hash + ")"); + } + } FileReceiver(Supervisor supervisor, Downloads downloads, File downloadDirectory) { @@ -291,4 +296,5 @@ public class FileReceiver { } return 0; } + } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java index cbe810fe9b6..c36bcd22606 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java @@ -7,16 +7,17 @@ import org.apache.commons.compress.archivers.ArchiveInputStream; import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; - -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; +import java.util.Objects; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -28,36 +29,36 @@ import java.util.zip.GZIPOutputStream; * * @author hmusum */ -public class CompressedFileReference { +public class FileReferenceCompressor { - private static final Logger log = Logger.getLogger(CompressedFileReference.class.getName()); + private static final Logger log = Logger.getLogger(FileReferenceCompressor.class.getName()); private static final int recurseDepth = 100; - public static File compress(File baseDir, List<File> inputFiles, File outputFile) throws IOException { - TarArchiveOutputStream archiveOutputStream = new TarArchiveOutputStream(new GZIPOutputStream(new FileOutputStream(outputFile))); - archiveOutputStream.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); - createArchiveFile(archiveOutputStream, baseDir, inputFiles); - return outputFile; - } + private final FileReferenceData.Type type; - public static File compress(File directory, File outputFile) throws IOException { - return compress(directory, Files.find(Paths.get(directory.getAbsolutePath()), - recurseDepth, - (p, basicFileAttributes) -> basicFileAttributes.isRegularFile()) - .map(Path::toFile).collect(Collectors.toList()), outputFile); + public FileReferenceCompressor(FileReferenceData.Type type) { + this.type = Objects.requireNonNull(type, "Type cannot be null"); } - public static byte[] compress(File baseDir, List<File> inputFiles) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - TarArchiveOutputStream archiveOutputStream = new TarArchiveOutputStream(new GZIPOutputStream(out)); + public File compress(File baseDir, List<File> inputFiles, File outputFile) throws IOException { + TarArchiveOutputStream archiveOutputStream = new TarArchiveOutputStream(compressedOutputStream(outputFile)); archiveOutputStream.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); createArchiveFile(archiveOutputStream, baseDir, inputFiles); - return out.toByteArray(); + return outputFile; + } + + public File compress(File directory, File outputFile) throws IOException { + return compress(directory, + Files.find(Paths.get(directory.getAbsolutePath()), + recurseDepth, + (p, basicFileAttributes) -> basicFileAttributes.isRegularFile()) + .map(Path::toFile).collect(Collectors.toList()), + outputFile); } - static void decompress(File inputFile, File outputDir) throws IOException { + public void decompress(File inputFile, File outputDir) throws IOException { log.log(Level.FINE, () -> "Decompressing '" + inputFile + "' into '" + outputDir + "'"); - try (ArchiveInputStream ais = new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(inputFile)))) { + try (ArchiveInputStream ais = new TarArchiveInputStream(decompressedInputStream(inputFile))) { decompress(ais, outputDir); } catch (IllegalArgumentException e) { throw new RuntimeException("Unable to decompress '" + inputFile.getAbsolutePath() + "': " + e.getMessage()); @@ -111,5 +112,28 @@ public class CompressedFileReference { ByteStreams.copy(new FileInputStream(file), taos); taos.closeArchiveEntry(); } + + private OutputStream compressedOutputStream(File outputFile) throws IOException { + switch (type) { + case compressed: + return new GZIPOutputStream(new FileOutputStream(outputFile)); + case file: + return new FileOutputStream(outputFile); + default: + throw new RuntimeException("Unknown file reference type " + type); + } + } + + private InputStream decompressedInputStream(File inputFile) throws IOException { + switch (type) { + case compressed: + return new GZIPInputStream(new FileInputStream(inputFile)); + case file: + return new FileInputStream(inputFile); + default: + throw new RuntimeException("Unknown file reference type " + type); + } + } + } diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java index e8bd63fc083..637fbbbd0a7 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java @@ -32,6 +32,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import static com.yahoo.jrt.ErrorCode.CONNECTION; +import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.compressed; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -147,9 +148,9 @@ public class FileDownloaderTest { File barFile = new File(subdir, "really-long-filename-over-100-bytes-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); IOUtils.writeFile(barFile, "bar", false); - File tarFile = CompressedFileReference.compress(tempPath.toFile(), Arrays.asList(fooFile, barFile), new File(tempPath.toFile(), filename)); + File tarFile = new FileReferenceCompressor(compressed).compress(tempPath.toFile(), Arrays.asList(fooFile, barFile), new File(tempPath.toFile(), filename)); byte[] tarredContent = IOUtils.readFileBytes(tarFile); - receiveFile(fileReference, filename, FileReferenceData.Type.compressed, tarredContent); + receiveFile(fileReference, filename, compressed, tarredContent); Optional<File> downloadedFile = getFile(fileReference); assertTrue(downloadedFile.isPresent()); diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java index 996f3cc2984..5c15f945ae3 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReceiverTest.java @@ -10,15 +10,15 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - -import static org.junit.Assert.assertEquals; - import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Files; +import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.compressed; +import static org.junit.Assert.assertEquals; + public class FileReceiverTest { private File root; private final XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); @@ -58,7 +58,7 @@ public class FileReceiverTest { writerB.close(); File tempFile = temporaryFolder.newFile(); - File file = CompressedFileReference.compress(dirWithFiles, tempFile); + File file = new FileReferenceCompressor(compressed).compress(dirWithFiles, tempFile); transferCompressedData(new FileReference("ref"), "a", IOUtils.readFileBytes(file)); File downloadDir = new File(root, "ref"); assertEquals("1", IOUtils.readFile(new File(downloadDir, "a"))); @@ -88,8 +88,7 @@ public class FileReceiverTest { } private void transferCompressedData(FileReference ref, String fileName, byte[] data) { - FileReceiver.Session session = - new FileReceiver.Session(root, 1, ref, FileReferenceData.Type.compressed, fileName, data.length); + FileReceiver.Session session = new FileReceiver.Session(root, 1, ref, compressed, fileName, data.length); session.addPart(0, data); session.close(hasher.hash(ByteBuffer.wrap(data), 0)); } diff --git a/flags/pom.xml b/flags/pom.xml index 5f0d3e1d481..10e0249ce80 100644 --- a/flags/pom.xml +++ b/flags/pom.xml @@ -55,11 +55,6 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>com.google.code.findbugs</groupId> - <artifactId>jsr305</artifactId> - <scope>provided</scope> - </dependency> - <dependency> <groupId>com.google.inject</groupId> <artifactId>guice</artifactId> <classifier>no_aop</classifier> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/BooleanFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/BooleanFlag.java index fb0335408cb..7dc1c66fd11 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/BooleanFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/BooleanFlag.java @@ -1,12 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags; -import javax.annotation.concurrent.Immutable; - /** * @author hakonhall */ -@Immutable public class BooleanFlag extends FlagImpl<Boolean, BooleanFlag> { public BooleanFlag(FlagId id, boolean defaultValue, FetchVector vector, FlagSerializer<Boolean> serializer, FlagSource source) { super(id, defaultValue, vector, serializer, source, BooleanFlag::new); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/DoubleFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/DoubleFlag.java index d8cddbff432..46db839905f 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/DoubleFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/DoubleFlag.java @@ -1,12 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags; -import javax.annotation.concurrent.Immutable; - /** * @author freva */ -@Immutable public class DoubleFlag extends FlagImpl<Double, DoubleFlag> { public DoubleFlag(FlagId id, Double defaultValue, FetchVector vector, FlagSerializer<Double> serializer, FlagSource source) { super(id, defaultValue, vector, serializer, source, DoubleFlag::new); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java index 0faef17bf63..9f40013f627 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.flags; import com.yahoo.vespa.flags.json.DimensionHelper; -import javax.annotation.concurrent.Immutable; import java.util.EnumMap; import java.util.Map; import java.util.Objects; @@ -16,7 +15,6 @@ import java.util.function.Consumer; * * @author hakonhall */ -@Immutable public class FetchVector { /** * Note: If this enum is changed, you must also change {@link DimensionHelper}. diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FlagDefinition.java b/flags/src/main/java/com/yahoo/vespa/flags/FlagDefinition.java index 0b4e18d496d..1773a03feb1 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FlagDefinition.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FlagDefinition.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags; -import javax.annotation.concurrent.Immutable; import java.time.Instant; import java.util.EnumSet; import java.util.List; @@ -10,7 +9,6 @@ import java.util.Set; /** * @author hakonhall */ -@Immutable public class FlagDefinition { private final UnboundFlag<?, ?, ?> unboundFlag; private final List<String> owners; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FlagId.java b/flags/src/main/java/com/yahoo/vespa/flags/FlagId.java index 36ec5c03300..e60e1cc1b38 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FlagId.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FlagId.java @@ -1,14 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags; -import javax.annotation.concurrent.Immutable; import java.util.Objects; import java.util.regex.Pattern; /** * @author hakonhall */ -@Immutable public class FlagId implements Comparable<FlagId> { private static final Pattern ID_PATTERN = Pattern.compile("^[a-zA-Z0-9][a-zA-Z0-9._-]*$"); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 3ae302f4958..6d5c5a00bba 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -230,7 +230,7 @@ public class Flags { public static final UnboundStringFlag JDK_VERSION = defineStringFlag( "jdk-version", "17", - List.of("hmusum"), "2021-10-25", "2022-06-01", + List.of("hmusum"), "2021-10-25", "2022-06-10", "JDK version to use on host and inside containers. Note application-id dimension only applies for container, " + "while hostname and node type applies for host.", "Takes effect on restart for Docker container and on next host-admin tick for host", @@ -269,7 +269,7 @@ public class Flags { public static final UnboundBooleanFlag FAIL_DEPLOYMENT_WITH_INVALID_JVM_OPTIONS = defineFeatureFlag( "fail-deployment-with-invalid-jvm-options", true, - List.of("hmusum"), "2021-12-20", "2022-06-01", + List.of("hmusum"), "2021-12-20", "2022-07-01", "Whether to fail deployments with invalid JVM options in services.xml", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -283,7 +283,7 @@ public class Flags { public static final UnboundBooleanFlag ENABLE_DATA_HIGHWAY_IN_AWS = defineFeatureFlag( "enable-data-highway-in-aws", false, - List.of("hmusum"), "2022-01-06", "2022-06-01", + List.of("hmusum"), "2022-01-06", "2022-08-01", "Enable Data Highway in AWS", "Takes effect on restart of Docker container", ZONE_ID, APPLICATION_ID); @@ -369,7 +369,7 @@ public class Flags { public static final UnboundStringFlag APPLICATION_FILES_WITH_UNKNOWN_EXTENSION = defineStringFlag( "fail-deployment-for-files-with-unknown-extension", "NOOP", - List.of("hmusum"), "2022-04-27", "2022-05-27", + List.of("hmusum"), "2022-04-27", "2022-06-27", "Whether to log, fail or do nothing for deployments when app has a file with unknown extension (valid values LOG, FAIL, NOOP)", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -381,13 +381,6 @@ public class Flags { "Takes effect immediately", TENANT_ID); - public static final UnboundBooleanFlag INSTALL_JDK_FROM_LATEST_APPSTREAM_REPO = defineFeatureFlag( - "install-jdk-from-latest-appstream-repo", false, - List.of("hmusum"), "2022-05-03", "2022-06-03", - "Whether we should install JDK from latest-appstream repo on hosts (e.g. for security issues fixed in newer JDK versions)", - "Takes effect immediately", - HOSTNAME); - public static final UnboundStringFlag PROVISION_IN_EXTERNAL_ACCOUNT = defineStringFlag( "provision-in-external-account", "", List.of("mpolden"), "2022-05-02", "2022-09-01", @@ -402,6 +395,20 @@ public class Flags { "Takes effect on redeployment", APPLICATION_ID); + public static final UnboundStringFlag FILE_DISTRIBUTION_COMPRESSION_ALGORITHM = defineStringFlag( + "file-distribution-compression-algorithm", "gzip", + List.of("hmusum"), "2022-05-24", "2022-06-24", + "Which algorithm to use for compressing file references when distributing files. Valid values: none, gzip", + "Takes effect immediately", + APPLICATION_ID); + + public static final UnboundBooleanFlag FILE_DISTRIBUTION_COMPRESS_SINGLE_FILES = defineFeatureFlag( + "file-distribution-compress-single-files", false, + List.of("hmusum"), "2022-05-24", "2022-06-24", + "Whether to compress a file references that is a single file (directories are compressed by default).", + "Takes effect immediately", + APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/flags/src/main/java/com/yahoo/vespa/flags/IntFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/IntFlag.java index 689ceb9fe24..9976013be9e 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/IntFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/IntFlag.java @@ -1,12 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags; -import javax.annotation.concurrent.Immutable; /** * @author hakonhall */ -@Immutable public class IntFlag extends FlagImpl<Integer, IntFlag> { public IntFlag(FlagId id, int defaultValue, FetchVector vector, FlagSerializer<Integer> serializer, FlagSource source) { super(id, defaultValue, vector, serializer, source, IntFlag::new); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/JacksonFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/JacksonFlag.java index ec3953be931..69d003275f1 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/JacksonFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/JacksonFlag.java @@ -1,12 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags; -import javax.annotation.concurrent.Immutable; - /** * @author hakonhall */ -@Immutable public class JacksonFlag<T> extends FlagImpl<T, JacksonFlag<T>> { public JacksonFlag(FlagId id, T defaultValue, FetchVector vector, FlagSerializer<T> serializer, FlagSource source) { super(id, defaultValue, vector, serializer, source, JacksonFlag::new); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/ListFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/ListFlag.java index be7d9429858..29f51ca2d68 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/ListFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/ListFlag.java @@ -1,13 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags; -import javax.annotation.concurrent.Immutable; import java.util.List; /** * @author freva */ -@Immutable public class ListFlag<T> extends FlagImpl<List<T>, ListFlag<T>> { public ListFlag(FlagId id, List<T> defaultValue, FetchVector vector, FlagSerializer<List<T>> serializer, FlagSource source) { super(id, defaultValue, vector, serializer, source, ListFlag::new); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/LongFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/LongFlag.java index b8b38ce1ffd..eea25644962 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/LongFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/LongFlag.java @@ -1,12 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags; -import javax.annotation.concurrent.Immutable; - /** * @author hakonhall */ -@Immutable public class LongFlag extends FlagImpl<Long, LongFlag> { public LongFlag(FlagId id, Long defaultValue, FetchVector vector, FlagSerializer<Long> serializer, FlagSource source) { super(id, defaultValue, vector, serializer, source, LongFlag::new); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/StringFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/StringFlag.java index f1b6784921d..5034d2a0001 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/StringFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/StringFlag.java @@ -1,12 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags; -import javax.annotation.concurrent.Immutable; - /** * @author hakonhall */ -@Immutable public class StringFlag extends FlagImpl<String, StringFlag> { public StringFlag(FlagId id, String defaultValue, FetchVector vector, FlagSerializer<String> serializer, FlagSource source) { super(id, defaultValue, vector, serializer, source, StringFlag::new); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/UnboundBooleanFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/UnboundBooleanFlag.java index 17f8328a67e..b787131326c 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/UnboundBooleanFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/UnboundBooleanFlag.java @@ -4,12 +4,9 @@ package com.yahoo.vespa.flags; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.BooleanNode; -import javax.annotation.concurrent.Immutable; - /** * @author hakonhall */ -@Immutable public class UnboundBooleanFlag extends UnboundFlagImpl<Boolean, BooleanFlag, UnboundBooleanFlag> { public UnboundBooleanFlag(FlagId id) { this(id, false); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/UnboundDoubleFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/UnboundDoubleFlag.java index 6b6911f0357..c3adfeb0014 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/UnboundDoubleFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/UnboundDoubleFlag.java @@ -4,12 +4,9 @@ package com.yahoo.vespa.flags; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.DoubleNode; -import javax.annotation.concurrent.Immutable; - /** * @author freva */ -@Immutable public class UnboundDoubleFlag extends UnboundFlagImpl<Double, DoubleFlag, UnboundDoubleFlag> { public UnboundDoubleFlag(FlagId id, double defaultValue) { this(id, defaultValue, new FetchVector()); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/UnboundIntFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/UnboundIntFlag.java index 1f7c0206ee6..eed60bfa852 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/UnboundIntFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/UnboundIntFlag.java @@ -4,12 +4,9 @@ package com.yahoo.vespa.flags; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.IntNode; -import javax.annotation.concurrent.Immutable; - /** * @author hakonhall */ -@Immutable public class UnboundIntFlag extends UnboundFlagImpl<Integer, IntFlag, UnboundIntFlag> { public UnboundIntFlag(FlagId id, int defaultValue) { this(id, defaultValue, new FetchVector()); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/UnboundJacksonFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/UnboundJacksonFlag.java index 21680e7d75b..bef15270787 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/UnboundJacksonFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/UnboundJacksonFlag.java @@ -1,12 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags; -import javax.annotation.concurrent.Immutable; - /** * @author hakonhall */ -@Immutable public class UnboundJacksonFlag<T> extends UnboundFlagImpl<T, JacksonFlag<T>, UnboundJacksonFlag<T>> { public UnboundJacksonFlag(FlagId id, T defaultValue, Class<T> jacksonClass) { this(id, defaultValue, new FetchVector(), jacksonClass); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/UnboundListFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/UnboundListFlag.java index 624f23dea7d..3905c3fab5e 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/UnboundListFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/UnboundListFlag.java @@ -1,13 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags; -import javax.annotation.concurrent.Immutable; import java.util.List; /** * @author freva */ -@Immutable public class UnboundListFlag<T> extends UnboundFlagImpl<List<T>, ListFlag<T>, UnboundListFlag<T>> { public UnboundListFlag(FlagId id, List<T> defaultValue, Class<T> clazz) { this(id, defaultValue, clazz, new FetchVector()); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/UnboundLongFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/UnboundLongFlag.java index 590d7958c0b..594fae9dfc7 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/UnboundLongFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/UnboundLongFlag.java @@ -4,12 +4,9 @@ package com.yahoo.vespa.flags; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.LongNode; -import javax.annotation.concurrent.Immutable; - /** * @author hakonhall */ -@Immutable public class UnboundLongFlag extends UnboundFlagImpl<Long, LongFlag, UnboundLongFlag> { public UnboundLongFlag(FlagId id, long defaultValue) { this(id, defaultValue, new FetchVector()); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/UnboundStringFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/UnboundStringFlag.java index ae4e2269ccb..f96be55e2eb 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/UnboundStringFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/UnboundStringFlag.java @@ -4,12 +4,9 @@ package com.yahoo.vespa.flags; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.TextNode; -import javax.annotation.concurrent.Immutable; - /** * @author hakonhall */ -@Immutable public class UnboundStringFlag extends UnboundFlagImpl<String, StringFlag, UnboundStringFlag> { public UnboundStringFlag(FlagId id, String defaultValue) { this(id, defaultValue, new FetchVector()); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java b/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java index ddcf53aa5a4..28dbc588648 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/json/FlagData.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.flags.json.wire.WireFlagData; import com.yahoo.vespa.flags.json.wire.WireFlagDataList; import com.yahoo.vespa.flags.json.wire.WireRule; -import javax.annotation.concurrent.Immutable; import java.io.InputStream; import java.io.OutputStream; import java.util.List; @@ -25,7 +24,6 @@ import java.util.stream.Stream; * * @author hakonhall */ -@Immutable public class FlagData { private final FlagId id; private final List<Rule> rules; diff --git a/fnet/src/tests/frt/detach_supervisor/detach_supervisor_test.cpp b/fnet/src/tests/frt/detach_supervisor/detach_supervisor_test.cpp index 953b2e7432e..3207094e56d 100644 --- a/fnet/src/tests/frt/detach_supervisor/detach_supervisor_test.cpp +++ b/fnet/src/tests/frt/detach_supervisor/detach_supervisor_test.cpp @@ -75,7 +75,7 @@ struct RpcFixture : FRT_Invokable { req->SubRef(); return target; }; - int check_result(FRT_RPCRequest *req, uint64_t expect) { + static int check_result(FRT_RPCRequest *req, uint64_t expect) { int num_ok = 0; if (!req->CheckReturnTypes("l")) { ASSERT_EQUAL(req->GetErrorCode(), FRTE_RPC_CONNECTION); @@ -87,15 +87,15 @@ struct RpcFixture : FRT_Invokable { req->SubRef(); return num_ok; } - int verify_rpc(FNET_Connection *conn) { - auto *req = orb.AllocRPCRequest(); + static int verify_rpc(FNET_Connection *conn) { + auto *req = FRT_Supervisor::AllocRPCRequest(); req->SetMethodName("inc"); req->GetParams()->AddInt64(7); FRT_Supervisor::InvokeSync(conn->Owner()->GetScheduler(), conn, req, 300.0); return check_result(req, 8); } - int verify_rpc(FRT_Target *target) { - auto *req = orb.AllocRPCRequest(); + static int verify_rpc(FRT_Target *target) { + auto *req = FRT_Supervisor::AllocRPCRequest(); req->SetMethodName("inc"); req->GetParams()->AddInt64(4); target->InvokeSync(req, 300.0); @@ -135,8 +135,8 @@ TEST_MT_FFFFF("require that supervisor can be detached from transport", 4, Basic std::this_thread::sleep_for(50ms); self.reset(); // <--- detach supervisor for server 1 TEST_BARRIER(); // #4 - EXPECT_EQUAL(self->verify_rpc(target), 0); // outgoing 2way target should be closed - EXPECT_EQUAL(self->verify_rpc(client_target), 1); // pure client target should not be closed + EXPECT_EQUAL(RpcFixture::verify_rpc(target), 0); // outgoing 2way target should be closed + EXPECT_EQUAL(RpcFixture::verify_rpc(client_target), 1); // pure client target should not be closed TEST_BARRIER(); // #5 target->SubRef(); client_target->SubRef(); diff --git a/fnet/src/vespa/fnet/frt/supervisor.h b/fnet/src/vespa/fnet/frt/supervisor.h index 610baca5d93..93272e93b4a 100644 --- a/fnet/src/vespa/fnet/frt/supervisor.h +++ b/fnet/src/vespa/fnet/frt/supervisor.h @@ -67,7 +67,7 @@ public: FRT_Target *GetTarget(const char *spec); FRT_Target *Get2WayTarget(const char *spec, FNET_Context connContext = FNET_Context()); FRT_Target *GetTarget(int port); - FRT_RPCRequest *AllocRPCRequest(FRT_RPCRequest *tradein = nullptr); + static FRT_RPCRequest *AllocRPCRequest(FRT_RPCRequest *tradein = nullptr); struct SchedulerPtr { FNET_Scheduler *ptr; diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml index ad61e1c28c0..734c096d373 100644 --- a/hosted-tenant-base/pom.xml +++ b/hosted-tenant-base/pom.xml @@ -222,6 +222,22 @@ </rules> </configuration> </execution> + <execution> + <id>enforce-no-jupiter-compile-scope</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <bannedDependencies> + <!-- Fail validation for apps with junit-jupiter-api in compile scope. --> + <excludes> + <exclude>org.junit.jupiter:junit-jupiter-api:*:jar:compile</exclude> + </excludes> + </bannedDependencies> + </rules> + </configuration> + </execution> </executions> </plugin> @@ -239,8 +255,8 @@ org.junit.jupiter:junit-jupiter-engine:test, org.junit.vintage:junit-vintage-engine:test, com.yahoo.vespa:application:test, - com.yahoo.vespa:container-test:runtime, - com.yahoo.vespa:vespa-feed-client:runtime, <!-- prevent effective compile scope of vespa-feed-client in test bundle --> + com.yahoo.vespa:container-test:test, + com.yahoo.vespa:vespa-feed-client:test, <!-- prevent effective compile scope of vespa-feed-client in test bundle --> com.yahoo.vespa:vespa-feed-client-api:provided, com.yahoo.vespa:tenant-cd-api:provided </testBundleScopeOverrides> diff --git a/messagebus/src/tests/messageordering/messageordering.cpp b/messagebus/src/tests/messageordering/messageordering.cpp index 919d242fc47..e82e58d1730 100644 --- a/messagebus/src/tests/messageordering/messageordering.cpp +++ b/messagebus/src/tests/messageordering/messageordering.cpp @@ -155,6 +155,7 @@ Test::Main() ssp.setTimeout(400s); SourceSession::UP ss = srcNet.mb.createSourceSession(src, ssp); DestinationSession::UP ds = dstNet.mb.createDestinationSession("session", true, dst); + dst.setDestinationSession(*ds); ASSERT_EQUAL(400s, ssp.getTimeout()); // wait for slobrok registration diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.h b/messagebus/src/vespa/messagebus/network/rpcnetwork.h index 3da7d88f994..38f35d1266a 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.h +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.h @@ -33,8 +33,7 @@ class RPCServiceAddress; * Network implementation based on RPC. This class is responsible for * keeping track of services and for sending messages to services. **/ -class RPCNetwork : public INetwork, - public FRT_Invokable { +class RPCNetwork : public FRT_Invokable, public INetwork { private: using CompressionConfig = vespalib::compression::CompressionConfig; struct SendContext : public RPCTarget::IVersionHandler { diff --git a/messagebus/src/vespa/messagebus/network/rpcsend.h b/messagebus/src/vespa/messagebus/network/rpcsend.h index 9b2e76a8a21..e548c4adca2 100644 --- a/messagebus/src/vespa/messagebus/network/rpcsend.h +++ b/messagebus/src/vespa/messagebus/network/rpcsend.h @@ -29,8 +29,8 @@ public: virtual void fill(const vespalib::Memory & name, vespalib::slime::Cursor & v) const = 0; }; -class RPCSend : public RPCSendAdapter, - public FRT_Invokable, +class RPCSend : public FRT_Invokable, + public RPCSendAdapter, public FRT_IRequestWait, public IDiscardHandler, public IReplyHandler diff --git a/messagebus/src/vespa/messagebus/routing/routingnode.cpp b/messagebus/src/vespa/messagebus/routing/routingnode.cpp index 30048d9cdd1..2c1de3d31bb 100644 --- a/messagebus/src/vespa/messagebus/routing/routingnode.cpp +++ b/messagebus/src/vespa/messagebus/routing/routingnode.cpp @@ -387,17 +387,17 @@ RoutingNode::lookupRoute() { RoutingTable::SP table = _mbus.getRoutingTable(_msg.getProtocol()); Hop &hop = _route.getHop(0); - const RouteDirective &dir = static_cast<const RouteDirective&>(hop.getDirective(0)); - if (dir.getType() == IHopDirective::TYPE_ROUTE) { + const auto *dir = dynamic_cast<const RouteDirective *>(&hop.getDirective(0)); + if (dir && dir->getType() == IHopDirective::TYPE_ROUTE) { - if (!table || !table->hasRoute(dir.getName())) { - setError(ErrorCode::ILLEGAL_ROUTE, make_string("Route '%s' does not exist.", dir.getName().c_str())); + if (!table || !table->hasRoute(dir->getName())) { + setError(ErrorCode::ILLEGAL_ROUTE, make_string("Route '%s' does not exist.", dir->getName().c_str())); return false; } - insertRoute(*table->getRoute(dir.getName())); + insertRoute(*table->getRoute(dir->getName())); _trace.trace(TraceLevel::SPLIT_MERGE, make_string("Route '%s' retrieved by directive; new route is '%s'.", - dir.getName().c_str(), _route.toString().c_str())); + dir->getName().c_str(), _route.toString().c_str())); return true; } if (table) { diff --git a/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java b/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java index 1831903d626..bc3f08ce3d6 100644 --- a/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java +++ b/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java @@ -11,6 +11,8 @@ import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorAddress; import com.yahoo.tensor.TensorType; +import java.io.File; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -58,13 +60,22 @@ public class BertBaseEmbedder implements Embedder { options.setInterOpThreads(modifyThreadCount(config.onnxInterOpThreads())); options.setIntraOpThreads(modifyThreadCount(config.onnxIntraOpThreads())); - // Todo: use either file or url - tokenizer = new WordPieceEmbedder.Builder(config.tokenizerVocabUrl().getAbsolutePath()).build(); - evaluator = new OnnxEvaluator(config.transformerModelUrl().getAbsolutePath(), options); + String tokenizerFile = pathOrUrl(config.tokenizerVocabPath(), config.tokenizerVocabUrl()); + String modelFile = pathOrUrl(config.transformerModelPath(), config.transformerModelUrl()); + + tokenizer = new WordPieceEmbedder.Builder(tokenizerFile).build(); + evaluator = new OnnxEvaluator(modelFile, options); validateModel(); } + private String pathOrUrl(Path path, File url) { + if (path.endsWith("services.xml")) { + return url.getAbsolutePath(); + } + return path.toAbsolutePath().toString(); + } + private void validateModel() { Map<String, TensorType> inputs = evaluator.getInputInfo(); validateName(inputs, inputIdsName, "input"); diff --git a/model-integration/src/test/java/ai/vespa/embedding/BertBaseEmbedderTest.java b/model-integration/src/test/java/ai/vespa/embedding/BertBaseEmbedderTest.java index 464e5941e89..c224b87982d 100644 --- a/model-integration/src/test/java/ai/vespa/embedding/BertBaseEmbedderTest.java +++ b/model-integration/src/test/java/ai/vespa/embedding/BertBaseEmbedderTest.java @@ -1,6 +1,7 @@ package ai.vespa.embedding; import ai.vespa.modelintegration.evaluator.OnnxEvaluator; +import com.yahoo.config.FileReference; import com.yahoo.config.UrlReference; import com.yahoo.embedding.BertBaseEmbedderConfig; import com.yahoo.tensor.Tensor; @@ -14,8 +15,6 @@ import static org.junit.Assume.assumeTrue; public class BertBaseEmbedderTest { - - @Test public void testEmbedder() { String vocabPath = "src/test/models/onnx/transformer/dummy_vocab.txt"; @@ -23,8 +22,10 @@ public class BertBaseEmbedderTest { assumeTrue(OnnxEvaluator.isRuntimeAvailable(modelPath)); BertBaseEmbedderConfig.Builder builder = new BertBaseEmbedderConfig.Builder(); - builder.tokenizerVocabUrl(new UrlReference(vocabPath)); - builder.transformerModelUrl(new UrlReference(modelPath)); + builder.tokenizerVocabPath(new FileReference(vocabPath)); + builder.tokenizerVocabUrl(new UrlReference("")); + builder.transformerModelPath(new FileReference(modelPath)); + builder.transformerModelUrl(new UrlReference("")); BertBaseEmbedder embedder = new BertBaseEmbedder(builder.build()); TensorType destType = TensorType.fromSpec("tensor<float>(x[7])"); diff --git a/node-admin/pom.xml b/node-admin/pom.xml index b58d20e8877..ff1e159de0d 100644 --- a/node-admin/pom.xml +++ b/node-admin/pom.xml @@ -56,10 +56,6 @@ <!-- Compile --> <dependency> - <groupId>com.google.code.findbugs</groupId> - <artifactId>jsr305</artifactId> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>orchestrator-restapi</artifactId> <version>${project.version}</version> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index b820bf42287..c893f7ffee4 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -81,7 +81,6 @@ public class StorageMaintainer { } public boolean syncLogs(NodeAgentContext context, boolean throttle) { - if (context.zone().getCloudName().value().equals("gcp")) return false; Optional<URI> archiveUri = context.node().archiveUri(); if (archiveUri.isEmpty()) return false; ApplicationId owner = context.node().owner().orElseThrow(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java index 3004fb6490b..cad6b7b76ac 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java @@ -262,7 +262,7 @@ public class CoredumpHandler { node.parentHostname().ifPresent(parent -> dimensionsBuilder.add("parentHostname", parent)); dimensionsBuilder.add("orchestratorState", node.orchestratorStatus().asString()); - node.currentVespaVersion().ifPresent(vespaVersion -> dimensionsBuilder.add("vespaVersion", vespaVersion.toFullString())); + dimensionsBuilder.add("system", context.zone().getSystemName().value()); return dimensionsBuilder.build(); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java index 280b28e87ca..ce6d3667dcc 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java @@ -20,6 +20,8 @@ public class ZstdCompressingInputStream extends InputStream { private final byte[] inputBuffer; private final byte[] outputBuffer; + private boolean firstRead = true; + private boolean eof = false; private int outputPosition = 0; private int outputLength = 0; private boolean isClosed = false; @@ -39,9 +41,15 @@ public class ZstdCompressingInputStream extends InputStream { throwIfClosed(); if (outputPosition >= outputLength) { - int readLength = is.read(inputBuffer); - if (readLength == -1) - return -1; + int readLength = eof ? -1 : is.read(inputBuffer); + if (readLength == -1) { + if (!firstRead) + return -1; + // zstd compressing an empty file results in a 13 bytes file. + eof = true; + readLength = 0; + } + firstRead = false; outputLength = compressor.compress(inputBuffer, 0, readLength, outputBuffer, 0, outputBuffer.length); outputPosition = 0; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java index 275a99a4993..6e74297f2e7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/DebugHandlerHelper.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.node.admin.provider; -import javax.annotation.concurrent.ThreadSafe; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -17,7 +16,6 @@ import java.util.stream.Collectors; * * @author hakonhall */ -@ThreadSafe public class DebugHandlerHelper implements NodeAdminDebugHandler { private final ConcurrentMap<String, Supplier<Object>> suppliers = new ConcurrentHashMap<>(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminDebugHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminDebugHandler.java index d2928891a53..f62f29b8b73 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminDebugHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminDebugHandler.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.node.admin.provider; -import javax.annotation.concurrent.ThreadSafe; import java.util.Map; /** @@ -10,7 +9,6 @@ import java.util.Map; * * @author hakonhall */ -@ThreadSafe public interface NodeAdminDebugHandler { /** * The Object in the map values must be serializable with Jackson's ObjectMapper. diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Mark.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Mark.java index 8058fecc631..9c6871010c8 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Mark.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Mark.java @@ -2,13 +2,11 @@ package com.yahoo.vespa.hosted.node.admin.task.util.editor; -import javax.annotation.concurrent.Immutable; import java.util.Objects; /** * @author hakon */ -@Immutable public class Mark { private final Position position; private final Version version; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Position.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Position.java index ff6ebf32b06..8f892cf8650 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Position.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Position.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.editor; -import javax.annotation.concurrent.Immutable; import java.util.Comparator; import java.util.Objects; @@ -10,7 +9,6 @@ import java.util.Objects; * * @author hakon */ -@Immutable public class Position implements Comparable<Position> { private static final Position START_POSITION = new Position(0, 0); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Version.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Version.java index 375f6ae1d2c..137fb677e83 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Version.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/editor/Version.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.node.admin.task.util.editor; -import javax.annotation.concurrent.Immutable; import java.util.Objects; /** @@ -10,7 +9,6 @@ import java.util.Objects; * * @author hakon */ -@Immutable public class Version { private final int version; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/LineEdit.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/LineEdit.java index 8015cc27fd4..a471d952dbc 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/LineEdit.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/LineEdit.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.file; -import javax.annotation.concurrent.Immutable; import java.util.List; import static com.yahoo.vespa.hosted.node.admin.task.util.file.LineEdit.Type.REPLACE; @@ -10,7 +9,6 @@ import static com.yahoo.vespa.hosted.node.admin.task.util.file.LineEdit.Type.NON /** * @author hakonhall */ -@Immutable public class LineEdit { enum Type { NONE, REPLACE } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java index ce8c260a7a4..ab43c3fd5fc 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java @@ -11,6 +11,7 @@ import java.util.Random; import static com.yahoo.vespa.hosted.node.admin.maintenance.sync.ZstdCompressingInputStream.compressor; import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; /** * @author freva @@ -25,7 +26,21 @@ public class ZstdCompressingInputStreamTest { assertCompression(data, 1 << 14); } + @Test + public void compress_empty_file_test() { + byte[] compressedData = compress(new byte[0], 1 << 10); + assertEquals("zstd compressing an empty file results in a 13 bytes file", 13, compressedData.length); + } + private static void assertCompression(byte[] data, int bufferSize) { + byte[] compressedData = compress(data, bufferSize); + byte[] decompressedData = new byte[data.length]; + compressor.decompress(compressedData, 0, compressedData.length, decompressedData, 0, decompressedData.length); + + assertArrayEquals(data, decompressedData); + } + + private static byte[] compress(byte[] data, int bufferSize) { ByteArrayInputStream bais = new ByteArrayInputStream(data); ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (ZstdCompressingInputStream zcis = new ZstdCompressingInputStream(bais, bufferSize)) { @@ -37,10 +52,6 @@ public class ZstdCompressingInputStreamTest { throw new UncheckedIOException(e); } - byte[] compressedData = baos.toByteArray(); - byte[] decompressedData = new byte[data.length]; - compressor.decompress(compressedData, 0, compressedData.length, decompressedData, 0, decompressedData.length); - - assertArrayEquals(data, decompressedData); + return baos.toByteArray(); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index 12df01a7538..5237e48907a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; @@ -14,7 +15,6 @@ import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.flags.custom.SharedHost; import com.yahoo.vespa.hosted.provision.NodeRepository; - import java.util.Map; import java.util.TreeMap; @@ -82,24 +82,16 @@ public class CapacityPolicies { public NodeResources defaultNodeResources(ClusterSpec clusterSpec, ApplicationId applicationId, boolean exclusive) { if (clusterSpec.type() == ClusterSpec.Type.admin) { - Architecture architecture = architecture(applicationId); - - // The lowest amount resources that can be exclusive allocated (i.e. a matching host flavor for this exists) - NodeResources smallestExclusiveResources = new NodeResources(0.5, 4, 50, 0.3); + Architecture architecture = adminClusterArchitecture(applicationId); if (clusterSpec.id().value().equals("cluster-controllers")) { - if (requiresExclusiveHost(clusterSpec.type(), exclusive)) { - return versioned(clusterSpec, Map.of(new Version("0"), smallestExclusiveResources)).with(architecture); - } - return versioned(clusterSpec, Map.of(new Version("0"), new NodeResources(0.25, 1.14, 10, 0.3), - new Version("7.586.50"), new NodeResources(0.25, 1.333, 10, 0.3), - new Version("7.586.54"), new NodeResources(0.25, 1.14, 10, 0.3))) + return clusterControllerResources(clusterSpec, exclusive) .with(architecture); } return (requiresExclusiveHost(clusterSpec.type(), exclusive) - ? versioned(clusterSpec, Map.of(new Version("0"), smallestExclusiveResources)) - : versioned(clusterSpec, Map.of(new Version("0"), new NodeResources(0.5, 2, 50, 0.3)))) + ? versioned(clusterSpec, Map.of(new Version("0"), smallestExclusiveResources())) + : versioned(clusterSpec, Map.of(new Version("0"), smallestSharedResources()))) .with(architecture); } @@ -108,7 +100,16 @@ public class CapacityPolicies { : versioned(clusterSpec, Map.of(new Version("0"), new NodeResources(1.5, 8, 50, 0.3))); } - private Architecture architecture(ApplicationId instance) { + private NodeResources clusterControllerResources(ClusterSpec clusterSpec, boolean exclusive) { + if (requiresExclusiveHost(clusterSpec.type(), exclusive)) { + return versioned(clusterSpec, Map.of(new Version("0"), smallestExclusiveResources())); + } + return versioned(clusterSpec, Map.of(new Version("0"), new NodeResources(0.25, 1.14, 10, 0.3), + new Version("7.586.50"), new NodeResources(0.25, 1.333, 10, 0.3), + new Version("7.586.54"), new NodeResources(0.25, 1.14, 10, 0.3))); + } + + private Architecture adminClusterArchitecture(ApplicationId instance) { return Architecture.valueOf(adminClusterNodeArchitecture.with(APPLICATION_ID, instance.serializedForm()).value()); } @@ -124,6 +125,20 @@ public class CapacityPolicies { .getValue(); } + // The lowest amount resources that can be exclusive allocated (i.e. a matching host flavor for this exists) + private NodeResources smallestExclusiveResources() { + return (zone.getCloud().name().equals(CloudName.from("gcp"))) + ? new NodeResources(1, 4, 50, 0.3) + : new NodeResources(0.5, 4, 50, 0.3); + } + + // The lowest amount resources that can be shared (i.e. a matching host flavor for this exists) + private NodeResources smallestSharedResources() { + return (zone.getCloud().name().equals(CloudName.from("gcp"))) + ? new NodeResources(1, 4, 50, 0.3) + : new NodeResources(0.5, 2, 50, 0.3); + } + /** * Returns whether the nodes requested can share physical host with other applications. * A security feature which only makes sense for prod. diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 16ee8281b9a..5d9d13c48dc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.transaction.Mutex; @@ -17,6 +18,8 @@ import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner.HostSharing import java.util.List; import java.util.Optional; import java.util.function.Supplier; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Collectors; /** @@ -27,6 +30,7 @@ import java.util.stream.Collectors; public class GroupPreparer { private static final Mutex PROBE_LOCK = () -> {}; + private static final Logger log = Logger.getLogger(GroupPreparer.class.getName()); private final NodeRepository nodeRepository; private final Optional<HostProvisioner> hostProvisioner; @@ -68,6 +72,7 @@ public class GroupPreparer { public PrepareResult prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, List<Node> surplusActiveNodes, NodeIndices indices, int wantedGroups, NodesAndHosts<LockedNodeList> allNodesAndHosts) { + log.log(Level.FINE, "Preparing " + cluster.type().name() + " " + cluster.id() + " with requested resources " + requestedNodes.resources().orElse(NodeResources.unspecified())); // Try preparing in memory without global unallocated lock. Most of the time there should be no changes and we // can return nodes previously allocated. NodeAllocation probeAllocation = prepareAllocation(application, cluster, requestedNodes, surplusActiveNodes, diff --git a/parent/pom.xml b/parent/pom.xml index 732a66710c6..b24390ae474 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -936,7 +936,7 @@ <asm.version>9.2</asm.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> - <athenz.version>1.10.14</athenz.version> <!-- WARNING: sync cloud-tenant-base-dependencies-enforcer/pom.xml --> + <athenz.version>1.10.54</athenz.version> <!-- WARNING: sync cloud-tenant-base-dependencies-enforcer/pom.xml --> <jjwt.version>0.11.2</jjwt.version> <aws.sdk.version>1.11.974</aws.sdk.version> <!-- Athenz END --> diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index ec48d8ebc65..64db35b7acb 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -149,8 +149,8 @@ public: schema, std::make_shared<DocumentDBMaintenanceConfig>(), search::LogDocumentStore::Config(), - std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make()), - std::make_shared<const AllocConfig>(), + ThreadingServiceConfig::make(), + AllocConfig::makeDefault(), "client", docTypeName.getName()); } diff --git a/searchcore/src/tests/proton/attribute/CMakeLists.txt b/searchcore/src/tests/proton/attribute/CMakeLists.txt index 56c0d7301b2..bea3e7c2416 100644 --- a/searchcore/src/tests/proton/attribute/CMakeLists.txt +++ b/searchcore/src/tests/proton/attribute/CMakeLists.txt @@ -7,6 +7,7 @@ vespa_add_executable(searchcore_attribute_test_app TEST searchcore_attribute searchcore_flushengine searchcore_pcommon + searchcore_test searchlib_test GTest::GTest ) diff --git a/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp b/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp index a0d0958d095..0ce67e8bfcb 100644 --- a/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_aspect_delayer/attribute_aspect_delayer_test.cpp @@ -24,16 +24,11 @@ using vespa::config::search::SummaryConfigBuilder; using vespa::config::search::SummarymapConfig; using vespa::config::search::SummarymapConfigBuilder; using search::attribute::Config; -using search::attribute::BasicType; -using search::attribute::CollectionType; -namespace std -{ +namespace std { -ostream &operator<<(ostream &os, const SummarymapConfig::Override &override) -{ - os << "{field=" << override.field << ", command=" << override.command << ", arguments=" << override.arguments << "}"; - return os; +ostream &operator<<(ostream &os, const SummarymapConfig::Override &override) { + return os << "{field=" << override.field << ", command=" << override.command << ", arguments=" << override.arguments << "}"; } } diff --git a/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp index cbc345f68d8..93cb456cc06 100644 --- a/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp @@ -1,14 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchcore/proton/attribute/attribute_directory.h> #include <vespa/searchcore/proton/attribute/attribute_factory.h> #include <vespa/searchcore/proton/attribute/attribute_initializer.h> #include <vespa/searchcore/proton/attribute/attributedisklayout.h> #include <vespa/searchcore/proton/test/attribute_utils.h> #include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/attribute/integerbase.h> #include <vespa/searchlib/test/directory_handler.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/vespalib/testkit/testapp.h> #include <vespa/log/log.h> LOG_SETUP("attribute_initializer_test"); @@ -93,7 +95,7 @@ struct Fixture vespalib::ThreadStackExecutor _executor; Fixture(); ~Fixture(); - std::unique_ptr<AttributeInitializer> createInitializer(const AttributeSpec &spec, SerialNum serialNum); + std::unique_ptr<AttributeInitializer> createInitializer(AttributeSpec && spec, SerialNum serialNum); }; Fixture::Fixture() @@ -107,9 +109,9 @@ Fixture::Fixture() Fixture::~Fixture() = default; std::unique_ptr<AttributeInitializer> -Fixture::createInitializer(const AttributeSpec &spec, SerialNum serialNum) +Fixture::createInitializer(AttributeSpec &&spec, SerialNum serialNum) { - return std::make_unique<AttributeInitializer>(_diskLayout->createAttributeDir(spec.getName()), "test.subdb", spec, serialNum, _factory, _executor); + return std::make_unique<AttributeInitializer>(_diskLayout->createAttributeDir(spec.getName()), "test.subdb", std::move(spec), serialNum, _factory, _executor); } TEST("require that integer attribute can be initialized") diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index 7b0451d0f9d..b446ca367c0 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -1,9 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/config-attributes.h> -#include <vespa/fastos/file.h> -#include <vespa/searchcommon/attribute/i_attribute_functor.h> -#include <vespa/searchcommon/attribute/iattributevector.h> #include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h> #include <vespa/searchcore/proton/attribute/attribute_manager_initializer.h> #include <vespa/searchcore/proton/attribute/attribute_writer.h> @@ -33,11 +29,16 @@ #include <vespa/searchlib/predicate/predicate_tree_annotator.h> #include <vespa/searchlib/test/directory_handler.h> #include <vespa/searchlib/test/mock_gid_to_lid_mapping.h> +#include <vespa/searchcommon/attribute/i_attribute_functor.h> +#include <vespa/searchcommon/attribute/iattributevector.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/foreground_thread_executor.h> #include <vespa/vespalib/util/foregroundtaskexecutor.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/config-attributes.h> +#include <vespa/fastos/file.h> #include <vespa/log/log.h> LOG_SETUP("attribute_manager_test"); @@ -106,19 +107,19 @@ public: } const string test_dir = "test_output"; -const AVConfig INT32_SINGLE = AttributeUtils::getInt32Config(); -const AVConfig INT32_ARRAY = AttributeUtils::getInt32ArrayConfig(); +const AVConfig & INT32_SINGLE = AttributeUtils::getInt32Config(); +const AVConfig & INT32_ARRAY = AttributeUtils::getInt32ArrayConfig(); void fillAttribute(const AttributeVector::SP &attr, uint32_t numDocs, int64_t value, uint64_t lastSyncToken) { - AttributeUtils::fillAttribute(attr, numDocs, value, lastSyncToken); + AttributeUtils::fillAttribute(*attr, numDocs, value, lastSyncToken); } void fillAttribute(const AttributeVector::SP &attr, uint32_t from, uint32_t to, int64_t value, uint64_t lastSyncToken) { - AttributeUtils::fillAttribute(attr, from, to, value, lastSyncToken); + AttributeUtils::fillAttribute(*attr, from, to, value, lastSyncToken); } search::SerialNum getCreateSerialNum(const AttributeGuard::UP &guard) @@ -138,7 +139,7 @@ struct ImportedAttributesRepoBuilder { ImportedAttributesRepo::UP _repo; ImportedAttributesRepoBuilder() : _repo(std::make_unique<ImportedAttributesRepo>()) {} void add(const vespalib::string &name) { - auto refAttr = std::make_shared<ReferenceAttribute>(name + "_ref", AVConfig(BasicType::REFERENCE)); + auto refAttr = std::make_shared<ReferenceAttribute>(name + "_ref"); refAttr->setGidToLidMapperFactory(std::make_shared<MockGidToLidMapperFactory>()); auto targetAttr = search::AttributeFactory::createAttribute(name + "_target", INT32_SINGLE); auto documentMetaStore = std::shared_ptr<search::IDocumentMetaStoreContext>(); @@ -215,10 +216,9 @@ struct SequentialAttributeManager { SequentialAttributesInitializer initializer; proton::AttributeManager mgr; - SequentialAttributeManager(const AttributeManager &currMgr, - const AttrMgrSpec &newSpec) + SequentialAttributeManager(const AttributeManager &currMgr, AttrMgrSpec && newSpec) : initializer(newSpec.getDocIdLimit()), - mgr(currMgr, newSpec, initializer) + mgr(currMgr, std::move(newSpec), initializer) { mgr.addInitializedAttributes(initializer.getInitializedAttributes()); } @@ -447,7 +447,7 @@ TEST_F("require that reconfig can add attributes", Fixture) newSpec.push_back(AttributeSpec("a2", INT32_SINGLE)); newSpec.push_back(AttributeSpec("a3", INT32_SINGLE)); - SequentialAttributeManager sam(f._m, AttrMgrSpec(newSpec, f._m.getNumDocs(), 10)); + SequentialAttributeManager sam(f._m, AttrMgrSpec(std::move(newSpec), f._m.getNumDocs(), 10)); std::vector<AttributeGuard> list; sam.mgr.getAttributeList(list); std::sort(list.begin(), list.end(), [](const AttributeGuard & a, const AttributeGuard & b) { @@ -470,7 +470,7 @@ TEST_F("require that reconfig can remove attributes", Fixture) AttrSpecList newSpec; newSpec.push_back(AttributeSpec("a2", INT32_SINGLE)); - SequentialAttributeManager sam(f._m, AttrMgrSpec(newSpec, 1, 10)); + SequentialAttributeManager sam(f._m, AttrMgrSpec(std::move(newSpec), 1, 10)); std::vector<AttributeGuard> list; sam.mgr.getAttributeList(list); EXPECT_EQUAL(1u, list.size()); @@ -493,7 +493,7 @@ TEST_F("require that new attributes after reconfig are initialized", Fixture) newSpec.push_back(AttributeSpec("a2", INT32_SINGLE)); newSpec.push_back(AttributeSpec("a3", INT32_ARRAY)); - SequentialAttributeManager sam(f._m, AttrMgrSpec(newSpec, 3, 4)); + SequentialAttributeManager sam(f._m, AttrMgrSpec(std::move(newSpec), 3, 4)); AttributeGuard::UP a2ap = sam.mgr.getAttribute("a2"); AttributeGuard &a2(*a2ap); EXPECT_EQUAL(3u, a2->getNumDocs()); @@ -519,13 +519,13 @@ TEST_F("require that removed attributes cannot resurrect", BaseFixture) } AttrSpecList ns1; - SequentialAttributeManager am2(*am1, AttrMgrSpec(ns1, 3, 16)); + SequentialAttributeManager am2(*am1, AttrMgrSpec(std::move(ns1), 3, 16)); am1.reset(); AttrSpecList ns2; ns2.push_back(AttributeSpec("a1", INT32_SINGLE)); // 2 new documents added since a1 was removed - SequentialAttributeManager am3(am2.mgr, AttrMgrSpec(ns2, 5, 20)); + SequentialAttributeManager am3(am2.mgr, AttrMgrSpec(std::move(ns2), 5, 20)); AttributeGuard::UP ag1ap = am3.mgr.getAttribute("a1"); AttributeGuard &ag1(*ag1ap); @@ -545,7 +545,7 @@ TEST_F("require that extra attribute is not treated as removed", Fixture) ex->commit(CommitParam(1)); AttrSpecList ns; - SequentialAttributeManager am2(f._m, AttrMgrSpec(ns, 2, 1)); + SequentialAttributeManager am2(f._m, AttrMgrSpec(std::move(ns), 2, 1)); EXPECT_TRUE(am2.mgr.getAttribute("ex")->operator->() == ex.get()); // reuse } @@ -558,7 +558,7 @@ TEST_F("require that removed fields can be pruned", Fixture) AttrSpecList newSpec; newSpec.push_back(AttributeSpec("a2", INT32_SINGLE)); - SequentialAttributeManager sam(f._m, AttrMgrSpec(newSpec, 1, 11)); + SequentialAttributeManager sam(f._m, AttrMgrSpec(std::move(newSpec), 1, 11)); sam.mgr.pruneRemovedFields(11); FastOS_StatInfo si; @@ -690,7 +690,7 @@ TEST_F("require that attributes can be initialized and loaded in sequence", Base newSpec.push_back(AttributeSpec("a2", INT32_SINGLE)); newSpec.push_back(AttributeSpec("a3", INT32_SINGLE)); - SequentialAttributeManager newMgr(amf._m, AttrMgrSpec(newSpec, 10, createSerialNum + 5)); + SequentialAttributeManager newMgr(amf._m, AttrMgrSpec(std::move(newSpec), 10, createSerialNum + 5)); AttributeGuard::UP a1 = newMgr.mgr.getAttribute("a1"); TEST_DO(validateAttribute(*a1->get())); @@ -819,7 +819,7 @@ TEST_F("require that attribute vector of wrong type is dropped", BaseFixture) newSpec.push_back(AttributeSpec("a4", dense_tensor)); newSpec.push_back(AttributeSpec("a5", predicate)); newSpec.push_back(AttributeSpec("a6", predicate2)); - SequentialAttributeManager am2(*am1, AttrMgrSpec(newSpec, 5, 20)); + SequentialAttributeManager am2(*am1, AttrMgrSpec(std::move(newSpec), 5, 20)); TEST_DO(assertCreateSerialNum(*am1, "a1", 1)); TEST_DO(assertCreateSerialNum(*am1, "a2", 2)); TEST_DO(assertCreateSerialNum(*am1, "a3", 3)); @@ -861,7 +861,7 @@ TEST_F("require that shrink flushtarget is handed over to new attribute manager" am1->addAttribute({"a1", INT32_SINGLE}, 4); AttrSpecList newSpec; newSpec.push_back(AttributeSpec("a1", INT32_SINGLE)); - auto am2 = am1->create(AttrMgrSpec(newSpec, 5, 20)); + auto am2 = am1->create(AttrMgrSpec(std::move(newSpec), 5, 20)); auto am3 = std::dynamic_pointer_cast<AttributeManager>(am2); TEST_DO(assertShrinkTargetSerial(*am3, "a1", 3)); EXPECT_EQUAL(am1->getShrinker("a1"), am3->getShrinker("a1")); diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp index 7ad427f303f..0d09d45c33a 100644 --- a/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp @@ -1,9 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/document/datatype/documenttype.h> -#include <vespa/document/fieldvalue/document.h> -#include <vespa/document/fieldvalue/intfieldvalue.h> -#include <vespa/document/repo/configbuilder.h> #include <vespa/searchcore/proton/attribute/attribute_populator.h> #include <vespa/searchcore/proton/attribute/attributemanager.h> #include <vespa/searchcore/proton/common/hw_info.h> @@ -11,6 +7,12 @@ #include <vespa/searchlib/attribute/interlock.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/test/directory_handler.h> +#include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/searchcommon/attribute/config.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/fieldvalue/intfieldvalue.h> +#include <vespa/document/repo/configbuilder.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/foreground_thread_executor.h> #include <vespa/vespalib/util/foregroundtaskexecutor.h> diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index de6b923bdc1..800cb8aa0ce 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -1,20 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/config-attributes.h> -#include <vespa/document/datatype/documenttype.h> -#include <vespa/document/datatype/mapdatatype.h> -#include <vespa/document/datatype/tensor_data_type.h> -#include <vespa/document/fieldvalue/document.h> -#include <vespa/document/predicate/predicate_slime_builder.h> -#include <vespa/document/update/arithmeticvalueupdate.h> -#include <vespa/document/update/assignvalueupdate.h> -#include <vespa/document/update/documentupdate.h> -#include <vespa/eval/eval/simple_value.h> -#include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/eval/test/value_compare.h> -#include <vespa/eval/eval/value.h> -#include <vespa/searchcommon/attribute/attributecontent.h> -#include <vespa/searchcommon/attribute/iattributevector.h> #include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h> #include <vespa/searchcore/proton/attribute/attribute_writer.h> #include <vespa/searchcore/proton/attribute/attributemanager.h> @@ -39,6 +24,22 @@ #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/tensor_attribute.h> #include <vespa/searchlib/test/directory_handler.h> +#include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/searchcommon/attribute/iattributevector.h> +#include <vespa/searchcommon/attribute/config.h> +#include <vespa/config-attributes.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/datatype/mapdatatype.h> +#include <vespa/document/datatype/tensor_data_type.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/predicate/predicate_slime_builder.h> +#include <vespa/document/update/arithmeticvalueupdate.h> +#include <vespa/document/update/assignvalueupdate.h> +#include <vespa/document/update/documentupdate.h> +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/test/value_compare.h> +#include <vespa/eval/eval/value.h> #include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/test/insertion_operators.h> @@ -118,13 +119,13 @@ const AVConfig INT32_ARRAY = unregister(AVConfig(AVBasicType::INT32, AVCollectio void fillAttribute(const AttributeVector::SP &attr, uint32_t numDocs, int64_t value, uint64_t lastSyncToken) { - AttributeUtils::fillAttribute(attr, numDocs, value, lastSyncToken); + AttributeUtils::fillAttribute(*attr, numDocs, value, lastSyncToken); } void fillAttribute(const AttributeVector::SP &attr, uint32_t from, uint32_t to, int64_t value, uint64_t lastSyncToken) { - AttributeUtils::fillAttribute(attr, from, to, value, lastSyncToken); + AttributeUtils::fillAttribute(*attr, from, to, value, lastSyncToken); } const std::shared_ptr<IDestructorCallback> emptyCallback; @@ -555,8 +556,7 @@ public: attr.fastaccess = fastAccess; _builder.attribute.push_back(attr); } - AttributeCollectionSpec::UP create(uint32_t docIdLimit, - search::SerialNum serialNum) { + std::unique_ptr<AttributeCollectionSpec> create(uint32_t docIdLimit, search::SerialNum serialNum) { return _factory.create(_builder, docIdLimit, serialNum); } }; @@ -573,7 +573,7 @@ struct FastAccessAttributeCollectionSpecTest : public AttributeCollectionSpecTes TEST_F(NormalAttributeCollectionSpecTest, spec_can_be_created) { - AttributeCollectionSpec::UP spec = create(10, 20); + auto spec = create(10, 20); EXPECT_EQ(2u, spec->getAttributes().size()); EXPECT_EQ("a1", spec->getAttributes()[0].getName()); EXPECT_EQ("a2", spec->getAttributes()[1].getName()); @@ -583,7 +583,7 @@ TEST_F(NormalAttributeCollectionSpecTest, spec_can_be_created) TEST_F(FastAccessAttributeCollectionSpecTest, spec_can_be_created) { - AttributeCollectionSpec::UP spec = create(10, 20); + auto spec = create(10, 20); EXPECT_EQ(1u, spec->getAttributes().size()); EXPECT_EQ("a2", spec->getAttributes()[0].getName()); EXPECT_EQ(10u, spec->getDocIdLimit()); @@ -594,13 +594,13 @@ const FilterAttributeManager::AttributeSet ACCEPTED_ATTRIBUTES = {"a2"}; class FilterAttributeManagerTest : public ::testing::Test { public: - DirectoryHandler _dirHandler; - DummyFileHeaderContext _fileHeaderContext; - ForegroundTaskExecutor _attributeFieldWriter; - ForegroundThreadExecutor _shared; - HwInfo _hwInfo; + DirectoryHandler _dirHandler; + DummyFileHeaderContext _fileHeaderContext; + ForegroundTaskExecutor _attributeFieldWriter; + ForegroundThreadExecutor _shared; + HwInfo _hwInfo; proton::AttributeManager::SP _baseMgr; - FilterAttributeManager _filterMgr; + FilterAttributeManager _filterMgr; FilterAttributeManagerTest() : _dirHandler(test_dir), diff --git a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp index 0dab0710209..ab7a0db5e7b 100644 --- a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp +++ b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp @@ -13,6 +13,7 @@ #include <vespa/searchlib/common/indexmetainfo.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/test/directory_handler.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/datastore/datastorebase.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp index 1209ab3e578..a7d41b6234c 100644 --- a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp @@ -8,6 +8,7 @@ #include <vespa/searchlib/attribute/interlock.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/test/directory_handler.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/foreground_thread_executor.h> diff --git a/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp b/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp index 2815bd7d88e..b08764289e6 100644 --- a/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp +++ b/searchcore/src/tests/proton/attribute/document_field_populator/document_field_populator_test.cpp @@ -1,11 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchcommon/common/schema.h> #include <vespa/searchcore/proton/attribute/document_field_populator.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/integerbase.h> #include <vespa/searchlib/index/docbuilder.h> +#include <vespa/searchcommon/common/schema.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> diff --git a/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp b/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp index 996ddac2d2a..c52978261a7 100644 --- a/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp +++ b/searchcore/src/tests/proton/attribute/imported_attributes_context/imported_attributes_context_test.cpp @@ -12,6 +12,7 @@ #include <vespa/searchlib/attribute/imported_attribute_vector_factory.h> #include <vespa/searchlib/attribute/reference_attribute.h> #include <vespa/searchlib/test/mock_gid_to_lid_mapping.h> +#include <vespa/searchcommon/attribute/config.h> #include <future> #include <vespa/log/log.h> @@ -30,7 +31,7 @@ using generation_t = AttributeVector::generation_t; std::shared_ptr<ReferenceAttribute> createReferenceAttribute(const vespalib::string &name) { - auto refAttr = std::make_shared<ReferenceAttribute>(name, Config(BasicType::REFERENCE)); + auto refAttr = std::make_shared<ReferenceAttribute>(name); refAttr->setGidToLidMapperFactory(std::make_shared<MockGidToLidMapperFactory>()); return refAttr; } diff --git a/searchcore/src/tests/proton/common/alloc_config/alloc_config_test.cpp b/searchcore/src/tests/proton/common/alloc_config/alloc_config_test.cpp index 59503464222..a27c66cfaa2 100644 --- a/searchcore/src/tests/proton/common/alloc_config/alloc_config_test.cpp +++ b/searchcore/src/tests/proton/common/alloc_config/alloc_config_test.cpp @@ -15,7 +15,7 @@ namespace { CompactionStrategy baseline_compaction_strategy(0.2, 0.25); GrowStrategy make_grow_strategy(uint32_t initial_docs) { - return GrowStrategy(initial_docs, 0.1, 1, 0.15); + return GrowStrategy(initial_docs, 0.1, 1, initial_docs, 0.15); } AllocStrategy make_alloc_strategy(uint32_t initial_docs) { diff --git a/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp b/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp index 8fa5c2994b0..2d7e242b9fb 100644 --- a/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp +++ b/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp @@ -1,5 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/searchcore/proton/common/attribute_updater.h> +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/attribute/reference_attribute.h> +#include <vespa/searchlib/tensor/dense_tensor_attribute.h> +#include <vespa/searchlib/tensor/serialized_fast_value_attribute.h> +#include <vespa/searchlib/test/weighted_type_test_utils.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/document/base/testdocrepo.h> #include <vespa/document/datatype/tensor_data_type.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> @@ -26,12 +33,6 @@ #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/value_codec.h> -#include <vespa/searchcore/proton/common/attribute_updater.h> -#include <vespa/searchlib/attribute/attributefactory.h> -#include <vespa/searchlib/attribute/reference_attribute.h> -#include <vespa/searchlib/tensor/dense_tensor_attribute.h> -#include <vespa/searchlib/tensor/serialized_fast_value_attribute.h> -#include <vespa/searchlib/test/weighted_type_test_utils.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchcore/src/tests/proton/common/selectpruner_test.cpp b/searchcore/src/tests/proton/common/selectpruner_test.cpp index c8c474e21bc..3cb8389a4ea 100644 --- a/searchcore/src/tests/proton/common/selectpruner_test.cpp +++ b/searchcore/src/tests/proton/common/selectpruner_test.cpp @@ -1,16 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/document/config/documenttypes_config_fwd.h> + +#include <vespa/searchcore/proton/common/selectpruner.h> +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/test/mock_attribute_manager.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/datatype/documenttype.h> -#include <vespa/searchcore/proton/common/selectpruner.h> #include <vespa/document/select/parser.h> #include <vespa/document/select/cloningvisitor.h> #include <vespa/document/fieldvalue/document.h> -#include <vespa/searchlib/attribute/attributefactory.h> -#include <vespa/searchlib/test/mock_attribute_manager.h> +#include <vespa/vespalib/testkit/testapp.h> #include <vespa/log/log.h> LOG_SETUP("selectpruner_test"); diff --git a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp index a460c65fbb2..f89af9a11a6 100644 --- a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp +++ b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp @@ -1,5 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/searchcore/proton/common/attribute_updater.h> +#include <vespa/searchcore/proton/common/pendinglidtracker.h> +#include <vespa/searchcore/proton/persistenceengine/document_iterator.h> +#include <vespa/searchcore/proton/persistenceengine/commit_and_wait_document_retriever.h> +#include <vespa/searchlib/attribute/attributecontext.h> +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/test/mock_attribute_manager.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/document/datatype/documenttype.h> @@ -8,13 +16,6 @@ #include <vespa/persistence/spi/docentry.h> #include <vespa/persistence/spi/result.h> #include <vespa/persistence/spi/test.h> -#include <vespa/searchcore/proton/common/attribute_updater.h> -#include <vespa/searchcore/proton/common/pendinglidtracker.h> -#include <vespa/searchcore/proton/persistenceengine/document_iterator.h> -#include <vespa/searchcore/proton/persistenceengine/commit_and_wait_document_retriever.h> -#include <vespa/searchlib/attribute/attributecontext.h> -#include <vespa/searchlib/attribute/attributefactory.h> -#include <vespa/searchlib/test/mock_attribute_manager.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/testkit/test_kit.h> #include <unordered_set> diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index d4e337f2717..948c9cfe25e 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -455,11 +455,11 @@ asAttributeManager(const proton::IAttributeManager::SP &attrMgr) TEST_F("require that we can reconfigure attribute manager", Fixture) { ViewPtrs o = f._views.getViewPtrs(); - AttributeCollectionSpec::AttributeList specList; - AttributeCollectionSpec spec(specList, 1, 0); ReconfigParams params(CCR().setAttributesChanged(true).setSchemaChanged(true)); // Use new config snapshot == old config snapshot (only relevant for reprocessing) - f._configurer->reconfigure(*createConfig(), *createConfig(), spec, params, f._resolver); + f._configurer->reconfigure(*createConfig(), *createConfig(), + AttributeCollectionSpec(AttributeCollectionSpec::AttributeList(), 1, 0), + params, f._resolver); ViewPtrs n = f._views.getViewPtrs(); { // verify search view @@ -494,11 +494,11 @@ void checkAttributeWriterChangeOnRepoChange(Fixture &f, bool docTypeRepoChanged) { auto oldAttributeWriter = getAttributeWriter(f); - AttributeCollectionSpec::AttributeList specList; - AttributeCollectionSpec spec(specList, 1, 0); ReconfigParams params(CCR().setDocumentTypeRepoChanged(docTypeRepoChanged)); // Use new config snapshot == old config snapshot (only relevant for reprocessing) - f._configurer->reconfigure(*createConfig(), *createConfig(), spec, params, f._resolver); + f._configurer->reconfigure(*createConfig(), *createConfig(), + AttributeCollectionSpec(AttributeCollectionSpec::AttributeList(), 1, 0), + params, f._resolver); auto newAttributeWriter = getAttributeWriter(f); if (docTypeRepoChanged) { EXPECT_NOT_EQUAL(oldAttributeWriter, newAttributeWriter); @@ -515,11 +515,11 @@ TEST_F("require that we get new attribute writer if document type repo changes", TEST_F("require that reconfigure returns reprocessing initializer when changing attributes", Fixture) { - AttributeCollectionSpec::AttributeList specList; - AttributeCollectionSpec spec(specList, 1, 0); ReconfigParams params(CCR().setAttributesChanged(true).setSchemaChanged(true)); IReprocessingInitializer::UP init = - f._configurer->reconfigure(*createConfig(), *createConfig(), spec, params, f._resolver); + f._configurer->reconfigure(*createConfig(), *createConfig(), + AttributeCollectionSpec(AttributeCollectionSpec::AttributeList(), 1, 0), + params, f._resolver); EXPECT_TRUE(init.get() != nullptr); EXPECT_TRUE((dynamic_cast<AttributeReprocessingInitializer *>(init.get())) != nullptr); @@ -528,10 +528,9 @@ TEST_F("require that reconfigure returns reprocessing initializer when changing TEST_F("require that we can reconfigure attribute writer", FastAccessFixture) { - AttributeCollectionSpec::AttributeList specList; - AttributeCollectionSpec spec(specList, 1, 0); FastAccessFeedView::SP o = f._view._feedView.get(); - f._configurer.reconfigure(*createConfig(), *createConfig(), spec); + f._configurer.reconfigure(*createConfig(), *createConfig(), + AttributeCollectionSpec(AttributeCollectionSpec::AttributeList(), 1, 0)); FastAccessFeedView::SP n = f._view._feedView.get(); FastAccessFeedViewComparer cmp(o, n); @@ -543,10 +542,8 @@ TEST_F("require that we can reconfigure attribute writer", FastAccessFixture) TEST_F("require that reconfigure returns reprocessing initializer", FastAccessFixture) { - AttributeCollectionSpec::AttributeList specList; - AttributeCollectionSpec spec(specList, 1, 0); - IReprocessingInitializer::UP init = - f._configurer.reconfigure(*createConfig(), *createConfig(), spec); + IReprocessingInitializer::UP init = f._configurer.reconfigure(*createConfig(), *createConfig(), + AttributeCollectionSpec(AttributeCollectionSpec::AttributeList(), 1, 0)); EXPECT_TRUE(init.get() != nullptr); EXPECT_TRUE((dynamic_cast<AttributeReprocessingInitializer *>(init.get())) != nullptr); diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index 0a49b15d001..cb3acdb7ec8 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -1,8 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/config-bucketspaces.h> -#include <vespa/config/subscription/sourcespec.h> -#include <vespa/document/test/make_bucket_space.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h> #include <vespa/searchcore/proton/common/hw_info.h> @@ -24,12 +21,17 @@ #include <vespa/searchcore/proton/server/minimal_document_retriever.h> #include <vespa/searchcore/proton/server/reconfig_params.h> #include <vespa/searchcore/proton/server/searchabledocsubdb.h> +#include <vespa/searchcore/proton/server/documentdbconfigmanager.h> #include <vespa/searchcore/proton/test/test.h> #include <vespa/searchcore/proton/test/thread_utils.h> #include <vespa/searchcore/proton/test/transport_helper.h> #include <vespa/searchlib/attribute/interlock.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/test/directory_handler.h> +#include <vespa/searchcommon/attribute/config.h> +#include <vespa/config-bucketspaces.h> +#include <vespa/config/subscription/sourcespec.h> +#include <vespa/document/test/make_bucket_space.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/destructor_callbacks.h> diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index 5018b203216..b326db47b5c 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -21,10 +21,12 @@ #include <vespa/searchcore/proton/test/threading_service_observer.h> #include <vespa/searchcore/proton/test/transport_helper.h> #include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/index/docbuilder.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/document/update/documentupdate.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/util/destructor_callbacks.h> -#include <vespa/searchlib/index/docbuilder.h> +#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/log/log.h> LOG_SETUP("feedview_test"); diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp index 8be44f0fd09..d5acd5d40bf 100644 --- a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp +++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp @@ -12,6 +12,7 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/searchcore/proton/server/bootstrapconfig.h> #include <vespa/searchcore/proton/server/fileconfigmanager.h> +#include <vespa/searchcore/proton/server/documentdbconfigmanager.h> #include <vespa/searchcore/proton/test/documentdb_config_builder.h> #include <vespa/searchcore/proton/test/transport_helper.h> #include <vespa/searchsummary/config/config-juniperrc.h> diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp index 73311c110a9..9a2655cdd8a 100644 --- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp @@ -11,6 +11,7 @@ #include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h> #include <vespa/searchcore/proton/server/itlssyncer.h> #include <vespa/searchlib/attribute/attributefilesavetarget.h> +#include <vespa/searchlib/attribute/search_context.h> #include <vespa/searchlib/common/flush_token.h> #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/searchlib/fef/matchdatalayout.h> diff --git a/searchcore/src/tests/proton/index/fusionrunner_test.cpp b/searchcore/src/tests/proton/index/fusionrunner_test.cpp index 1b04415b78f..4a84e3972ad 100644 --- a/searchcore/src/tests/proton/index/fusionrunner_test.cpp +++ b/searchcore/src/tests/proton/index/fusionrunner_test.cpp @@ -1,6 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/file.h> #include <vespa/searchcore/proton/index/indexmanager.h> #include <vespa/searchcore/proton/test/transport_helper.h> #include <vespa/searchcorespi/index/fusionrunner.h> @@ -14,11 +13,13 @@ #include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/test/index/mock_field_length_inspector.h> +#include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/vespalib/util/gate.h> #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/searchlib/queryeval/fake_requestcontext.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/fastos/file.h> #include <set> using document::Document; diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index 6cca28f5b43..60864019419 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -1,6 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/file.h> #include <vespa/searchcore/proton/index/indexmanager.h> #include <vespa/searchcore/proton/test/transport_helper.h> #include <vespa/searchcorespi/index/index_manager_stats.h> @@ -25,6 +24,8 @@ #include <vespa/vespalib/util/gate.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/fastos/file.h> #include <set> #include <thread> diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index e3d0b37c0d6..36b79c7bb8e 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -1,10 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/document/base/globalid.h> -#include <vespa/eval/eval/simple_value.h> -#include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/eval/value_codec.h> -#include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include <vespa/searchcore/proton/matching/fakesearchcontext.h> @@ -34,10 +29,16 @@ #include <vespa/searchlib/query/tree/stackdumpcreator.h> #include <vespa/searchlib/queryeval/isourceselector.h> #include <vespa/searchlib/test/mock_attribute_context.h> +#include <vespa/searchcommon/attribute/iattributecontext.h> +#include <vespa/document/base/globalid.h> +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/value_codec.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/simple_thread_bundle.h> #include <vespa/vespalib/util/testclock.h> +#include <vespa/vespalib/stllike/asciistream.h> #include <initializer_list> #include <vespa/log/log.h> diff --git a/searchcore/src/tests/proton/matching/query_test.cpp b/searchcore/src/tests/proton/matching/query_test.cpp index 0acd403d73a..eacb4108686 100644 --- a/searchcore/src/tests/proton/matching/query_test.cpp +++ b/searchcore/src/tests/proton/matching/query_test.cpp @@ -27,8 +27,8 @@ #include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/queryeval/termasstring.h> #include <vespa/searchlib/parsequery/stackdumpiterator.h> - #include <vespa/document/datatype/positiondatatype.h> +#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/log/log.h> diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index 4022bd68c92..f6c1518ba90 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -404,6 +404,10 @@ TEST_FF("require that docstore config computes cachesize automatically if unset" EXPECT_EQUAL(500000ul, config->getStoreConfig().getMaxCacheBytes()); } +GrowStrategy +growStrategy(uint32_t initial) { + return GrowStrategy(initial, 0.1, 1, initial, 0.15); +} TEST_FF("require that allocation config is propagated", ConfigTestFixture("test"), DocumentDBConfigManager(f1.configId + "/test", "test")) @@ -424,9 +428,9 @@ TEST_FF("require that allocation config is propagated", auto config = getDocumentDBConfig(f1, f2); { auto& alloc_config = config->get_alloc_config(); - EXPECT_EQUAL(AllocStrategy(GrowStrategy(20000000, 0.1, 1, 0.15), CompactionStrategy(0.25, 0.3), 10000), alloc_config.make_alloc_strategy(SubDbType::READY)); - EXPECT_EQUAL(AllocStrategy(GrowStrategy(100000, 0.1, 1, 0.15), CompactionStrategy(0.25, 0.3), 10000), alloc_config.make_alloc_strategy(SubDbType::REMOVED)); - EXPECT_EQUAL(AllocStrategy(GrowStrategy(30000000, 0.1, 1, 0.15), CompactionStrategy(0.25, 0.3), 10000), alloc_config.make_alloc_strategy(SubDbType::NOTREADY)); + EXPECT_EQUAL(AllocStrategy(growStrategy(20000000), CompactionStrategy(0.25, 0.3), 10000), alloc_config.make_alloc_strategy(SubDbType::READY)); + EXPECT_EQUAL(AllocStrategy(growStrategy(100000), CompactionStrategy(0.25, 0.3), 10000), alloc_config.make_alloc_strategy(SubDbType::REMOVED)); + EXPECT_EQUAL(AllocStrategy(growStrategy(30000000), CompactionStrategy(0.25, 0.3), 10000), alloc_config.make_alloc_strategy(SubDbType::NOTREADY)); } } diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp index 8a12219de3c..3aa014ca8ac 100644 --- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp +++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp @@ -106,8 +106,8 @@ struct DBConfigFixture { buildSchema(), std::make_shared<DocumentDBMaintenanceConfig>(), search::LogDocumentStore::Config(), - std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make()), - std::make_shared<const AllocConfig>(), + ThreadingServiceConfig::make(), + AllocConfig::makeDefault(), configId, docTypeName); } diff --git a/searchcore/src/tests/proton/reference/document_db_reference/document_db_reference_test.cpp b/searchcore/src/tests/proton/reference/document_db_reference/document_db_reference_test.cpp index aba27827d56..526b0837212 100644 --- a/searchcore/src/tests/proton/reference/document_db_reference/document_db_reference_test.cpp +++ b/searchcore/src/tests/proton/reference/document_db_reference/document_db_reference_test.cpp @@ -1,11 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> + #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/reference/document_db_reference.h> #include <vespa/searchcore/proton/test/mock_attribute_manager.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/imported_attribute_vector.h> - +#include <vespa/searchcommon/attribute/config.h> +#include <vespa/vespalib/testkit/testapp.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP("document_db_reference_test"); diff --git a/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp b/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp index 061a3fd4c32..edd987ce841 100644 --- a/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp +++ b/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp @@ -1,10 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/config-imported-fields.h> -#include <vespa/document/datatype/documenttype.h> -#include <vespa/document/datatype/referencedatatype.h> -#include <vespa/log/log.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/reference/document_db_reference_resolver.h> #include <vespa/searchcore/proton/reference/gid_to_lid_change_listener.h> @@ -18,11 +13,17 @@ #include <vespa/searchlib/attribute/reference_attribute.h> #include <vespa/searchlib/common/i_gid_to_lid_mapper.h> #include <vespa/searchlib/common/i_gid_to_lid_mapper_factory.h> +#include <vespa/searchlib/test/mock_attribute_manager.h> +#include <vespa/searchcommon/attribute/config.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/datatype/referencedatatype.h> #include <vespa/vespalib/util/monitored_refcount.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> -#include <vespa/searchlib/test/mock_attribute_manager.h> #include <vespa/vespalib/test/insertion_operators.h> +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/config-imported-fields.h> +#include <vespa/log/log.h> LOG_SETUP("document_db_reference_resolver_test"); using namespace document; @@ -120,7 +121,7 @@ struct MyAttributeManager : public MockAttributeManager { addAttribute(name, AttributeFactory::createAttribute(name, Config(BasicType::INT32))); } void addReferenceAttribute(const vespalib::string &name) { - addAttribute(name, std::make_shared<ReferenceAttribute>(name, Config(BasicType::REFERENCE))); + addAttribute(name, std::make_shared<ReferenceAttribute>(name)); } const ReferenceAttribute *getReferenceAttribute(const vespalib::string &name) const { AttributeGuard::UP guard = getAttribute(name); diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp index e3b1fd3aa15..dd33e34efdd 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp @@ -57,7 +57,7 @@ struct Fixture std::unique_ptr<GidToLidChangeListener> _listener; Fixture() - : _attr(std::make_shared<ReferenceAttribute>("test", Config(BasicType::REFERENCE))), + : _attr(std::make_shared<ReferenceAttribute>("test")), _writer(vespalib::SequencedTaskExecutor::create(test_executor, 1)), _refCount(), _listener() diff --git a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/CMakeLists.txt b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/CMakeLists.txt index 99b7ad3821d..70906aa92f1 100644 --- a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/CMakeLists.txt +++ b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/CMakeLists.txt @@ -6,6 +6,7 @@ vespa_add_executable(searchcore_attribute_reprocessing_initializer_test_app TEST searchcore_reprocessing searchcore_attribute searchcore_pcommon + searchcore_test GTest::GTest ) vespa_add_test(NAME searchcore_attribute_reprocessing_initializer_test_app COMMAND searchcore_attribute_reprocessing_initializer_test_app) diff --git a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp index c2664d4e143..ebec0841ebf 100644 --- a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp +++ b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp @@ -14,6 +14,8 @@ #include <vespa/searchlib/attribute/interlock.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/test/directory_handler.h> +#include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/util/foreground_thread_executor.h> diff --git a/searchcore/src/tests/proton/server/documentretriever_test.cpp b/searchcore/src/tests/proton/server/documentretriever_test.cpp index f5c163fefb5..bf277f86367 100644 --- a/searchcore/src/tests/proton/server/documentretriever_test.cpp +++ b/searchcore/src/tests/proton/server/documentretriever_test.cpp @@ -1,6 +1,21 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for documentretriever. +#include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h> +#include <vespa/searchcore/proton/server/documentretriever.h> +#include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> +#include <vespa/searchcore/proton/test/dummy_document_store.h> +#include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchlib/attribute/attributeguard.h> +#include <vespa/searchlib/attribute/attributemanager.h> +#include <vespa/searchlib/attribute/floatbase.h> +#include <vespa/searchlib/attribute/integerbase.h> +#include <vespa/searchlib/attribute/predicate_attribute.h> +#include <vespa/searchlib/attribute/stringbase.h> +#include <vespa/searchlib/predicate/predicate_index.h> +#include <vespa/searchlib/tensor/tensor_attribute.h> +#include <vespa/searchcommon/attribute/config.h> +#include <vespa/searchcommon/common/schema.h> #include <vespa/document/base/documentid.h> #include <vespa/document/bucket/bucketid.h> #include <vespa/document/datatype/datatype.h> @@ -21,29 +36,15 @@ #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/test/fieldvalue_helpers.h> +#include <vespa/vespalib/geo/zcurve.h> +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value.h> #include <vespa/eval/eval/test/value_compare.h> #include <vespa/persistence/spi/bucket.h> #include <vespa/persistence/spi/test.h> -#include <vespa/searchcommon/common/schema.h> -#include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h> -#include <vespa/searchcore/proton/server/documentretriever.h> -#include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> -#include <vespa/searchcore/proton/test/dummy_document_store.h> -#include <vespa/searchlib/attribute/attributefactory.h> -#include <vespa/searchlib/attribute/attributeguard.h> -#include <vespa/searchlib/attribute/attributemanager.h> -#include <vespa/searchlib/attribute/floatbase.h> -#include <vespa/searchlib/attribute/integerbase.h> -#include <vespa/searchlib/attribute/predicate_attribute.h> -#include <vespa/searchlib/attribute/stringbase.h> -#include <vespa/searchlib/predicate/predicate_index.h> -#include <vespa/searchlib/tensor/tensor_attribute.h> -#include <vespa/vespalib/geo/zcurve.h> -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> diff --git a/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp b/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp index 14c96faef42..981106f018e 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp +++ b/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp @@ -202,8 +202,8 @@ std::shared_ptr<DocumentDBConfig> make_document_db_config(std::shared_ptr<Docume schema, std::make_shared<proton::DocumentDBMaintenanceConfig>(), search::LogDocumentStore::Config(), - std::make_shared<const proton::ThreadingServiceConfig>(proton::ThreadingServiceConfig::make()), - std::make_shared<const proton::AllocConfig>(), + proton::ThreadingServiceConfig::make(), + proton::AllocConfig::makeDefault(), "client", doc_type_name.getName()); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp index 78603512d6b..6e935a577fd 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_aspect_delayer.cpp @@ -9,6 +9,7 @@ #include <vespa/searchcore/proton/common/i_document_type_inspector.h> #include <vespa/searchcore/proton/common/i_indexschema_inspector.h> #include <vespa/searchlib/attribute/configconverter.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/stllike/hash_set.hpp> using search::attribute::isUpdateableInMemoryOnly; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec.cpp index 17707c99705..35c54d31fd0 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec.cpp @@ -4,15 +4,23 @@ namespace proton { -AttributeCollectionSpec::AttributeCollectionSpec(const AttributeList &attributes, - uint32_t docIdLimit, - SerialNum currentSerialNum) - : _attributes(attributes), +AttributeCollectionSpec::AttributeCollectionSpec(AttributeList && attributes, uint32_t docIdLimit, SerialNum currentSerialNum) + : _attributes(std::move(attributes)), _docIdLimit(docIdLimit), _currentSerialNum(currentSerialNum) { } -AttributeCollectionSpec::~AttributeCollectionSpec() { } +AttributeCollectionSpec::~AttributeCollectionSpec() = default; + +bool +AttributeCollectionSpec::hasAttribute(const vespalib::string &name) const { + for (const auto &attr : _attributes) { + if (attr.getName() == name) { + return true; + } + } + return false; +} } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec.h index 607be32e952..73f1febd822 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec.h @@ -4,7 +4,6 @@ #include "attribute_spec.h" #include <vespa/searchlib/common/serialnum.h> -#include <memory> #include <vector> namespace proton { @@ -15,9 +14,7 @@ namespace proton { class AttributeCollectionSpec { public: - typedef std::unique_ptr<AttributeCollectionSpec> UP; - - typedef std::vector<AttributeSpec> AttributeList; + using AttributeList = std::vector<AttributeSpec>; private: typedef search::SerialNum SerialNum; @@ -27,27 +24,21 @@ private: SerialNum _currentSerialNum; public: - AttributeCollectionSpec(const AttributeList &attributes, - uint32_t docIdLimit, - SerialNum currentSerialNum); + AttributeCollectionSpec(AttributeList && attributes, uint32_t docIdLimit, SerialNum currentSerialNum); ~AttributeCollectionSpec(); const AttributeList &getAttributes() const { return _attributes; } + AttributeList stealAttributes() { + return std::move(_attributes); + } uint32_t getDocIdLimit() const { return _docIdLimit; } SerialNum getCurrentSerialNum() const { return _currentSerialNum; } - bool hasAttribute(const vespalib::string &name) const { - for (const auto &attr : _attributes) { - if (attr.getName() == name) { - return true; - } - } - return false; - } + bool hasAttribute(const vespalib::string &name) const; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp index da1b733ca0a..663491e9c64 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp @@ -2,15 +2,14 @@ #include "attribute_collection_spec_factory.h" #include <vespa/searchlib/attribute/configconverter.h> +#include <vespa/searchcommon/attribute/config.h> using search::attribute::ConfigConverter; using search::GrowStrategy; namespace proton { -AttributeCollectionSpecFactory::AttributeCollectionSpecFactory( - const AllocStrategy &alloc_strategy, - bool fastAccessOnly) +AttributeCollectionSpecFactory::AttributeCollectionSpecFactory(const AllocStrategy &alloc_strategy, bool fastAccessOnly) : _alloc_strategy(alloc_strategy), _fastAccessOnly(fastAccessOnly) { @@ -18,7 +17,7 @@ AttributeCollectionSpecFactory::AttributeCollectionSpecFactory( AttributeCollectionSpecFactory::~AttributeCollectionSpecFactory() = default; -AttributeCollectionSpec::UP +std::unique_ptr<AttributeCollectionSpec> AttributeCollectionSpecFactory::create(const AttributesConfig &attrCfg, uint32_t docIdLimit, search::SerialNum serialNum) const @@ -27,18 +26,18 @@ AttributeCollectionSpecFactory::create(const AttributesConfig &attrCfg, // Amortize memory spike cost over N docs const size_t skew = _alloc_strategy.get_amortize_count()/(attrCfg.attribute.size()+1); GrowStrategy grow = _alloc_strategy.get_grow_strategy(); - grow.setDocsInitialCapacity(std::max(grow.getDocsInitialCapacity(),docIdLimit)); + grow.setInitialCapacity(std::max(grow.getInitialCapacity(), size_t(docIdLimit))); for (const auto &attr : attrCfg.attribute) { search::attribute::Config cfg = ConfigConverter::convert(attr); if (_fastAccessOnly && !cfg.fastAccess()) { continue; } - grow.setDocsGrowDelta(grow.getDocsGrowDelta() + skew); + grow.setGrowDelta(grow.getGrowDelta() + skew); cfg.setGrowStrategy(grow); cfg.setCompactionStrategy(_alloc_strategy.get_compaction_strategy()); attrs.push_back(AttributeSpec(attr.name, cfg)); } - return std::make_unique<AttributeCollectionSpec>(attrs, docIdLimit, serialNum); + return std::make_unique<AttributeCollectionSpec>(std::move(attrs), docIdLimit, serialNum); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h index 3565b533e02..8ef06f20bac 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h @@ -3,10 +3,10 @@ #pragma once #include "attribute_collection_spec.h" -#include <vespa/searchcommon/attribute/config.h> #include <vespa/searchcore/proton/common/alloc_strategy.h> #include <vespa/searchlib/common/serialnum.h> -#include <vespa/config-attributes.h> + +namespace vespa::config::search::internal { class InternalAttributesType; }; namespace proton { @@ -17,19 +17,16 @@ namespace proton { class AttributeCollectionSpecFactory { private: - typedef vespa::config::search::AttributesConfig AttributesConfig; + using AttributesConfig = const vespa::config::search::internal::InternalAttributesType; const AllocStrategy _alloc_strategy; const bool _fastAccessOnly; public: - AttributeCollectionSpecFactory(const AllocStrategy& alloc_strategy, - bool fastAccessOnly); + AttributeCollectionSpecFactory(const AllocStrategy& alloc_strategy, bool fastAccessOnly); ~AttributeCollectionSpecFactory(); - AttributeCollectionSpec::UP create(const AttributesConfig &attrCfg, - uint32_t docIdLimit, - search::SerialNum serialNum) const; + std::unique_ptr<AttributeCollectionSpec> create(const AttributesConfig &attrCfg, uint32_t docIdLimit, search::SerialNum serialNum) const; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.cpp index f4692013cdf..444b8973a9c 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.cpp @@ -2,10 +2,12 @@ #include "attribute_config_inspector.h" #include <vespa/searchlib/attribute/configconverter.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/config-attributes.h> #include <vespa/vespalib/stllike/hash_map.hpp> using search::attribute::ConfigConverter; +using search::attribute::Config; namespace proton { @@ -13,18 +15,18 @@ AttributeConfigInspector::AttributeConfigInspector(const AttributesConfig& confi : _hash() { for (auto& attr : config.attribute) { - auto res = _hash.insert(std::make_pair(attr.name, ConfigConverter::convert(attr))); + auto res = _hash.insert(std::make_pair(attr.name, std::make_unique<Config>(ConfigConverter::convert(attr)))); assert(res.second); } } AttributeConfigInspector::~AttributeConfigInspector() = default; -const search::attribute::Config* +const Config* AttributeConfigInspector::get_config(const vespalib::string& name) const { auto itr = _hash.find(name); - return (itr != _hash.end()) ? &itr->second : nullptr; + return (itr != _hash.end()) ? itr->second.get() : nullptr; } } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.h index 42c920fcc5f..23c0fb3b685 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_config_inspector.h @@ -4,9 +4,9 @@ #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/hash_map.h> -#include <vespa/searchcommon/attribute/config.h> namespace vespa::config::search::internal { class InternalAttributesType; } +namespace search::attribute { class Config; } namespace proton { @@ -15,12 +15,13 @@ namespace proton { * from config server. */ class AttributeConfigInspector { - vespalib::hash_map<vespalib::string, search::attribute::Config> _hash; + using Config = search::attribute::Config; + vespalib::hash_map<vespalib::string, std::unique_ptr<Config>> _hash; public: using AttributesConfig = const vespa::config::search::internal::InternalAttributesType; AttributeConfigInspector(const AttributesConfig& config); ~AttributeConfigInspector(); - const search::attribute::Config* get_config(const vespalib::string& name) const; + const Config* get_config(const vespalib::string& name) const; }; } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp index 713582bf1ce..19e387a2b64 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp @@ -9,6 +9,7 @@ #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/searchcommon/attribute/persistent_predicate_params.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/util/fileutil.h> #include <vespa/searchlib/attribute/attribute_header.h> #include <vespa/searchlib/attribute/attributevector.h> @@ -232,13 +233,13 @@ AttributeInitializer::createAndSetupEmptyAttribute() const AttributeInitializer::AttributeInitializer(const std::shared_ptr<AttributeDirectory> &attrDir, const vespalib::string &documentSubDbName, - const AttributeSpec &spec, + AttributeSpec && spec, uint64_t currentSerialNum, const IAttributeFactory &factory, vespalib::Executor& shared_executor) : _attrDir(attrDir), _documentSubDbName(documentSubDbName), - _spec(spec), + _spec(std::move(spec)), _currentSerialNum(currentSerialNum), _factory(factory), _shared_executor(shared_executor), diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h index 78a798c929e..a3844219d24 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.h @@ -47,7 +47,7 @@ private: public: AttributeInitializer(const std::shared_ptr<AttributeDirectory> &attrDir, const vespalib::string &documentSubDbName, - const AttributeSpec &spec, uint64_t currentSerialNum, const IAttributeFactory &factory, + AttributeSpec && spec, uint64_t currentSerialNum, const IAttributeFactory &factory, vespalib::Executor& shared_executor); ~AttributeInitializer(); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp index d8c2d4e6e72..fb99ca51e3a 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp @@ -132,7 +132,7 @@ AttributeInitializerTasksBuilder::add(AttributeInitializer::UP initializer) { } -AttributeCollectionSpec::UP +std::unique_ptr<AttributeCollectionSpec> AttributeManagerInitializer::createAttributeSpec() const { uint32_t docIdLimit = 1; // The real docIdLimit is used after attributes are loaded to pad them @@ -161,8 +161,8 @@ AttributeManagerInitializer::AttributeManagerInitializer(SerialNum configSerialN { addDependency(documentMetaStoreInitTask); AttributeInitializerTasksBuilder tasksBuilder(*this, documentMetaStoreInitTask, documentMetaStore, _attributesResult); - AttributeCollectionSpec::UP attrSpec = createAttributeSpec(); - _attrMgr = std::make_shared<AttributeManager>(*baseAttrMgr, *attrSpec, tasksBuilder); + std::unique_ptr<AttributeCollectionSpec> attrSpec = createAttributeSpec(); + _attrMgr = std::make_shared<AttributeManager>(*baseAttrMgr, std::move(*attrSpec), tasksBuilder); } AttributeManagerInitializer::~AttributeManagerInitializer() = default; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h index cd2e044dee6..99b6b026f8f 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h @@ -20,17 +20,17 @@ namespace proton { class AttributeManagerInitializer : public initializer::InitializerTask { private: - search::SerialNum _configSerialNum; - DocumentMetaStore::SP _documentMetaStore; - AttributeManager::SP _attrMgr; + search::SerialNum _configSerialNum; + DocumentMetaStore::SP _documentMetaStore; + AttributeManager::SP _attrMgr; vespa::config::search::AttributesConfig _attrCfg; - AllocStrategy _alloc_strategy; - bool _fastAccessAttributesOnly; - searchcorespi::index::IThreadService &_master; - InitializedAttributesResult _attributesResult; - std::shared_ptr<AttributeManager::SP> _attrMgrResult; + AllocStrategy _alloc_strategy; + bool _fastAccessAttributesOnly; + searchcorespi::index::IThreadService &_master; + InitializedAttributesResult _attributesResult; + std::shared_ptr<AttributeManager::SP> _attrMgrResult; - AttributeCollectionSpec::UP createAttributeSpec() const; + std::unique_ptr<AttributeCollectionSpec> createAttributeSpec() const; public: AttributeManagerInitializer(search::SerialNum configSerialNum, @@ -44,7 +44,7 @@ public: std::shared_ptr<AttributeManager::SP> attrMgrResult); ~AttributeManagerInitializer() override; - virtual void run() override; + void run() override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp index 10e8947c38a..d47d2dd2877 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_populator.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/gate.h> #include <vespa/searchlib/attribute/attributevector.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".proton.attribute.attribute_populator"); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_spec.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_spec.cpp index ffbf5720b4b..11e03595caf 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_spec.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_spec.cpp @@ -1,21 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attribute_spec.h" +#include <vespa/searchcommon/attribute/config.h> namespace proton { -AttributeSpec::AttributeSpec(const vespalib::string &name, - const search::attribute::Config &cfg) +AttributeSpec::AttributeSpec(const vespalib::string &name, const Config &cfg) : _name(name), - _cfg(cfg) + _cfg(std::make_unique<Config>(cfg)) { } -AttributeSpec::AttributeSpec(const AttributeSpec &) = default; - -AttributeSpec & -AttributeSpec::operator=(const AttributeSpec &) = default; - AttributeSpec::AttributeSpec(AttributeSpec &&) noexcept = default; AttributeSpec & diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_spec.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_spec.h index d56510709b2..8b63837b69a 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_spec.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_spec.h @@ -2,7 +2,10 @@ #pragma once -#include <vespa/searchcommon/attribute/config.h> +#include <vespa/vespalib/stllike/string.h> +#include <memory> + +namespace search::attribute { class Config; } namespace proton { @@ -13,17 +16,16 @@ namespace proton { class AttributeSpec { private: + using Config = search::attribute::Config; vespalib::string _name; - search::attribute::Config _cfg; + std::unique_ptr<Config> _cfg; public: AttributeSpec(const vespalib::string &name, const search::attribute::Config &cfg); - AttributeSpec(const AttributeSpec &); - AttributeSpec & operator=(const AttributeSpec &); AttributeSpec(AttributeSpec &&) noexcept; AttributeSpec & operator=(AttributeSpec &&) noexcept; ~AttributeSpec(); const vespalib::string &getName() const { return _name; } - const search::attribute::Config &getConfig() const { return _cfg; } + const Config &getConfig() const { return *_cfg; } bool operator==(const AttributeSpec &rhs) const; }; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp index 5f1a59584e4..bb25b3da7be 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp @@ -9,6 +9,7 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/document/update/assignvalueupdate.h> #include <vespa/searchcommon/attribute/attribute_utils.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/common/attribute_updater.h> #include <vespa/searchlib/attribute/imported_attribute_vector.h> diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp index 92989da9b99..6fb0279733f 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp @@ -10,6 +10,7 @@ #include "imported_attributes_repo.h" #include "sequential_attributes_initializer.h" #include <vespa/searchcommon/attribute/i_attribute_functor.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h> #include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/searchlib/attribute/attributecontext.h> @@ -24,6 +25,7 @@ #include <vespa/vespalib/util/gate.h> #include <vespa/vespalib/util/isequencedtaskexecutor.h> #include <vespa/vespalib/util/threadexecutor.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".proton.attribute.attributemanager"); @@ -118,11 +120,12 @@ AttributeManager::FlushableWrap::FlushableWrap(FlushableAttributeSP flusher, Shr AttributeManager::FlushableWrap::~FlushableWrap() = default; AttributeVector::SP -AttributeManager::internalAddAttribute(const AttributeSpec &spec, +AttributeManager::internalAddAttribute(AttributeSpec && spec, uint64_t serialNum, const IAttributeFactory &factory) { - AttributeInitializer initializer(_diskLayout->createAttributeDir(spec.getName()), _documentSubDbName, spec, serialNum, factory, _shared_executor); + vespalib::string name = spec.getName(); + AttributeInitializer initializer(_diskLayout->createAttributeDir(name), _documentSubDbName, std::move(spec), serialNum, factory, _shared_executor); AttributeInitializerResult result = initializer.init(); if (result) { result.getAttribute()->setInterlock(_interlock); @@ -164,15 +167,15 @@ AttributeManager::findFlushable(const vespalib::string &name) const return (itr != _flushables.end()) ? &itr->second : nullptr; } -void +AttributeCollectionSpec::AttributeList AttributeManager::transferExistingAttributes(const AttributeManager &currMgr, - const Spec &newSpec, - Spec::AttributeList &toBeAdded) + Spec::AttributeList && newAttributes) { + Spec::AttributeList toBeAdded; vespalib::Gate gate; { auto gateCallback = std::make_shared<vespalib::GateCallback>(gate); - for (const auto &aspec: newSpec.getAttributes()) { + for (auto &aspec: newAttributes) { AttributeVector::SP av = currMgr.findAttribute(aspec.getName()); if (matchingTypes(av, aspec.getConfig())) { // transfer attribute LOG(debug, @@ -190,24 +193,25 @@ AttributeManager::transferExistingAttributes(const AttributeManager &currMgr, av->update_config(cfg); }); } else { - toBeAdded.push_back(aspec); + toBeAdded.push_back(std::move(aspec)); } } } gate.await(); + return toBeAdded; } void AttributeManager::addNewAttributes(const Spec &newSpec, - const Spec::AttributeList &toBeAdded, + Spec::AttributeList && toBeAdded, IAttributeInitializerRegistry &initializerRegistry) { - for (const auto &aspec : toBeAdded) { + for (auto &aspec : toBeAdded) { LOG(debug, "Creating initializer for attribute vector '%s': docIdLimit=%u, serialNumber=%" PRIu64, aspec.getName().c_str(), newSpec.getDocIdLimit(), newSpec.getCurrentSerialNum()); auto initializer = std::make_unique<AttributeInitializer>(_diskLayout->createAttributeDir(aspec.getName()), - _documentSubDbName, aspec, newSpec.getCurrentSerialNum(), + _documentSubDbName, std::move(aspec), newSpec.getCurrentSerialNum(), *_factory, _shared_executor); initializerRegistry.add(std::move(initializer)); @@ -287,7 +291,7 @@ AttributeManager::AttributeManager(const vespalib::string &baseDir, } AttributeManager::AttributeManager(const AttributeManager &currMgr, - const Spec &newSpec, + Spec && newSpec, IAttributeInitializerRegistry &initializerRegistry) : proton::IAttributeManager(), _attributes(), @@ -304,18 +308,17 @@ AttributeManager::AttributeManager(const AttributeManager &currMgr, _hwInfo(currMgr._hwInfo), _importedAttributes() { - Spec::AttributeList toBeAdded; - transferExistingAttributes(currMgr, newSpec, toBeAdded); - addNewAttributes(newSpec, toBeAdded, initializerRegistry); + Spec::AttributeList toBeAdded = transferExistingAttributes(currMgr, newSpec.stealAttributes()); + addNewAttributes(newSpec, std::move(toBeAdded), initializerRegistry); transferExtraAttributes(currMgr); } AttributeManager::~AttributeManager() = default; AttributeVector::SP -AttributeManager::addAttribute(const AttributeSpec &spec, uint64_t serialNum) +AttributeManager::addAttribute(AttributeSpec && spec, uint64_t serialNum) { - return internalAddAttribute(spec, serialNum, *_factory); + return internalAddAttribute(std::move(spec), serialNum, *_factory); } void @@ -480,10 +483,10 @@ AttributeManager::createContext() const } proton::IAttributeManager::SP -AttributeManager::create(const Spec &spec) const +AttributeManager::create(Spec && spec) const { SequentialAttributesInitializer initializer(spec.getDocIdLimit()); - proton::AttributeManager::SP result = std::make_shared<AttributeManager>(*this, spec, initializer); + proton::AttributeManager::SP result = std::make_shared<AttributeManager>(*this, std::move(spec), initializer); result->addInitializedAttributes(initializer.getInitializedAttributes()); return result; } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h index 41e553b47d1..2d5b75e71f4 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h @@ -84,7 +84,7 @@ private: HwInfo _hwInfo; std::unique_ptr<ImportedAttributesRepo> _importedAttributes; - AttributeVectorSP internalAddAttribute(const AttributeSpec &spec, uint64_t serialNum, const IAttributeFactory &factory); + AttributeVectorSP internalAddAttribute(AttributeSpec && spec, uint64_t serialNum, const IAttributeFactory &factory); void addAttribute(const AttributeWrap &attribute, const ShrinkerSP &shrinker); @@ -92,9 +92,9 @@ private: const FlushableWrap *findFlushable(const vespalib::string &name) const; - void transferExistingAttributes(const AttributeManager &currMgr, const Spec &newSpec, Spec::AttributeList &toBeAdded); + Spec::AttributeList transferExistingAttributes(const AttributeManager &currMgr, Spec::AttributeList && newAttributes); - void addNewAttributes(const Spec &newSpec, const Spec::AttributeList &toBeAdded, + void addNewAttributes(const Spec &newSpec, Spec::AttributeList && toBeAdded, IAttributeInitializerRegistry &initializerRegistry); void transferExtraAttributes(const AttributeManager &currMgr); @@ -121,11 +121,11 @@ public: const IAttributeFactory::SP &factory, const HwInfo &hwInfo); - AttributeManager(const AttributeManager &currMgr, const Spec &newSpec, + AttributeManager(const AttributeManager &currMgr, Spec && newSpec, IAttributeInitializerRegistry &initializerRegistry); ~AttributeManager() override; - AttributeVectorSP addAttribute(const AttributeSpec &spec, uint64_t serialNum); + AttributeVectorSP addAttribute(AttributeSpec && spec, uint64_t serialNum); void addInitializedAttributes(const std::vector<AttributeInitializerResult> &attributes); @@ -155,7 +155,7 @@ public: // Implements proton::IAttributeManager - proton::IAttributeManager::SP create(const Spec &spec) const override; + proton::IAttributeManager::SP create(Spec && spec) const override; std::vector<IFlushTargetSP> getFlushTargets() const override; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributes_initializer_base.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributes_initializer_base.cpp index cc330bc205c..7fcf541fd02 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributes_initializer_base.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributes_initializer_base.cpp @@ -3,7 +3,7 @@ #include "attributes_initializer_base.h" #include "attributemanager.h" #include <vespa/searchlib/attribute/attributevector.h> - +#include <cassert> namespace proton { void diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributes_initializer_base.h b/searchcore/src/vespa/searchcore/proton/attribute/attributes_initializer_base.h index f249cf637dd..9da268fff21 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributes_initializer_base.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributes_initializer_base.h @@ -4,6 +4,7 @@ #include "i_attribute_initializer_registry.h" #include "attribute_initializer_result.h" +#include <vector> namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp index 7439421ea59..e21230a1d58 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp @@ -3,6 +3,7 @@ #include "attributesconfigscout.h" #include "attribute_type_matcher.h" #include <vespa/searchlib/attribute/configconverter.h> +#include <vespa/searchcommon/attribute/config.h> using search::attribute::ConfigConverter; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp index d0caf92be17..88475db472c 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp @@ -78,7 +78,7 @@ FilterAttributeManager::createContext() const { } IAttributeManager::SP -FilterAttributeManager::create(const AttributeCollectionSpec &) const { +FilterAttributeManager::create(AttributeCollectionSpec &&) const { throw vespalib::IllegalArgumentException("Not implemented"); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h index e291aca6922..1a0fdcb32aa 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h @@ -38,7 +38,7 @@ public: std::unique_ptr<search::attribute::AttributeReadGuard> getAttributeReadGuard(const vespalib::string &name, bool stableEnumGuard) const override; // Implements proton::IAttributeManager - IAttributeManager::SP create(const AttributeCollectionSpec &) const override; + IAttributeManager::SP create(AttributeCollectionSpec &&) const override; std::vector<searchcorespi::IFlushTarget::SP> getFlushTargets() const override; search::SerialNum getOldestFlushedSerialNumber() const override; search::SerialNum getNewestFlushedSerialNumber() const override; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp index d074025dc32..b455d8862b5 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp @@ -11,6 +11,7 @@ #include <vespa/searchlib/common/serialnumfileheadercontext.h> #include <vespa/searchlib/attribute/attributememorysavetarget.h> #include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/util/isequencedtaskexecutor.h> #include <vespa/vespalib/util/stringfmt.h> #include <fstream> diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h index d32052fe4fa..4b6b8dc687c 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h @@ -39,7 +39,7 @@ struct IAttributeManager : public search::IAttributeManager * Create a new attribute manager based on the content of the current one and * the given attribute collection spec. */ - virtual IAttributeManager::SP create(const AttributeCollectionSpec &spec) const = 0; + virtual IAttributeManager::SP create(AttributeCollectionSpec && spec) const = 0; /** * Return the list of flush targets for this attribute manager. diff --git a/searchcore/src/vespa/searchcore/proton/common/alloc_config.cpp b/searchcore/src/vespa/searchcore/proton/common/alloc_config.cpp index 69a2d4f3ea9..1c8eae81949 100644 --- a/searchcore/src/vespa/searchcore/proton/common/alloc_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/alloc_config.cpp @@ -17,11 +17,6 @@ AllocConfig::AllocConfig(const AllocStrategy& alloc_strategy, { } -AllocConfig::AllocConfig() - : AllocConfig(AllocStrategy(), 1, 1) -{ -} - AllocConfig::~AllocConfig() = default; bool @@ -35,8 +30,8 @@ AllocConfig::operator==(const AllocConfig &rhs) const noexcept AllocStrategy AllocConfig::make_alloc_strategy(SubDbType sub_db_type) const { - auto &baseline_grow_strategy = _alloc_strategy.get_grow_strategy(); - size_t initial_capacity = baseline_grow_strategy.getDocsInitialCapacity(); + auto &baseline = _alloc_strategy.get_grow_strategy(); + size_t initial_capacity = baseline.getInitialCapacity(); switch (sub_db_type) { case SubDbType::READY: initial_capacity *= _searchable_copies; @@ -49,7 +44,7 @@ AllocConfig::make_alloc_strategy(SubDbType sub_db_type) const initial_capacity = std::max(1024ul, initial_capacity / 100); break; } - GrowStrategy grow_strategy(initial_capacity, baseline_grow_strategy.getDocsGrowFactor(), baseline_grow_strategy.getDocsGrowDelta(), baseline_grow_strategy.getMultiValueAllocGrowFactor()); + GrowStrategy grow_strategy(initial_capacity, baseline.getGrowFactor(), baseline.getGrowDelta(), initial_capacity, baseline.getMultiValueAllocGrowFactor()); return AllocStrategy(grow_strategy, _alloc_strategy.get_compaction_strategy(), _alloc_strategy.get_amortize_count()); } diff --git a/searchcore/src/vespa/searchcore/proton/common/alloc_config.h b/searchcore/src/vespa/searchcore/proton/common/alloc_config.h index 4eb604edb49..79a6cc51164 100644 --- a/searchcore/src/vespa/searchcore/proton/common/alloc_config.h +++ b/searchcore/src/vespa/searchcore/proton/common/alloc_config.h @@ -21,7 +21,6 @@ class AllocConfig public: AllocConfig(const AllocStrategy& alloc_strategy, uint32_t redundancy, uint32_t searchable_copies); - AllocConfig(); ~AllocConfig(); bool operator==(const AllocConfig &rhs) const noexcept; @@ -29,6 +28,7 @@ public: return !operator==(rhs); } AllocStrategy make_alloc_strategy(SubDbType sub_db_type) const; + static AllocConfig makeDefault() { return AllocConfig(AllocStrategy(), 1, 1); } }; } diff --git a/searchcore/src/vespa/searchcore/proton/common/attribute_updater.cpp b/searchcore/src/vespa/searchcore/proton/common/attribute_updater.cpp index fe29a11639b..e9f6536d50d 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attribute_updater.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/attribute_updater.cpp @@ -24,6 +24,7 @@ #include <vespa/searchlib/attribute/reference_attribute.h> #include <vespa/searchlib/tensor/tensor_attribute.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/classname.h> #include <sstream> #include <vespa/log/log.h> @@ -31,6 +32,7 @@ LOG_SETUP(".proton.common.attribute_updater"); using namespace document; using vespalib::make_string; +using vespalib::getClassName; using search::tensor::PrepareResult; using search::tensor::TensorAttribute; using search::attribute::ReferenceAttribute; @@ -270,21 +272,20 @@ AttributeUpdater::handleUpdate(AttributeVector & vec, uint32_t lid, const FieldU } } - const vespalib::Identifiable::RuntimeClass &info = vec.getClass(); - if (info.inherits(IntegerAttribute::classId)) { + if (vec.isIntegerType()) { handleUpdateT(static_cast<IntegerAttribute &>(vec), GetLong(), lid, vUp); - } else if (info.inherits(FloatingPointAttribute::classId)) { + } else if (vec.isFloatingPointType()) { handleUpdateT(static_cast<FloatingPointAttribute &>(vec), GetDouble(), lid, vUp); - } else if (info.inherits(StringAttribute::classId)) { + } else if (vec.isStringType()) { handleUpdateT(static_cast<StringAttribute &>(vec), GetString(), lid, vUp); - } else if (info.inherits(PredicateAttribute::classId)) { + } else if (vec.isPredicateType()) { handleUpdate(static_cast<PredicateAttribute &>(vec), lid, vUp); - } else if (info.inherits(TensorAttribute::classId)) { + } else if (vec.isTensorType()) { handleUpdate(static_cast<TensorAttribute &>(vec), lid, vUp); - } else if (info.inherits(ReferenceAttribute::classId)) { + } else if (vec.isReferenceType()) { handleUpdate(static_cast<ReferenceAttribute &>(vec), lid, vUp); } else { - LOG(warning, "Unsupported attribute vector '%s' (classname=%s)", vec.getName().c_str(), info.name()); + LOG(warning, "Unsupported attribute vector '%s' (classname=%s)", vec.getName().c_str(), getClassName(vec).c_str()); return; } } @@ -294,24 +295,23 @@ void AttributeUpdater::handleValue(AttributeVector & vec, uint32_t lid, const FieldValue & val) { LOG(spam, "handleValue(%s, %u): %s", vec.getName().c_str(), lid, toString(val).c_str()); - const vespalib::Identifiable::RuntimeClass & rc = vec.getClass(); - if (rc.inherits(IntegerAttribute::classId)) { + if (vec.isIntegerType()) { handleValueT(static_cast<IntegerAttribute &>(vec), GetLong(), lid, val); - } else if (rc.inherits(FloatingPointAttribute::classId)) { + } else if (vec.isFloatingPointType()) { handleValueT(static_cast<FloatingPointAttribute &>(vec), GetDouble(), lid, val); - } else if (rc.inherits(StringAttribute::classId)) { + } else if (vec.isStringType()) { handleValueT(static_cast<StringAttribute &>(vec), GetString(), lid, val); - } else if (rc.inherits(PredicateAttribute::classId)) { + } else if (vec.isPredicateType()) { // PredicateAttribute is never multivalue. updateValue(static_cast<PredicateAttribute &>(vec), lid, val); - } else if (rc.inherits(TensorAttribute::classId)) { + } else if (vec.isTensorType()) { // TensorAttribute is never multivalue. updateValue(static_cast<TensorAttribute &>(vec), lid, val); - } else if (rc.inherits(ReferenceAttribute::classId)) { + } else if (vec.isReferenceType()) { // ReferenceAttribute is never multivalue. updateValue(static_cast<ReferenceAttribute &>(vec), lid, val); } else { - LOG(warning, "Unsupported attribute vector '%s' (classname=%s)", vec.getName().c_str(), rc.name()); + LOG(warning, "Unsupported attribute vector '%s' (classname=%s)", vec.getName().c_str(), getClassName(vec).c_str()); return; } } @@ -504,11 +504,10 @@ namespace { void validate_tensor_attribute_type(AttributeVector& attr) { - const auto& info = attr.getClass(); - if (!info.inherits(TensorAttribute::classId)) { + if (!attr.isTensorType()) { throw UpdateException( make_string("Expected attribute vector '%s' to be a TensorAttribute, but was '%s'", - attr.getName().c_str(), info.name())); + attr.getName().c_str(), getClassName(attr).c_str())); } } diff --git a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp index 8555af7c685..f568e8ed702 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/vespalib/util/exceptions.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".proton.common.attribute_field_value_node"); diff --git a/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp b/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp index 342975c7e9f..b0244c34a0f 100644 --- a/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp @@ -9,6 +9,7 @@ #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/searchlib/attribute/iattributemanager.h> +#include <cassert> namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index 46b5f051422..c0d291ca695 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -5,8 +5,6 @@ #include "operation_listener.h" #include "search_context.h" #include "document_meta_store_versions.h" -#include <vespa/fastos/file.h> -#include <vespa/persistence/spi/bucket_limits.h> #include <vespa/searchcore/proton/bucketdb/bucketsessionbase.h> #include <vespa/searchcore/proton/bucketdb/joinbucketssession.h> #include <vespa/searchcore/proton/bucketdb/remove_batch_entry.h> @@ -16,6 +14,8 @@ #include <vespa/searchlib/common/i_gid_to_lid_mapper.h> #include <vespa/searchlib/query/query_term_simple.h> #include <vespa/searchlib/util/bufferwriter.h> +#include <vespa/searchcommon/attribute/config.h> +#include <vespa/persistence/spi/bucket_limits.h> #include <vespa/vespalib/btree/btree.hpp> #include <vespa/vespalib/btree/btreebuilder.hpp> #include <vespa/vespalib/btree/btreenodeallocator.hpp> @@ -24,6 +24,7 @@ #include <vespa/vespalib/datastore/buffer_type.hpp> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/rcuvector.hpp> +#include <vespa/fastos/file.h> #include <vespa/log/log.h> LOG_SETUP(".proton.documentmetastore"); @@ -403,22 +404,23 @@ DocumentMetaStore::unload() unloadBucket(*_bucketDB, prev, prevDelta); } +DocumentMetaStore::DocumentMetaStore(BucketDBOwnerSP bucketDB) + : DocumentMetaStore(std::move(bucketDB), getFixedName()) +{} +DocumentMetaStore::DocumentMetaStore(BucketDBOwnerSP bucketDB, const vespalib::string &name) + : DocumentMetaStore(std::move(bucketDB), name, search::GrowStrategy()) +{} DocumentMetaStore::DocumentMetaStore(BucketDBOwnerSP bucketDB, const vespalib::string &name, const GrowStrategy &grow, SubDbType subDbType) : DocumentMetaStoreAttribute(name), - _metaDataStore(grow.getDocsInitialCapacity(), - grow.getDocsGrowPercent(), - grow.getDocsGrowDelta(), - getGenerationHolder()), + _metaDataStore(grow, getGenerationHolder()), _gidToLidMap(), _gid_to_lid_map_write_itr(vespalib::datastore::EntryRef(), _gidToLidMap.getAllocator()), _gid_to_lid_map_write_itr_prepare_serial_num(0u), - _lidAlloc(_metaDataStore.size(), - _metaDataStore.capacity(), - getGenerationHolder()), + _lidAlloc(_metaDataStore.size(), _metaDataStore.capacity(), getGenerationHolder()), _bucketDB(std::move(bucketDB)), _shrinkLidSpaceBlockers(0), _subDbType(subDbType), diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h index adb2778c2d2..d1672eeb6dd 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h @@ -9,9 +9,9 @@ #include "lid_hold_list.h" #include "raw_document_meta_data.h" #include <vespa/searchcore/proton/common/subdbtype.h> -#include <vespa/searchlib/attribute/singlesmallnumericattribute.h> #include <vespa/searchlib/queryeval/blueprint.h> #include <vespa/searchlib/docstore/ibucketizer.h> +#include <vespa/searchcommon/common/growstrategy.h> #include <vespa/vespalib/util/rcuvector.h> namespace proton::bucketdb { @@ -148,9 +148,11 @@ public: sizeof(uint32_t) + GlobalId::LENGTH + sizeof(uint8_t) + sizeof(Timestamp); + DocumentMetaStore(BucketDBOwnerSP bucketDB); + DocumentMetaStore(BucketDBOwnerSP bucketDB, const vespalib::string & name); DocumentMetaStore(BucketDBOwnerSP bucketDB, - const vespalib::string & name=getFixedName(), - const search::GrowStrategy & grow=search::GrowStrategy(), + const vespalib::string & name, + const search::GrowStrategy & grow, SubDbType subDbType = SubDbType::READY); ~DocumentMetaStore(); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.cpp index 4999c93df8d..57c3159645b 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreattribute.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "documentmetastoreattribute.h" +#include <vespa/searchcommon/attribute/config.h> namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.cpp index e27bd8706e9..3be9282987c 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.cpp @@ -10,12 +10,15 @@ DocumentMetaStoreContext::ReadGuard::ReadGuard(const search::AttributeVector::SP { } +DocumentMetaStoreContext::DocumentMetaStoreContext(std::shared_ptr<bucketdb::BucketDBOwner> bucketDB) + : DocumentMetaStoreContext(std::move(bucketDB), DocumentMetaStore::getFixedName(), search::GrowStrategy()) +{} DocumentMetaStoreContext::DocumentMetaStoreContext(std::shared_ptr<bucketdb::BucketDBOwner> bucketDB, const vespalib::string &name, - const search::GrowStrategy &grow) : - _metaStoreAttr(std::make_shared<DocumentMetaStore>(std::move(bucketDB), name, grow)), - _metaStore(std::dynamic_pointer_cast<IDocumentMetaStore>(_metaStoreAttr)) + const search::GrowStrategy &grow) + : _metaStoreAttr(std::make_shared<DocumentMetaStore>(std::move(bucketDB), name, grow)), + _metaStore(std::dynamic_pointer_cast<IDocumentMetaStore>(_metaStoreAttr)) { } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h index 6a07d6566cd..c4117f0b751 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h @@ -27,13 +27,14 @@ private: IDocumentMetaStore::SP _metaStore; public: + DocumentMetaStoreContext(std::shared_ptr<bucketdb::BucketDBOwner> bucketDB); /** * Create a new context instantiating a document meta store * with the given name, grow strategy, and comparator. */ DocumentMetaStoreContext(std::shared_ptr<bucketdb::BucketDBOwner> bucketDB, - const vespalib::string &name = DocumentMetaStore::getFixedName(), - const search::GrowStrategy &grow = search::GrowStrategy()); + const vespalib::string &name, + const search::GrowStrategy &grow); ~DocumentMetaStoreContext() override; /** * Create a new context with the given document meta store encapsulated diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp index ea815d102f7..82e1aa3b57c 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "search_context.h" +#include "documentmetastore.h" #include <vespa/searchlib/attribute/attributeiterators.h> #include <vespa/searchlib/query/query_term_simple.h> #include <vespa/vespalib/util/exceptions.h> diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h index a2a265ad519..ca4b026e2a4 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/search_context.h @@ -3,10 +3,9 @@ #pragma once #include <vespa/document/base/globalid.h> -#include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/attribute/search_context.h> -#include "documentmetastore.h" +namespace proton { class DocumentMetaStore; } namespace proton::documentmetastore { /** @@ -15,7 +14,7 @@ namespace proton::documentmetastore { class SearchContext : public search::attribute::SearchContext { private: - using DocId = search::AttributeVector::DocId; + using DocId = uint32_t; bool _isWord; document::GlobalId _gid; @@ -24,7 +23,7 @@ private: int32_t onFind(DocId docId, int32_t elemId, int32_t &weight) const override; int32_t onFind(DocId docId, int32_t elemId) const override; - search::queryeval::SearchIterator::UP + std::unique_ptr<search::queryeval::SearchIterator> createIterator(search::fef::TermFieldMatchData *matchData, bool strict) override; const DocumentMetaStore &getStore() const; diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 7e9f763a758..5a0bcb1cd41 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -527,7 +527,7 @@ PersistenceEngine::createIterator(const Bucket &bucket, FieldSetSP fields, const entry->handler_sequence = HandlerSnapshot::release(std::move(snapshot)); std::lock_guard<std::mutex> guard(_iterators_lock); - static IteratorId id_counter(0); + static std::atomic<IteratorId::Type> id_counter(0); IteratorId id(++id_counter); _iterators[id] = entry.release(); return CreateIteratorResult(id); diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp b/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp index 8e5e457b80e..294294194f4 100644 --- a/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.cpp @@ -3,6 +3,7 @@ #include "attribute_reprocessing_initializer.h" #include <vespa/searchcore/proton/attribute/attribute_populator.h> #include <vespa/searchcommon/attribute/attribute_utils.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/searchcore/proton/attribute/document_field_populator.h> #include <vespa/searchcore/proton/attribute/filter_attribute_manager.h> #include <vespa/searchcore/proton/common/i_indexschema_inspector.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/attribute_writer_factory.h b/searchcore/src/vespa/searchcore/proton/server/attribute_writer_factory.h index d95dfd1314e..6ca4556d666 100644 --- a/searchcore/src/vespa/searchcore/proton/server/attribute_writer_factory.h +++ b/searchcore/src/vespa/searchcore/proton/server/attribute_writer_factory.h @@ -14,11 +14,11 @@ struct AttributeWriterFactory : public IAttributeWriterFactory { AttributeWriterFactory() {} IAttributeWriter::SP create(const IAttributeWriter::SP &old, - const AttributeCollectionSpec &attrSpec) const override + AttributeCollectionSpec && attrSpec) const override { const AttributeWriter &oldAdapter = dynamic_cast<const AttributeWriter &>(*old.get()); const proton::IAttributeManager::SP &oldMgr = oldAdapter.getAttributeManager(); - proton::IAttributeManager::SP newMgr = oldMgr->create(attrSpec); + proton::IAttributeManager::SP newMgr = oldMgr->create(std::move(attrSpec)); return std::make_shared<AttributeWriter>(newMgr); } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h index 4ae0adeb5a6..e69b015e782 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h @@ -2,16 +2,18 @@ #pragma once -#include "documentdbconfig.h" #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/config/config-proton.h> #include <vespa/document/config/config-documenttypes.h> +#include <vespa/document/config/documenttypes_config_fwd.h> #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/config/retriever/configsnapshot.h> #include <vespa/fileacquirer/config-filedistributorrpc.h> namespace vespa::config::content::core::internal { class InternalBucketspacesType; } +namespace document { class DocumentTypeRepo; } + namespace proton { /** @@ -25,7 +27,7 @@ public: using FiledistributorrpcConfig = const cloud::config::filedistribution::FiledistributorrpcConfig; using FiledistributorrpcConfigSP = std::shared_ptr<FiledistributorrpcConfig>; - typedef DocumentDBConfig::DocumenttypesConfigSP DocumenttypesConfigSP; + using DocumenttypesConfigSP = std::shared_ptr<DocumenttypesConfig>; using BucketspacesConfig = const vespa::config::content::core::internal::InternalBucketspacesType; using BucketspacesConfigSP = std::shared_ptr<BucketspacesConfig>; diff --git a/searchcore/src/vespa/searchcore/proton/server/configstore.h b/searchcore/src/vespa/searchcore/proton/server/configstore.h index 4971c77369c..b6923e45e2a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/configstore.h +++ b/searchcore/src/vespa/searchcore/proton/server/configstore.h @@ -2,7 +2,6 @@ #pragma once -#include "documentdbconfig.h" #include "feedconfigstore.h" #include <vespa/searchcommon/common/schema.h> @@ -11,6 +10,8 @@ namespace vespa::config::search::core::internal { } namespace proton { +class DocumentDBConfig; + struct ConfigStore : FeedConfigStore { typedef std::unique_ptr<ConfigStore> UP; typedef search::SerialNum SerialNum; @@ -25,7 +26,7 @@ struct ConfigStore : FeedConfigStore { * resulting config snapshot. */ virtual void loadConfig(const DocumentDBConfig ¤tSnapshot, SerialNum serialNum, - DocumentDBConfig::SP &loadedSnapshot) = 0; + std::shared_ptr<DocumentDBConfig> &loadedSnapshot) = 0; virtual void saveConfig(const DocumentDBConfig &snapshot, SerialNum serialNum) = 0; virtual void removeInvalid() = 0; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 66b4d293100..3fdf82ea61d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -449,7 +449,7 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot, SerialNum serialNum return; } - ConfigComparisonResult cmpres; + DocumentDBConfig::ComparisonResult cmpres; Schema::SP oldSchema; int64_t generation = configSnapshot->getGeneration(); { diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index 078fce65ff2..96944ba25e7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -8,7 +8,6 @@ #include "disk_mem_usage_forwarder.h" #include "document_db_config_owner.h" #include "documentdb_metrics_updater.h" -#include "documentdbconfig.h" #include "documentsubdbcollection.h" #include "executorthreadingservice.h" #include "i_document_subdb_owner.h" @@ -77,11 +76,10 @@ private: using IFlushTargetList = std::vector<std::shared_ptr<searchcorespi::IFlushTarget>>; using StatusReportUP = std::unique_ptr<StatusReport>; using ProtonConfig = const vespa::config::search::core::internal::InternalProtonType; - using ConfigComparisonResult = DocumentDBConfig::ComparisonResult; using lock_guard = std::lock_guard<std::mutex>; using SerialNum = search::SerialNum; using Schema = search::index::Schema; - + using DocumentDBConfigSP = std::shared_ptr<DocumentDBConfig>; DocTypeName _docTypeName; document::BucketSpace _bucketSpace; @@ -93,52 +91,52 @@ private: InitializeThreads _initializeThreads; // variables related to reconfig - DocumentDBConfig::SP _initConfigSnapshot; - SerialNum _initConfigSerialNum; - vespalib::VarHolder<DocumentDBConfig::SP> _pendingConfigSnapshot; - mutable std::mutex _configMutex; // protects _active* below. - mutable std::condition_variable _configCV; - DocumentDBConfig::SP _activeConfigSnapshot; - int64_t _activeConfigSnapshotGeneration; - const bool _validateAndSanitizeDocStore; - vespalib::Gate _initGate; - - ClusterStateHandler _clusterStateHandler; - BucketHandler _bucketHandler; - index::IndexConfig _indexCfg; - std::unique_ptr<ReplayThrottlingPolicy> _replay_throttling_policy; - ConfigStore::UP _config_store; - std::shared_ptr<matching::SessionManager> _sessionManager; // TODO: This should not have to be a shared pointer. - MetricsWireService &_metricsWireService; - DocumentDBTaggedMetrics _metrics; - std::unique_ptr<metrics::UpdateHook> _metricsHook; - vespalib::VarHolder<IFeedView::SP> _feedView; - vespalib::MonitoredRefCount _refCount; - IDocumentDBOwner &_owner; - storage::spi::BucketExecutor &_bucketExecutor; - DDBState _state; - DiskMemUsageForwarder _dmUsageForwarder; - AttributeUsageFilter _writeFilter; + DocumentDBConfigSP _initConfigSnapshot; + SerialNum _initConfigSerialNum; + vespalib::VarHolder<DocumentDBConfigSP> _pendingConfigSnapshot; + mutable std::mutex _configMutex; // protects _active* below. + mutable std::condition_variable _configCV; + DocumentDBConfigSP _activeConfigSnapshot; + int64_t _activeConfigSnapshotGeneration; + const bool _validateAndSanitizeDocStore; + vespalib::Gate _initGate; + + ClusterStateHandler _clusterStateHandler; + BucketHandler _bucketHandler; + index::IndexConfig _indexCfg; + std::unique_ptr<ReplayThrottlingPolicy> _replay_throttling_policy; + ConfigStore::UP _config_store; + std::shared_ptr<matching::SessionManager> _sessionManager; // TODO: This should not have to be a shared pointer. + MetricsWireService &_metricsWireService; + DocumentDBTaggedMetrics _metrics; + std::unique_ptr<metrics::UpdateHook> _metricsHook; + vespalib::VarHolder<IFeedView::SP> _feedView; + vespalib::MonitoredRefCount _refCount; + IDocumentDBOwner &_owner; + storage::spi::BucketExecutor &_bucketExecutor; + DDBState _state; + DiskMemUsageForwarder _dmUsageForwarder; + AttributeUsageFilter _writeFilter; std::shared_ptr<ITransientResourceUsageProvider> _transient_usage_provider; - std::unique_ptr<FeedHandler> _feedHandler; - DocumentSubDBCollection _subDBs; - MaintenanceController _maintenanceController; - DocumentDBJobTrackers _jobTrackers; - std::shared_ptr<IBucketStateCalculator> _calc; - DocumentDBMetricsUpdater _metricsUpdater; + std::unique_ptr<FeedHandler> _feedHandler; + DocumentSubDBCollection _subDBs; + MaintenanceController _maintenanceController; + DocumentDBJobTrackers _jobTrackers; + std::shared_ptr<IBucketStateCalculator> _calc; + DocumentDBMetricsUpdater _metricsUpdater; void registerReference(); - void setActiveConfig(const DocumentDBConfig::SP &config, int64_t generation); - DocumentDBConfig::SP getActiveConfig() const; + void setActiveConfig(const DocumentDBConfigSP &config, int64_t generation); + DocumentDBConfigSP getActiveConfig() const; void internalInit(); void initManagers(); - void initFinish(DocumentDBConfig::SP configSnapshot); - void performReconfig(DocumentDBConfig::SP configSnapshot); + void initFinish(DocumentDBConfigSP configSnapshot); + void performReconfig(DocumentDBConfigSP configSnapshot); void closeSubDBs(); void applySubDBConfig(const DocumentDBConfig &newConfigSnapshot, SerialNum serialNum, const ReconfigParams ¶ms); - void applyConfig(DocumentDBConfig::SP configSnapshot, SerialNum serialNum); + void applyConfig(DocumentDBConfigSP configSnapshot, SerialNum serialNum); /** * Save initial config if we don't have any saved config snapshots. @@ -195,7 +193,7 @@ private: friend class InitDoneTask; DocumentDB(const vespalib::string &baseDir, - DocumentDBConfig::SP currentSnapshot, + DocumentDBConfigSP currentSnapshot, const vespalib::string &tlsSpec, matching::QueryLimiter &queryLimiter, const DocTypeName &docTypeName, @@ -225,7 +223,7 @@ public: */ static DocumentDB::SP create(const vespalib::string &baseDir, - DocumentDBConfig::SP currentSnapshot, + DocumentDBConfigSP currentSnapshot, const vespalib::string &tlsSpec, matching::QueryLimiter &queryLimiter, const DocTypeName &docTypeName, @@ -371,8 +369,8 @@ public: bool getDelayedConfig() const { return _state.getDelayedConfig(); } void replayConfig(SerialNum serialNum) override; const DocTypeName & getDocTypeName() const { return _docTypeName; } - void newConfigSnapshot(DocumentDBConfig::SP snapshot); - void reconfigure(const DocumentDBConfig::SP & snapshot) override; + void newConfigSnapshot(DocumentDBConfigSP snapshot); + void reconfigure(const DocumentDBConfigSP & snapshot) override; int64_t getActiveGeneration() const; /* * Implements IDocumentSubDBOwner diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp index 5bc54404721..8c067cfc3db 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp @@ -4,7 +4,6 @@ #include "threading_service_config.h" #include <vespa/config-attributes.h> #include <vespa/config-imported-fields.h> -#include <vespa/config-indexschema.h> #include <vespa/config-rank-profiles.h> #include <vespa/config-summary.h> #include <vespa/config-summarymap.h> @@ -51,7 +50,6 @@ DocumentDBConfig::ComparisonResult::ComparisonResult() storeChanged(false), visibilityDelayChanged(false), flushChanged(false), - threading_service_config_changed(false), alloc_config_changed(false) { } @@ -73,8 +71,8 @@ DocumentDBConfig::DocumentDBConfig( const Schema::SP &schema, const DocumentDBMaintenanceConfig::SP &maintenance, const search::LogDocumentStore::Config & storeConfig, - std::shared_ptr<const ThreadingServiceConfig> threading_service_config, - std::shared_ptr<const AllocConfig> alloc_config, + const ThreadingServiceConfig & threading_service_config, + const AllocConfig & alloc_config, const vespalib::string &configId, const vespalib::string &docTypeName) noexcept : _configId(configId), @@ -96,8 +94,8 @@ DocumentDBConfig::DocumentDBConfig( _schema(schema), _maintenance(maintenance), _storeConfig(storeConfig), - _threading_service_config(std::move(threading_service_config)), - _alloc_config(std::move(alloc_config)), + _threading_service_config(threading_service_config), + _alloc_config(alloc_config), _orig(), _delayedAttributeAspects(false) { } @@ -150,9 +148,9 @@ DocumentDBConfig::operator==(const DocumentDBConfig & rhs) const equals<TuneFileDocumentDB>(_tuneFileDocumentDB.get(), rhs._tuneFileDocumentDB.get()) && equals<Schema>(_schema.get(), rhs._schema.get()) && equals<DocumentDBMaintenanceConfig>(_maintenance.get(), rhs._maintenance.get()) && - _storeConfig == rhs._storeConfig && - equals<ThreadingServiceConfig>(_threading_service_config.get(), rhs._threading_service_config.get()) && - equals<AllocConfig>(_alloc_config.get(), rhs._alloc_config.get()); + (_storeConfig == rhs._storeConfig) && + (_threading_service_config == rhs._threading_service_config) && + (_alloc_config == rhs._alloc_config); } @@ -178,8 +176,7 @@ DocumentDBConfig::compare(const DocumentDBConfig &rhs) const retval.storeChanged = (_storeConfig != rhs._storeConfig); retval.visibilityDelayChanged = (_maintenance->getVisibilityDelay() != rhs._maintenance->getVisibilityDelay()); retval.flushChanged = !equals<DocumentDBMaintenanceConfig>(_maintenance.get(), rhs._maintenance.get(), [](const auto &l, const auto &r) { return l.getFlushConfig() == r.getFlushConfig(); }); - retval.threading_service_config_changed = !equals<ThreadingServiceConfig>(_threading_service_config.get(), rhs._threading_service_config.get()); - retval.alloc_config_changed = !equals<AllocConfig>(_alloc_config.get(), rhs._alloc_config.get()); + retval.alloc_config_changed = (_alloc_config != rhs._alloc_config); return retval; } @@ -201,9 +198,7 @@ DocumentDBConfig::valid() const _importedFields && _tuneFileDocumentDB && _schema && - _maintenance && - _threading_service_config && - _alloc_config; + _maintenance; } namespace diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h index c0b0eb9dd1b..5f5887df6ea 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h @@ -3,15 +3,18 @@ #pragma once #include "document_db_maintenance_config.h" -#include <vespa/document/config/documenttypes_config_fwd.h> -#include <vespa/searchlib/common/tunefileinfo.h> -#include <vespa/searchcommon/common/schema.h> +#include "threading_service_config.h" +#include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/matching/ranking_constants.h> #include <vespa/searchcore/proton/matching/ranking_expressions.h> #include <vespa/searchcore/proton/matching/onnx_models.h> +#include <vespa/searchlib/common/tunefileinfo.h> +#include <vespa/searchlib/docstore/logdocumentstore.h> +#include <vespa/searchcommon/common/schema.h> +#include <vespa/document/config/documenttypes_config_fwd.h> + #include <vespa/config/retriever/configkeyset.h> #include <vespa/config/retriever/configsnapshot.h> -#include <vespa/searchlib/docstore/logdocumentstore.h> namespace vespa::config::search::internal { class InternalSummaryType; @@ -30,9 +33,6 @@ namespace document { namespace proton { -class ThreadingServiceConfig; -class AllocConfig; - class DocumentDBConfig { public: @@ -57,7 +57,6 @@ public: bool storeChanged; bool visibilityDelayChanged; bool flushChanged; - bool threading_service_config_changed; bool alloc_config_changed; ComparisonResult(); @@ -92,7 +91,6 @@ public: } return *this; } - ComparisonResult &set_threading_service_config_changed(bool val) { threading_service_config_changed = val; return *this; } ComparisonResult &set_alloc_config_changed(bool val) { alloc_config_changed = val; return *this; } }; @@ -137,8 +135,8 @@ private: search::index::Schema::SP _schema; MaintenanceConfigSP _maintenance; search::LogDocumentStore::Config _storeConfig; - std::shared_ptr<const ThreadingServiceConfig> _threading_service_config; - std::shared_ptr<const AllocConfig> _alloc_config; + const ThreadingServiceConfig _threading_service_config; + const AllocConfig _alloc_config; SP _orig; bool _delayedAttributeAspects; @@ -146,18 +144,18 @@ private: template <typename T> bool equals(const T * lhs, const T * rhs) const { - if (lhs == NULL) { - return rhs == NULL; + if (lhs == nullptr) { + return rhs == nullptr; } - return rhs != NULL && *lhs == *rhs; + return rhs != nullptr && *lhs == *rhs; } template <typename T, typename Func> bool equals(const T *lhs, const T *rhs, Func isEqual) const { - if (lhs == NULL) { - return rhs == NULL; + if (lhs == nullptr) { + return rhs == nullptr; } - return rhs != NULL && isEqual(*lhs, *rhs); + return rhs != nullptr && isEqual(*lhs, *rhs); } public: DocumentDBConfig(int64_t generation, @@ -177,8 +175,8 @@ public: const search::index::Schema::SP &schema, const DocumentDBMaintenanceConfig::SP &maintenance, const search::LogDocumentStore::Config & storeConfig, - std::shared_ptr<const ThreadingServiceConfig> threading_service_config, - std::shared_ptr<const AllocConfig> alloc_config, + const ThreadingServiceConfig & threading_service_config, + const AllocConfig & alloc_config, const vespalib::string &configId, const vespalib::string &docTypeName) noexcept; @@ -220,10 +218,8 @@ public: const MaintenanceConfigSP &getMaintenanceConfigSP() const { return _maintenance; } const search::TuneFileDocumentDB::SP &getTuneFileDocumentDBSP() const { return _tuneFileDocumentDB; } bool getDelayedAttributeAspects() const { return _delayedAttributeAspects; } - const ThreadingServiceConfig& get_threading_service_config() const { return *_threading_service_config; } - const std::shared_ptr<const ThreadingServiceConfig>& get_threading_service_config_shared_ptr() const { return _threading_service_config; } - const AllocConfig& get_alloc_config() const { return *_alloc_config; } - const std::shared_ptr<const AllocConfig>& get_alloc_config_shared_ptr() const { return _alloc_config; } + const ThreadingServiceConfig& get_threading_service_config() const { return _threading_service_config; } + const AllocConfig& get_alloc_config() const { return _alloc_config; } bool operator==(const DocumentDBConfig &rhs) const; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index bb6d45ac482..1aadc845744 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -248,17 +248,16 @@ find_document_db_config_entry(const ProtonConfig::DocumentdbVector& document_dbs return default_document_db_config_entry; } -std::shared_ptr<const AllocConfig> +const AllocConfig build_alloc_config(const ProtonConfig& proton_config, const vespalib::string& doc_type_name) { auto& document_db_config_entry = find_document_db_config_entry(proton_config.documentdb, doc_type_name); auto& alloc_config = document_db_config_entry.allocation; auto& distribution_config = proton_config.distribution; - search::GrowStrategy grow_strategy(alloc_config.initialnumdocs, alloc_config.growfactor, alloc_config.growbias, alloc_config.multivaluegrowfactor); + search::GrowStrategy grow_strategy(alloc_config.initialnumdocs, alloc_config.growfactor, alloc_config.growbias, alloc_config.initialnumdocs, alloc_config.multivaluegrowfactor); CompactionStrategy compaction_strategy(alloc_config.maxDeadBytesRatio, alloc_config.maxDeadAddressSpaceRatio, alloc_config.maxCompactBuffers, alloc_config.activeBuffersRatio); - return std::make_shared<const AllocConfig> - (AllocStrategy(grow_strategy, compaction_strategy, alloc_config.amortizecount), - distribution_config.redundancy, distribution_config.searchablecopies); + return AllocConfig(AllocStrategy(grow_strategy, compaction_strategy, alloc_config.amortizecount), + distribution_config.redundancy, distribution_config.searchablecopies); } vespalib::string @@ -305,8 +304,6 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot IndexschemaConfigSP newIndexschemaConfig; MaintenanceConfigSP oldMaintenanceConfig; MaintenanceConfigSP newMaintenanceConfig; - std::shared_ptr<const ThreadingServiceConfig> old_threading_service_config; - std::shared_ptr<const AllocConfig> old_alloc_config; constexpr vespalib::duration file_resolve_timeout = 60min; if (!_ignoreForwardedConfig) { @@ -332,8 +329,6 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot newOnnxModels = current->getOnnxModelsSP(); newIndexschemaConfig = current->getIndexschemaConfigSP(); oldMaintenanceConfig = current->getMaintenanceConfigSP(); - old_threading_service_config = current->get_threading_service_config_shared_ptr(); - old_alloc_config = current->get_alloc_config_shared_ptr(); currentGeneration = current->getGeneration(); } @@ -408,15 +403,6 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot if (newMaintenanceConfig && oldMaintenanceConfig && (*newMaintenanceConfig == *oldMaintenanceConfig)) { newMaintenanceConfig = oldMaintenanceConfig; } - auto new_threading_service_config = std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make(_bootstrapConfig->getProtonConfig())); - if (new_threading_service_config && old_threading_service_config && - (*new_threading_service_config == *old_threading_service_config)) { - new_threading_service_config = old_threading_service_config; - } - auto new_alloc_config = build_alloc_config(_bootstrapConfig->getProtonConfig(), _docTypeName); - if (new_alloc_config && old_alloc_config &&(*new_alloc_config == *old_alloc_config)) { - new_alloc_config = old_alloc_config; - } auto newSnapshot = std::make_shared<DocumentDBConfig>(generation, newRankProfilesConfig, newRankingConstants, @@ -434,8 +420,8 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot schema, newMaintenanceConfig, storeConfig, - new_threading_service_config, - new_alloc_config, + ThreadingServiceConfig::make(_bootstrapConfig->getProtonConfig()), + build_alloc_config(_bootstrapConfig->getProtonConfig(), _docTypeName), _configId, _docTypeName); assert(newSnapshot->valid()); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigscout.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigscout.cpp index 07786eba1bb..f7df7086316 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigscout.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigscout.cpp @@ -1,20 +1,19 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "documentdbconfigscout.h" +#include "documentdbconfig.h" #include <vespa/searchcore/proton/attribute/attributesconfigscout.h> using vespa::config::search::AttributesConfig; namespace proton { - -DocumentDBConfig::SP -DocumentDBConfigScout::scout(const DocumentDBConfig::SP &config, +std::shared_ptr<DocumentDBConfig> +DocumentDBConfigScout::scout(const std::shared_ptr<DocumentDBConfig> &config, const DocumentDBConfig &liveConfig) { AttributesConfigScout acScout(liveConfig.getAttributesConfig()); - std::shared_ptr<AttributesConfig> - ac(acScout.adjust(config->getAttributesConfig())); + std::shared_ptr<AttributesConfig> ac(acScout.adjust(config->getAttributesConfig())); if (*ac == config->getAttributesConfig()) return config; // no change return config->newFromAttributesConfig(ac); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigscout.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigscout.h index 5c1c9e9ac56..c0c19dab423 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigscout.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigscout.h @@ -2,10 +2,11 @@ #pragma once -#include "documentdbconfig.h" +#include <memory> -namespace proton -{ +namespace proton { + +class DocumentDBConfig; /** * Class to create adjusted document db config that minimizes the number of @@ -16,9 +17,8 @@ namespace proton class DocumentDBConfigScout { public: - static DocumentDBConfig::SP - scout(const DocumentDBConfig::SP &config, - const DocumentDBConfig &liveConfig); + static std::shared_ptr<DocumentDBConfig> + scout(const std::shared_ptr<DocumentDBConfig> &config, const DocumentDBConfig &liveConfig); }; } diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp index 228a46123a4..73d08785d5f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp @@ -16,7 +16,6 @@ #include <vespa/searchcore/proton/matching/sessionmanager.h> #include <vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h> #include <vespa/searchcore/proton/reprocessing/reprocess_documents_task.h> -#include <vespa/searchlib/docstore/document_store_visitor_progress.h> #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/log/log.h> @@ -104,7 +103,7 @@ FastAccessDocSubDB::setupAttributeManager(AttributeManager::SP attrMgrResult) } -AttributeCollectionSpec::UP +std::unique_ptr<AttributeCollectionSpec> FastAccessDocSubDB::createAttributeSpec(const AttributesConfig &attrCfg, const AllocStrategy& alloc_strategy, SerialNum serialNum) const { uint32_t docIdLimit(_dms->getCommittedDocIdLimit()); @@ -267,10 +266,10 @@ FastAccessDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const FastAccessDocSubDBConfigurer configurer(_fastAccessFeedView, std::make_unique<AttributeWriterFactory>(), getSubDbName()); proton::IAttributeManager::SP oldMgr = extractAttributeManager(_fastAccessFeedView.get()); - AttributeCollectionSpec::UP attrSpec = + std::unique_ptr<AttributeCollectionSpec> attrSpec = createAttributeSpec(newConfigSnapshot.getAttributesConfig(), alloc_strategy, serialNum); IReprocessingInitializer::UP initializer = - configurer.reconfigure(newConfigSnapshot, oldConfigSnapshot, *attrSpec); + configurer.reconfigure(newConfigSnapshot, oldConfigSnapshot, std::move(*attrSpec)); if (initializer->hasReprocessors()) { tasks.push_back(IReprocessingTask::SP(createReprocessingTask(*initializer, newConfigSnapshot.getDocumentTypeRepoSP()).release())); diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h index 51c2378789b..d29c71ea43c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h @@ -91,7 +91,7 @@ protected: std::shared_ptr<search::attribute::Interlock> _attribute_interlock; DocIdLimit _docIdLimit; - AttributeCollectionSpec::UP createAttributeSpec(const AttributesConfig &attrCfg, const AllocStrategy& alloc_strategy, SerialNum serialNum) const; + std::unique_ptr<AttributeCollectionSpec> createAttributeSpec(const AttributesConfig &attrCfg, const AllocStrategy& alloc_strategy, SerialNum serialNum) const; AttributeManager::SP getAndResetInitAttributeManager(); virtual IFlushTargetList getFlushTargetsInternal() override; void reconfigureAttributeMetrics(const IAttributeManager &newMgr, const IAttributeManager &oldMgr); diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp index ee811f93789..4c04dc694fa 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp @@ -2,6 +2,7 @@ #include "fast_access_doc_subdb_configurer.h" #include "i_attribute_writer_factory.h" +#include "documentdbconfig.h" #include <vespa/searchcore/proton/attribute/attribute_writer.h> #include <vespa/searchcore/proton/common/document_type_inspector.h> #include <vespa/searchcore/proton/common/indexschema_inspector.h> @@ -46,10 +47,11 @@ FastAccessDocSubDBConfigurer::~FastAccessDocSubDBConfigurer() = default; IReprocessingInitializer::UP FastAccessDocSubDBConfigurer::reconfigure(const DocumentDBConfig &newConfig, const DocumentDBConfig &oldConfig, - const AttributeCollectionSpec &attrSpec) + AttributeCollectionSpec && attrSpec) { FastAccessFeedView::SP oldView = _feedView.get(); - IAttributeWriter::SP writer = _factory->create(oldView->getAttributeWriter(), attrSpec); + search::SerialNum currentSerialNum = attrSpec.getCurrentSerialNum(); + IAttributeWriter::SP writer = _factory->create(oldView->getAttributeWriter(), std::move(attrSpec)); reconfigureFeedView(*oldView, newConfig.getSchemaSP(), newConfig.getDocumentTypeRepoSP(), writer); const document::DocumentType *newDocType = newConfig.getDocumentType(); @@ -61,7 +63,7 @@ FastAccessDocSubDBConfigurer::reconfigure(const DocumentDBConfig &newConfig, return std::make_unique<AttributeReprocessingInitializer> (ARIConfig(writer->getAttributeManager(), *newConfig.getSchemaSP()), ARIConfig(oldView->getAttributeWriter()->getAttributeManager(), *oldConfig.getSchemaSP()), - inspector, oldIndexschemaInspector, _subDbName, attrSpec.getCurrentSerialNum()); + inspector, oldIndexschemaInspector, _subDbName, currentSerialNum); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.h index 6cde1a71794..9435cc14e30 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.h +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.h @@ -35,7 +35,7 @@ public: IReprocessingInitializer::UP reconfigure(const DocumentDBConfig &newConfig, const DocumentDBConfig &oldConfig, - const AttributeCollectionSpec &attrSpec); + AttributeCollectionSpec && attrSpec); }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp index 8ffa6c0a84d..de0d2e65acd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp @@ -2,6 +2,7 @@ #include "fileconfigmanager.h" #include "bootstrapconfig.h" +#include "documentdbconfigmanager.h" #include <vespa/searchcore/proton/common/hw_info_sampler.h> #include <vespa/config/print/fileconfigwriter.h> #include <vespa/config-bucketspaces.h> @@ -17,6 +18,7 @@ #include <vespa/config-summary.h> #include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/config/helper/configgetter.hpp> +#include <vespa/fastos/file.h> #include <sstream> #include <cassert> #include <fcntl.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.h b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.h index cca68d7d84c..36fb2c5492b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.h +++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.h @@ -2,11 +2,12 @@ #pragma once #include "configstore.h" -#include "documentdbconfigmanager.h" #include <vespa/searchlib/common/indexmetainfo.h> #include <vespa/searchlib/common/serialnum.h> #include <vespa/vespalib/objects/nbostream.h> +class FNET_Transport; + namespace proton { class FileConfigManager : public ConfigStore { @@ -47,7 +48,7 @@ public: * resulting config snapshot. */ void loadConfig(const DocumentDBConfig ¤tSnapshot, SerialNum serialNum, - DocumentDBConfig::SP &loadedSnapshot) override; + std::shared_ptr<DocumentDBConfig> &loadedSnapshot) override; void removeInvalid() override; void prune(SerialNum serialNum) override; diff --git a/searchcore/src/vespa/searchcore/proton/server/i_attribute_writer_factory.h b/searchcore/src/vespa/searchcore/proton/server/i_attribute_writer_factory.h index da1f76b7a38..7662d1cb164 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_attribute_writer_factory.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_attribute_writer_factory.h @@ -12,9 +12,9 @@ namespace proton { struct IAttributeWriterFactory { using UP = std::unique_ptr<IAttributeWriterFactory>; - virtual ~IAttributeWriterFactory() {} + virtual ~IAttributeWriterFactory() = default; virtual IAttributeWriter::SP create(const IAttributeWriter::SP &old, - const AttributeCollectionSpec &attrSpec) const = 0; + AttributeCollectionSpec && attrSpec) const = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/matchview.h b/searchcore/src/vespa/searchcore/proton/server/matchview.h index ee0c10cd333..f7bf0abff39 100644 --- a/searchcore/src/vespa/searchcore/proton/server/matchview.h +++ b/searchcore/src/vespa/searchcore/proton/server/matchview.h @@ -9,7 +9,6 @@ #include <vespa/searchcore/proton/matching/match_context.h> #include <vespa/searchcore/proton/summaryengine/isearchhandler.h> #include <vespa/searchcorespi/index/indexsearchable.h> -#include <vespa/searchlib/attribute/attributevector.h> namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.cpp b/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.cpp index 0681ff2c40b..989254c7aa1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "memoryconfigstore.h" +#include "documentdbconfig.h" #include <cassert> #include <vespa/log/log.h> @@ -38,12 +39,12 @@ void MemoryConfigStore::saveConfig(const DocumentDBConfig &config, SerialNum serial) { - _maps->configs[serial].reset(new DocumentDBConfig(config)); + _maps->configs[serial] = std::make_shared<DocumentDBConfig>(config); _maps->_valid.insert(serial); } void MemoryConfigStore::loadConfig(const DocumentDBConfig &, SerialNum serial, - DocumentDBConfig::SP &loaded_config) + std::shared_ptr<DocumentDBConfig> &loaded_config) { assert(hasValidSerial(serial)); loaded_config = _maps->configs[serial]; @@ -88,7 +89,7 @@ MemoryConfigStores::getConfigStore(const std::string &type) { if (!_config_maps[type].get()) { _config_maps[type].reset(new ConfigMaps); } - return ConfigStore::UP(new MemoryConfigStore(_config_maps[type])); + return std::make_unique<MemoryConfigStore>(_config_maps[type]); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.h b/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.h index 735006389dc..e42328d9fd1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.h +++ b/searchcore/src/vespa/searchcore/proton/server/memoryconfigstore.h @@ -3,15 +3,15 @@ #pragma once #include "configstore.h" -#include "documentdbconfig.h" #include <vespa/searchcommon/common/schema.h> #include <map> +#include <set> namespace proton { struct ConfigMaps { typedef std::shared_ptr<ConfigMaps> SP; - std::map<search::SerialNum, DocumentDBConfig::SP> configs; + std::map<search::SerialNum, std::shared_ptr<DocumentDBConfig>> configs; std::set<search::SerialNum> _valid; ~ConfigMaps(); }; @@ -32,7 +32,7 @@ public: bool hasValidSerial(SerialNum serial) const override; SerialNum getPrevValidSerial(SerialNum serial) const override; void saveConfig(const DocumentDBConfig &config, SerialNum serial) override; - void loadConfig(const DocumentDBConfig &, SerialNum serial, DocumentDBConfig::SP &loaded_config) override; + void loadConfig(const DocumentDBConfig &, SerialNum serial, std::shared_ptr<DocumentDBConfig> &loaded_config) override; void removeInvalid() override; void prune(SerialNum serial) override; void serializeConfig(SerialNum, vespalib::nbostream &) override; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index c6e352e6df7..fd6f785fa58 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -15,6 +15,7 @@ #include "resource_usage_explorer.h" #include "searchhandlerproxy.h" #include "simpleflush.h" +#include "documentdbconfig.h" #include <vespa/document/base/exceptions.h> #include <vespa/document/datatype/documenttype.h> @@ -46,6 +47,7 @@ #include <vespa/vespalib/util/random.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/fastos/file.h> #ifdef __linux__ #include <malloc.h> #endif @@ -398,7 +400,7 @@ Proton::addDocumentDB(const DocTypeName &docTypeName, document::BucketSpace bucketSpace, const vespalib::string &configId, const BootstrapConfig::SP &bootstrapConfig, - const DocumentDBConfig::SP &documentDBConfig, + const std::shared_ptr<DocumentDBConfig> &documentDBConfig, InitializeThreads initializeThreads) { try { @@ -586,7 +588,7 @@ DocumentDB::SP Proton::addDocumentDB(const document::DocumentType &docType, document::BucketSpace bucketSpace, const BootstrapConfig::SP &bootstrapConfig, - const DocumentDBConfig::SP &documentDBConfig, + const std::shared_ptr<DocumentDBConfig> &documentDBConfig, InitializeThreads initializeThreads) { const ProtonConfig &config(bootstrapConfig->getProtonConfig()); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp index b1a7bd87067..64a09a0bb25 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp @@ -4,6 +4,7 @@ #include "bootstrapconfig.h" #include "proton_config_snapshot.h" #include "i_proton_configurer.h" +#include "documentdbconfigmanager.h" #include <vespa/config/common/exceptions.h> #include <vespa/vespalib/util/exceptions.h> #include <thread> diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h index 77184f01604..a0c7c276e62 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h @@ -3,18 +3,21 @@ #pragma once #include "bootstrapconfigmanager.h" -#include "documentdbconfigmanager.h" #include "i_document_db_config_owner.h" #include <vespa/fastos/thread.h> #include <vespa/searchcore/proton/common/doctypename.h> #include <vespa/config/retriever/configretriever.h> #include <vespa/config/subscription/configuri.h> +#include <deque> + +class FNET_Transport; namespace document { class DocumentTypeRepo; } namespace proton { class BootstrapConfig; +class DocumentDBConfigManager; class IProtonConfigurer; /** @@ -46,7 +49,7 @@ public: void Run(FastOS_ThreadInterface * thread, void *arg) override; private: - typedef std::map<DocTypeName, DocumentDBConfigManager::SP> DBManagerMap; + typedef std::map<DocTypeName, std::shared_ptr<DocumentDBConfigManager>> DBManagerMap; using OldDocumentTypeRepo = std::pair<vespalib::steady_time, std::shared_ptr<const document::DocumentTypeRepo>>; using lock_guard = std::lock_guard<std::mutex>; diff --git a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp index a2817601c55..a320dd0f9ac 100644 --- a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp @@ -4,6 +4,7 @@ #include "proton.h" #include <vespa/searchcore/proton/matchengine/matchengine.h> #include <vespa/vespalib/util/lambdatask.h> +#include <vespa/vespalib/util/compressionconfig.h> #include <vespa/fnet/frt/supervisor.h> #include <vespa/fnet/transport.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp index 9c8feebef50..a182c4efb5c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "reconfig_params.h" #include "searchable_doc_subdb_configurer.h" +#include "reconfig_params.h" #include <vespa/searchcore/proton/matching/matcher.h> #include <vespa/searchcore/proton/attribute/attribute_writer.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> @@ -144,7 +144,7 @@ reconfigure(const DocumentDBConfig &newConfig, { assert(!params.shouldAttributeManagerChange()); AttributeCollectionSpec attrSpec(AttributeCollectionSpec::AttributeList(), 0, 0); - reconfigure(newConfig, oldConfig, attrSpec, params, resolver); + reconfigure(newConfig, oldConfig, std::move(attrSpec), params, resolver); } namespace { @@ -174,14 +174,14 @@ createAttributeReprocessingInitializer(const DocumentDBConfig &newConfig, IReprocessingInitializer::UP SearchableDocSubDBConfigurer::reconfigure(const DocumentDBConfig &newConfig, const DocumentDBConfig &oldConfig, - const AttributeCollectionSpec &attrSpec, + AttributeCollectionSpec && attrSpec, const ReconfigParams ¶ms, IDocumentDBReferenceResolver &resolver) { bool shouldMatchViewChange = false; bool shouldSearchViewChange = false; bool shouldFeedViewChange = params.shouldSchemaChange(); - + search::SerialNum currentSerialNum = attrSpec.getCurrentSerialNum(); SearchView::SP searchView = _searchView.get(); Matchers::SP matchers = searchView->getMatchers(); if (params.shouldMatchersChange()) { @@ -197,7 +197,7 @@ SearchableDocSubDBConfigurer::reconfigure(const DocumentDBConfig &newConfig, IAttributeManager::SP attrMgr = searchView->getAttributeManager(); IAttributeWriter::SP attrWriter = _feedView.get()->getAttributeWriter(); if (params.shouldAttributeManagerChange()) { - IAttributeManager::SP newAttrMgr = attrMgr->create(attrSpec); + IAttributeManager::SP newAttrMgr = attrMgr->create(std::move(attrSpec)); newAttrMgr->setImportedAttributes(resolver.resolve(*newAttrMgr, *attrMgr, searchView->getDocumentMetaStore(), newConfig.getMaintenanceConfigSP()->getVisibilityDelay())); @@ -209,7 +209,7 @@ SearchableDocSubDBConfigurer::reconfigure(const DocumentDBConfig &newConfig, attrWriter = newAttrWriter; shouldFeedViewChange = true; initializer = createAttributeReprocessingInitializer(newConfig, newAttrMgr, oldConfig, oldAttrMgr, - _subDbName, attrSpec.getCurrentSerialNum()); + _subDbName, currentSerialNum); } else if (params.shouldAttributeWriterChange()) { attrWriter = std::make_shared<AttributeWriter>(attrMgr); shouldFeedViewChange = true; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h index f05a8e39ef4..b19ebb1546e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h @@ -2,7 +2,6 @@ #pragma once -#include "documentdbconfigmanager.h" #include "isummaryadapter.h" #include "matchers.h" #include "matchview.h" @@ -23,6 +22,10 @@ #include <vespa/vespalib/util/varholder.h> #include <vespa/searchcore/proton/reference/i_document_db_reference_resolver.h> +namespace proton::matching { + class RankingExpressions; + class OnnxModels; +} namespace proton { struct IDocumentDBReferenceResolver; @@ -87,7 +90,7 @@ public: IReprocessingInitializer::UP reconfigure(const DocumentDBConfig &newConfig, const DocumentDBConfig &oldConfig, - const AttributeCollectionSpec &attrSpec, + AttributeCollectionSpec && attrSpec, const ReconfigParams ¶ms, IDocumentDBReferenceResolver &resolver); }; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index 6e87f33e1c6..ca731d1e379 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -147,10 +147,10 @@ SearchableDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const } if (params.shouldAttributeManagerChange()) { proton::IAttributeManager::SP oldMgr = getAttributeManager(); - AttributeCollectionSpec::UP attrSpec = + std::unique_ptr<AttributeCollectionSpec> attrSpec = createAttributeSpec(newConfigSnapshot.getAttributesConfig(), alloc_strategy, serialNum); IReprocessingInitializer::UP initializer = - _configurer.reconfigure(newConfigSnapshot, oldConfigSnapshot, *attrSpec, params, resolver); + _configurer.reconfigure(newConfigSnapshot, oldConfigSnapshot, std::move(*attrSpec), params, resolver); if (initializer && initializer->hasReprocessors()) { tasks.emplace_back(createReprocessingTask(*initializer, newConfigSnapshot.getDocumentTypeRepoSP())); } diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h index e6da0e958e8..08b34a3fbe6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h @@ -7,6 +7,7 @@ #include "searchview.h" #include "summaryadapter.h" #include "igetserialnum.h" +#include "document_db_flush_config.h" #include <vespa/eval/eval/value_cache/constant_tensor_loader.h> #include <vespa/eval/eval/value_cache/constant_value_cache.h> #include <vespa/searchcore/proton/attribute/attributemanager.h> diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index 2736a1eaa6f..c3f9bba24e6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -24,6 +24,7 @@ #include <vespa/searchlib/common/flush_token.h> #include <vespa/searchlib/docstore/document_store_visitor_progress.h> #include <vespa/searchlib/util/fileheadertk.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/exceptions.h> @@ -252,7 +253,7 @@ createDocumentMetaStoreInitializer(const AllocStrategy& alloc_strategy, { GrowStrategy grow = alloc_strategy.get_grow_strategy(); // Amortize memory spike cost over N docs - grow.setDocsGrowDelta(grow.getDocsGrowDelta() + alloc_strategy.get_amortize_count()); + grow.setGrowDelta(grow.getGrowDelta() + alloc_strategy.get_amortize_count()); vespalib::string baseDir(_baseDir + "/documentmetastore"); vespalib::string name = DocumentMetaStore::getFixedName(); vespalib::string attrFileName = baseDir + "/" + name; // XXX: Wrong @@ -456,7 +457,7 @@ StoreOnlyDocSubDB::reconfigure(const search::LogDocumentStore::Config & config, auto cfg = _dms->getConfig(); GrowStrategy grow = alloc_strategy.get_grow_strategy(); // Amortize memory spike cost over N docs - grow.setDocsGrowDelta(grow.getDocsGrowDelta() + alloc_strategy.get_amortize_count()); + grow.setGrowDelta(grow.getGrowDelta() + alloc_strategy.get_amortize_count()); cfg.setGrowStrategy(grow); cfg.setCompactionStrategy(computeCompactionStrategy(alloc_strategy.get_compaction_strategy())); _dms->update_config(cfg); // Update grow and compaction config diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h index d43b865c000..178ed73aef5 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "documentdbconfig.h" #include "idocumentsubdb.h" #include "storeonlyfeedview.h" #include "summaryadapter.h" @@ -16,11 +15,13 @@ #include <vespa/searchcore/proton/persistenceengine/i_document_retriever.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/vespalib/util/varholder.h> +#include <vespa/vespalib/datastore/compaction_strategy.h> #include <mutex> namespace proton { class AllocStrategy; +class DocumentDBConfig; struct DocumentDBTaggedMetrics; class DocumentMetaStoreInitializerResult; class FeedHandler; diff --git a/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt index 5df9060ea07..25570c4ad0c 100644 --- a/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchcore_test STATIC SOURCES + attribute_utils.cpp attribute_vectors.cpp bucketfactory.cpp buckethandler.cpp diff --git a/searchcore/src/vespa/searchcore/proton/test/attribute_utils.cpp b/searchcore/src/vespa/searchcore/proton/test/attribute_utils.cpp new file mode 100644 index 00000000000..9dde79b342a --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/test/attribute_utils.cpp @@ -0,0 +1,76 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "attribute_utils.h" +#include <vespa/vespalib/util/hdr_abort.h> +#include <vespa/searchcommon/attribute/config.h> +#include <vespa/searchlib/attribute/integerbase.h> + +using search::attribute::Config; +using search::attribute::BasicType; +using search::attribute::CollectionType; + +namespace proton::test { + +void +AttributeUtils::fillAttribute(search::AttributeVector & attr, uint32_t numDocs, int64_t value, uint64_t lastSyncToken) { + search::IntegerAttribute &ia = static_cast<search::IntegerAttribute &>(attr); + ia.addDocs(numDocs); + for (uint32_t i = 1; i < ia.getNumDocs(); ++i) { + ia.update(i, value); + } + ia.commit(search::CommitParam(lastSyncToken)); +} + +void +AttributeUtils::fillAttribute(search::AttributeVector & attr, uint32_t from, uint32_t to, int64_t value, uint64_t lastSyncToken) { + search::IntegerAttribute &ia = static_cast<search::IntegerAttribute &>(attr); + while (ia.getNumDocs() < to) { + uint32_t docId; + if (!ia.addDoc(docId)) { HDR_ABORT("should not be reached"); } + } + for (uint32_t i = from; i < to; ++i) { + ia.update(i, value); + } + ia.commit(search::CommitParam(lastSyncToken)); +} + +const Config & +AttributeUtils::getInt32Config() { + static Config cfg(BasicType::INT32); + return cfg; +} + +const Config & +AttributeUtils::getInt32ArrayConfig() { + static Config cfg(BasicType::INT32, CollectionType::ARRAY); + return cfg; +} + +const Config & +AttributeUtils::getStringConfig() { + static Config cfg(BasicType::STRING); + return cfg; +} + +const Config & +AttributeUtils::getPredicateConfig() { + static Config cfg(BasicType::PREDICATE); + return cfg; +} + +namespace { + +Config tensorConfig() { + return Config(BasicType::TENSOR).setTensorType(vespalib::eval::ValueType::from_spec("tensor(x{},y{})")); +} + +} + +const Config & +AttributeUtils::getTensorConfig() { + static Config cfg = tensorConfig(); + return cfg; +} + +} + diff --git a/searchcore/src/vespa/searchcore/proton/test/attribute_utils.h b/searchcore/src/vespa/searchcore/proton/test/attribute_utils.h index c296129984b..bb0a7318b21 100644 --- a/searchcore/src/vespa/searchcore/proton/test/attribute_utils.h +++ b/searchcore/src/vespa/searchcore/proton/test/attribute_utils.h @@ -1,51 +1,23 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/util/hdr_abort.h> -#include <vespa/searchcommon/attribute/config.h> -#include <vespa/searchlib/attribute/integerbase.h> +#include <cstdint> + +namespace search { class AttributeVector; } +namespace search::attribute { class Config; } namespace proton::test { struct AttributeUtils { - static void fillAttribute(const search::AttributeVector::SP &attr, - uint32_t numDocs, int64_t value, uint64_t lastSyncToken) { - search::IntegerAttribute &ia = static_cast<search::IntegerAttribute &>(*attr); - ia.addDocs(numDocs); - for (uint32_t i = 1; i < ia.getNumDocs(); ++i) { - ia.update(i, value); - } - ia.commit(search::CommitParam(lastSyncToken)); - } - static void fillAttribute(const search::AttributeVector::SP &attr, - uint32_t from, uint32_t to, int64_t value, uint64_t lastSyncToken) { - search::IntegerAttribute &ia = static_cast<search::IntegerAttribute &>(*attr); - while (ia.getNumDocs() < to) { - uint32_t docId; - if (!ia.addDoc(docId)) { HDR_ABORT("should not be reached"); } - } - for (uint32_t i = from; i < to; ++i) { - ia.update(i, value); - } - ia.commit(search::CommitParam(lastSyncToken)); - } - static search::attribute::Config getInt32Config() { - return search::attribute::Config(search::attribute::BasicType::INT32); - } - static search::attribute::Config getInt32ArrayConfig() { - return search::attribute::Config(search::attribute::BasicType::INT32, - search::attribute::CollectionType::ARRAY); - } - static search::attribute::Config getStringConfig() { - return search::attribute::Config(search::attribute::BasicType::STRING); - } - static search::attribute::Config getPredicateConfig() { - return search::attribute::Config(search::attribute::BasicType::PREDICATE); - } - static search::attribute::Config getTensorConfig() { - return search::attribute::Config(search::attribute::BasicType::TENSOR).setTensorType(vespalib::eval::ValueType::from_spec("tensor(x{},y{})")); - } + using Config = search::attribute::Config; + static void fillAttribute(search::AttributeVector &attr, uint32_t numDocs, int64_t value, uint64_t lastSyncToken); + static void fillAttribute(search::AttributeVector &attr, uint32_t from, uint32_t to, int64_t value, uint64_t lastSyncToken); + static const Config & getInt32Config(); + static const Config & getInt32ArrayConfig(); + static const Config & getStringConfig(); + static const Config & getPredicateConfig(); + static const Config & getTensorConfig(); }; } diff --git a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp index 301f2a97d14..5c57f821fe1 100644 --- a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp +++ b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp @@ -48,15 +48,15 @@ DocumentDBConfigBuilder::DocumentDBConfigBuilder(int64_t generation, _schema(schema), _maintenance(std::make_shared<DocumentDBMaintenanceConfig>()), _store(), - _threading_service_config(std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make())), - _alloc_config(std::make_shared<const AllocConfig>()), + _threading_service_config(ThreadingServiceConfig::make()), + _alloc_config(AllocConfig::makeDefault()), _configId(configId), _docTypeName(docTypeName) { } DocumentDBConfigBuilder::DocumentDBConfigBuilder(const DocumentDBConfig &cfg) - : _generation(cfg.getGeneration()), + : _generation(cfg.getGeneration()), _rankProfiles(cfg.getRankProfilesConfigSP()), _rankingConstants(cfg.getRankingConstantsSP()), _rankingExpressions(cfg.getRankingExpressionsSP()), @@ -73,13 +73,13 @@ DocumentDBConfigBuilder::DocumentDBConfigBuilder(const DocumentDBConfig &cfg) _schema(cfg.getSchemaSP()), _maintenance(cfg.getMaintenanceConfigSP()), _store(cfg.getStoreConfig()), - _threading_service_config(cfg.get_threading_service_config_shared_ptr()), - _alloc_config(cfg.get_alloc_config_shared_ptr()), + _threading_service_config(cfg.get_threading_service_config()), + _alloc_config(cfg.get_alloc_config()), _configId(cfg.getConfigId()), _docTypeName(cfg.getDocTypeName()) {} -DocumentDBConfigBuilder::~DocumentDBConfigBuilder() {} +DocumentDBConfigBuilder::~DocumentDBConfigBuilder() = default; DocumentDBConfig::SP DocumentDBConfigBuilder::build() diff --git a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h index 1d927e93d16..c8d49c3175b 100644 --- a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h +++ b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h @@ -28,8 +28,8 @@ private: search::index::Schema::SP _schema; DocumentDBConfig::MaintenanceConfigSP _maintenance; search::LogDocumentStore::Config _store; - std::shared_ptr<const ThreadingServiceConfig> _threading_service_config; - std::shared_ptr<const AllocConfig> _alloc_config; + const ThreadingServiceConfig _threading_service_config; + const AllocConfig _alloc_config; vespalib::string _configId; vespalib::string _docTypeName; diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h index 0e2491e62fa..75bb3291dd0 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h @@ -1,10 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/util/hdr_abort.h> -#include <vespa/searchlib/test/mock_attribute_manager.h> #include <vespa/searchcore/proton/attribute/i_attribute_manager.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> +#include <vespa/searchlib/test/mock_attribute_manager.h> +#include <vespa/vespalib/util/hdr_abort.h> +#include <cassert> namespace proton::test { @@ -48,7 +49,7 @@ public: search::attribute::IAttributeContext::UP createContext() const override { return _mock.createContext(); } - IAttributeManager::SP create(const AttributeCollectionSpec &) const override { + IAttributeManager::SP create(AttributeCollectionSpec &&) const override { return IAttributeManager::SP(); } std::vector<searchcorespi::IFlushTarget::SP> getFlushTargets() const override { diff --git a/searchlib/src/apps/tests/document_weight_attribute_lookup_stress_test.cpp b/searchlib/src/apps/tests/document_weight_attribute_lookup_stress_test.cpp index 0fb115a564e..181aaed4117 100644 --- a/searchlib/src/apps/tests/document_weight_attribute_lookup_stress_test.cpp +++ b/searchlib/src/apps/tests/document_weight_attribute_lookup_stress_test.cpp @@ -3,6 +3,7 @@ #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/integerbase.h> #include <vespa/searchlib/attribute/i_document_weight_attribute.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/gtest/gtest.h> #include <random> #include <chrono> diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index 07747a66892..d8f8cd9d67c 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -1,18 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/document/annotation/spanlist.h> -#include <vespa/document/annotation/spantree.h> -#include <vespa/document/datatype/documenttype.h> -#include <vespa/document/fieldvalue/document.h> -#include <vespa/document/fieldvalue/stringfieldvalue.h> -#include <vespa/document/repo/configbuilder.h> -#include <vespa/document/repo/documenttyperepo.h> -#include <vespa/document/repo/fixedtyperepo.h> + #include <vespa/searchlib/common/scheduletaskcallback.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> -#include <vespa/searchlib/index/i_field_length_inspector.h> #include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/queryeval/fake_requestcontext.h> @@ -20,11 +12,20 @@ #include <vespa/searchlib/queryeval/fake_searchable.h> #include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/searchlib/test/index/mock_field_length_inspector.h> +#include <vespa/document/annotation/spanlist.h> +#include <vespa/document/annotation/spantree.h> +#include <vespa/document/datatype/documenttype.h> +#include <vespa/document/fieldvalue/document.h> +#include <vespa/document/fieldvalue/stringfieldvalue.h> +#include <vespa/document/repo/configbuilder.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/repo/fixedtyperepo.h> #include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/log/log.h> LOG_SETUP("memoryindexstress_test"); diff --git a/searchlib/src/apps/vespa-attribute-inspect/vespa-attribute-inspect.cpp b/searchlib/src/apps/vespa-attribute-inspect/vespa-attribute-inspect.cpp index ffcb4ca4019..d9d6bb56152 100644 --- a/searchlib/src/apps/vespa-attribute-inspect/vespa-attribute-inspect.cpp +++ b/searchlib/src/apps/vespa-attribute-inspect/vespa-attribute-inspect.cpp @@ -4,6 +4,7 @@ #include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/data/fileheader.h> #include <fstream> @@ -39,7 +40,7 @@ void LoadAttribute::applyUpdate(const AttributePtr & ptr) { std::cout << "applyUpdate" << std::endl; - if (ptr->getClass().inherits(IntegerAttribute::classId)) { + if (ptr->isIntegerType()) { IntegerAttribute * a = static_cast<IntegerAttribute *>(ptr.get()); if (ptr->hasMultiValue()) { a->append(0, 123456789, 1); @@ -47,7 +48,7 @@ LoadAttribute::applyUpdate(const AttributePtr & ptr) a->update(0, 123456789); } a->commit(); - } else if (ptr->getClass().inherits(FloatingPointAttribute::classId)) { + } else if (ptr->isFloatingPointType()) { FloatingPointAttribute * a = static_cast<FloatingPointAttribute *>(ptr.get()); if (ptr->hasMultiValue()) { a->append(0, 123456789.5f, 1); @@ -55,7 +56,7 @@ LoadAttribute::applyUpdate(const AttributePtr & ptr) a->update(0, 123456789); } a->commit(); - } else if (ptr->getClass().inherits(StringAttribute::classId)) { + } else if (ptr->isStringType()) { StringAttribute * a = static_cast<StringAttribute *>(ptr.get()); if (ptr->hasMultiValue()) { a->append(0, "non-existing string value", 1); diff --git a/searchlib/src/tests/aggregator/perdocexpr.cpp b/searchlib/src/tests/aggregator/perdocexpr.cpp index 6e3e6bf3353..7bb10433f31 100644 --- a/searchlib/src/tests/aggregator/perdocexpr.cpp +++ b/searchlib/src/tests/aggregator/perdocexpr.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/aggregation/perdocexpression.h> #include <vespa/searchlib/attribute/extendableattributes.h> #include <vespa/searchlib/attribute/singleboolattribute.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/objects/objectdumper.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/document/base/testdocman.h> diff --git a/searchlib/src/tests/attribute/attribute_operation/attribute_operation_test.cpp b/searchlib/src/tests/attribute/attribute_operation/attribute_operation_test.cpp index baecb54fbec..7883a497d8b 100644 --- a/searchlib/src/tests/attribute/attribute_operation/attribute_operation_test.cpp +++ b/searchlib/src/tests/attribute/attribute_operation/attribute_operation_test.cpp @@ -4,6 +4,7 @@ #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/common/bitvector.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index 64edb30520b..6f86112a0f2 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -1,16 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/document/fieldvalue/intfieldvalue.h> -#include <vespa/document/fieldvalue/stringfieldvalue.h> -#include <vespa/document/update/arithmeticvalueupdate.h> -#include <vespa/document/update/assignvalueupdate.h> -#include <vespa/document/update/mapvalueupdate.h> + #include <vespa/searchlib/attribute/address_space_components.h> #include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/attribute/attributememorysavetarget.h> -#include <vespa/searchlib/attribute/attributevector.hpp> #include <vespa/searchlib/attribute/multienumattribute.hpp> #include <vespa/searchlib/attribute/multistringattribute.h> #include <vespa/searchlib/attribute/multivalueattribute.hpp> @@ -20,11 +15,17 @@ #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/test/weighted_type_test_utils.h> #include <vespa/searchlib/util/randomgenerator.h> +#include <vespa/document/fieldvalue/intfieldvalue.h> +#include <vespa/document/fieldvalue/stringfieldvalue.h> +#include <vespa/document/update/arithmeticvalueupdate.h> +#include <vespa/document/update/assignvalueupdate.h> +#include <vespa/document/update/mapvalueupdate.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/mmap_file_allocator_factory.h> #include <vespa/vespalib/util/round_up_to_page_size.h> #include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/fastos/file.h> #include <cmath> #include <iostream> @@ -302,7 +303,7 @@ AttributeTest::AttributeTest() = default; void AttributeTest::testBaseName() { - AttributeVector::BaseName v("attr1"); + attribute::BaseName v("attr1"); EXPECT_EQ(v.getAttributeName(), "attr1"); EXPECT_TRUE(v.getDirName().empty()); v = "attribute/attr1/attr1"; @@ -319,12 +320,10 @@ void AttributeTest::testBaseName() EXPECT_EQ(v.getDirName(), "index.1/1.ready/attribute/attr1/snapshot-X"); v = "/index.1/1.ready/attribute/attr1/snapshot-X/attr1"; EXPECT_EQ(v.getAttributeName(), "attr1"); - EXPECT_EQ(v.getDirName(), - "/index.1/1.ready/attribute/attr1/snapshot-X"); + EXPECT_EQ(v.getDirName(), "/index.1/1.ready/attribute/attr1/snapshot-X"); v = "xxxyyyy/zzz/index.1/1.ready/attribute/attr1/snapshot-X/attr1"; EXPECT_EQ(v.getAttributeName(), "attr1"); - EXPECT_EQ(v.getDirName(), - "xxxyyyy/zzz/index.1/1.ready/attribute/attr1/snapshot-X"); + EXPECT_EQ(v.getDirName(), "xxxyyyy/zzz/index.1/1.ready/attribute/attr1/snapshot-X"); } void AttributeTest::addDocs(const AttributePtr & v, size_t sz) @@ -1412,7 +1411,7 @@ AttributeTest::testArithmeticValueUpdate(const AttributePtr & ptr) ASSERT_TRUE(vec.update(0, 100)); EXPECT_TRUE(vec.apply(0, Arith(Arith::Div, 0))); ptr->commit(); - if (ptr->getClass().inherits(FloatingPointAttribute::classId)) { + if (ptr->isFloatingPointType()) { EXPECT_EQ(ptr->getStatus().getUpdateCount(), 86u); EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 66u); } else { // does not apply for interger attributes @@ -1427,7 +1426,7 @@ AttributeTest::testArithmeticValueUpdate(const AttributePtr & ptr) // try divide by zero with empty change vector EXPECT_TRUE(vec.apply(0, Arith(Arith::Div, 0))); ptr->commit(); - if (ptr->getClass().inherits(FloatingPointAttribute::classId)) { + if (ptr->isFloatingPointType()) { EXPECT_EQ(ptr->getStatus().getUpdateCount(), 87u); EXPECT_EQ(ptr->getStatus().getNonIdempotentUpdateCount(), 67u); } else { // does not apply for interger attributes @@ -1486,7 +1485,7 @@ AttributeTest::testArithmeticWithUndefinedValue(const AttributePtr & ptr, BaseTy std::vector<BufferType> buf(1); ptr->get(0, &buf[0], 1); - if (ptr->getClass().inherits(FloatingPointAttribute::classId)) { + if (ptr->isFloatingPointType()) { EXPECT_TRUE(std::isnan(buf[0])); } else { EXPECT_EQ(buf[0], after); diff --git a/searchlib/src/tests/attribute/benchmark/attributebenchmark.cpp b/searchlib/src/tests/attribute/benchmark/attributebenchmark.cpp index abdcd27b09a..cb6c66915f1 100644 --- a/searchlib/src/tests/attribute/benchmark/attributebenchmark.cpp +++ b/searchlib/src/tests/attribute/benchmark/attributebenchmark.cpp @@ -4,10 +4,10 @@ #include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/attribute/attributefactory.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/fastos/thread.h> #include <vespa/vespalib/util/signalhandler.h> #include <iostream> -#include <fstream> #include "attributesearcher.h" #include "attributeupdater.h" #include <sys/resource.h> diff --git a/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp b/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp index e34c0f52e41..d1dde5ef6ea 100644 --- a/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp @@ -4,19 +4,19 @@ #include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/attribute/attributefactory.h> -#include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/util/randomgenerator.h> #include <vespa/vespalib/util/compress.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> - #include <vespa/searchlib/attribute/i_document_weight_attribute.h> #include <vespa/searchlib/queryeval/document_weight_search_iterator.h> #include <vespa/searchlib/test/searchiteratorverifier.h> #include <vespa/searchlib/common/bitvectoriterator.h> +#include <vespa/searchlib/queryeval/executeinfo.h> #include <vespa/searchlib/parsequery/parse.h> +#include <vespa/searchcommon/attribute/config.h> +#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/log/log.h> -#include <vespa/searchlib/queryeval/executeinfo.h> LOG_SETUP("bitvector_test"); diff --git a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp index fbec89d27eb..c5d70109015 100644 --- a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp +++ b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp @@ -4,6 +4,7 @@ #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/attribute/integerbase.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/stringfmt.h> diff --git a/searchlib/src/tests/attribute/document_weight_iterator/document_weight_iterator_test.cpp b/searchlib/src/tests/attribute/document_weight_iterator/document_weight_iterator_test.cpp index 5e9626baad7..56fe791021e 100644 --- a/searchlib/src/tests/attribute/document_weight_iterator/document_weight_iterator_test.cpp +++ b/searchlib/src/tests/attribute/document_weight_iterator/document_weight_iterator_test.cpp @@ -6,13 +6,11 @@ #include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/searchlib/attribute/attributememorysavetarget.h> #include <vespa/searchlib/attribute/i_document_weight_attribute.h> -#include <vespa/searchlib/attribute/iattributemanager.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> -#include <vespa/searchlib/query/tree/location.h> -#include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/queryeval/document_weight_search_iterator.h> #include <vespa/searchlib/test/searchiteratorverifier.h> #include <vespa/searchlib/util/randomgenerator.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/test/insertion_operators.h> diff --git a/searchlib/src/tests/attribute/enum_attribute_compaction/enum_attribute_compaction_test.cpp b/searchlib/src/tests/attribute/enum_attribute_compaction/enum_attribute_compaction_test.cpp index b30b3e4eb71..f54e1efdf5c 100644 --- a/searchlib/src/tests/attribute/enum_attribute_compaction/enum_attribute_compaction_test.cpp +++ b/searchlib/src/tests/attribute/enum_attribute_compaction/enum_attribute_compaction_test.cpp @@ -1,11 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/gtest/gtest.h> -#include <vespa/searchcommon/attribute/attributecontent.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/integerbase.h> #include <vespa/searchlib/attribute/stringbase.h> #include <vespa/searchlib/test/weighted_type_test_utils.h> +#include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/searchcommon/attribute/config.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP("enum_attribute_compaction_test"); diff --git a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp index 9a4728edba8..92c3da40fe9 100644 --- a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp +++ b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp @@ -1,7 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/document/fieldvalue/intfieldvalue.h> -#include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attributefilesavetarget.h> @@ -17,13 +15,14 @@ #include <vespa/searchlib/util/bufferwriter.h> #include <vespa/searchlib/util/file_settings.h> #include <vespa/searchlib/util/randomgenerator.h> +#include <vespa/searchcommon/attribute/config.h> +#include <vespa/document/fieldvalue/intfieldvalue.h> +#include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/compress.h> -#include <vespa/vespalib/util/size_literals.h> - +#include <vespa/vespalib/stllike/asciistream.h> #include <limits> -#include <iostream> #include <cmath> using search::AttributeFactory; diff --git a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp index 167eaccc2f7..b9f3c23213e 100644 --- a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp +++ b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp @@ -429,7 +429,7 @@ struct MockAttributeVector : NotImplementedAttribute { long _return_value{1234}; MockAttributeVector() - : NotImplementedAttribute("mock", Config(BasicType::STRING)) { + : NotImplementedAttribute("mock") { } void set_received_args(DocId doc_id, void* ser_to, diff --git a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp index 566001cbe17..735ebcff6cf 100644 --- a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp +++ b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp @@ -3,6 +3,7 @@ #include <vespa/searchlib/attribute/multi_value_mapping.h> #include <vespa/searchlib/attribute/multi_value_mapping.hpp> #include <vespa/searchlib/attribute/not_implemented_attribute.h> +#include <vespa/vespalib/datastore/compaction_strategy.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/test/insertion_operators.h> @@ -49,7 +50,7 @@ class MyAttribute : public search::NotImplementedAttribute public: MyAttribute(MvMapping &mvMapping) - : NotImplementedAttribute("test", AttributeVector::Config()), + : NotImplementedAttribute("test"), _mvMapping(mvMapping) {} virtual bool addDoc(DocId &doc) override { diff --git a/searchlib/src/tests/attribute/multi_value_read_view/multi_value_read_view_test.cpp b/searchlib/src/tests/attribute/multi_value_read_view/multi_value_read_view_test.cpp index edebaa84ddb..2b47bf0a7c9 100644 --- a/searchlib/src/tests/attribute/multi_value_read_view/multi_value_read_view_test.cpp +++ b/searchlib/src/tests/attribute/multi_value_read_view/multi_value_read_view_test.cpp @@ -3,6 +3,7 @@ #include <vespa/document/base/documentid.h> #include <vespa/searchcommon/attribute/i_multi_value_attribute.h> #include <vespa/searchcommon/attribute/multi_value_traits.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/searchlib/attribute/extendableattributes.h> diff --git a/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp b/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp index e58954e1e93..55266841c67 100644 --- a/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp +++ b/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/gtest/gtest.h> -#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/document/update/arithmeticvalueupdate.h> #include <vespa/searchlib/attribute/attribute.h> @@ -13,12 +12,14 @@ #include <vespa/searchlib/attribute/multistringpostattribute.h> #include <vespa/searchlib/common/growablebitvector.h> #include <vespa/searchlib/queryeval/executeinfo.h> +#include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/parsequery/parse.h> -#include <vespa/searchlib/attribute/enumstore.hpp> +#include <vespa/searchcommon/attribute/config.h> +#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/compress.h> -#include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/fastos/file.h> +#include <vespa/searchlib/attribute/enumstore.hpp> #include <iostream> #include <vespa/log/log.h> @@ -476,7 +477,7 @@ PostingListAttributeTest::checkPostingList(const VectorType & vec, const std::ve } EXPECT_EQ(doc, docEnd); } else { - EXPECT_TRUE(has_bitvector && vec.getConfig().getEnableOnlyBitVector()); + EXPECT_TRUE(has_bitvector && vec.getEnableOnlyBitVector()); numHits = postingList.getBitVectorEntry(find_result.second)->_bv->reader().countTrueBits(); } if (has_bitvector) { diff --git a/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp b/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp index 5cee31bf909..b5101f1ea58 100644 --- a/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp +++ b/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp @@ -11,10 +11,10 @@ #include <vespa/searchlib/queryeval/fake_result.h> #include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/searchlib/test/mock_gid_to_lid_mapping.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/test/insertion_operators.h> -#include <vespa/vespalib/util/traits.h> #include <vespa/log/log.h> LOG_SETUP("reference_attribute_test"); diff --git a/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp b/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp index 48f56394800..6df60f2ac4c 100644 --- a/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp @@ -279,7 +279,7 @@ void add_docs(AttributeVector *attr, size_t n) { AttributeVector::DocId docid; for (size_t i = 0; i < n; ++i) { attr->addDoc(docid); - if (attr->inherits(PredicateAttribute::classId)) { + if (attr->isPredicateType()) { const_cast<uint8_t *>(static_cast<PredicateAttribute *>(attr)->getMinFeatureVector().first)[docid] = 0; } } diff --git a/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp b/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp index 0bdd4f5b4dc..26fbed81146 100644 --- a/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attribute_weighted_set_blueprint_test.cpp @@ -4,7 +4,6 @@ #include <vespa/searchlib/attribute/attribute_weighted_set_blueprint.h> #include <vespa/searchlib/attribute/attributecontext.h> #include <vespa/searchlib/attribute/attributevector.h> -#include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/searchlib/attribute/extendableattributes.h> #include <vespa/searchlib/attribute/singlestringattribute.h> #include <vespa/searchlib/attribute/attributefactory.h> @@ -15,6 +14,7 @@ #include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/test/mock_attribute_manager.h> #include <vespa/searchlib/attribute/enumstore.hpp> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/log/log.h> LOG_SETUP("attribute_weighted_set_blueprint_test"); diff --git a/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp b/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp index 1c869716d1e..67bd0286884 100644 --- a/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp +++ b/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp @@ -1,9 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/eval/value.h> -#include <vespa/eval/eval/value_codec.h> -#include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchlib/attribute/attribute_blueprint_factory.h> #include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/searchlib/attribute/attributecontext.h> @@ -18,6 +14,11 @@ #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/searchlib/queryeval/nearest_neighbor_blueprint.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> +#include <vespa/searchcommon/attribute/iattributecontext.h> +#include <vespa/searchcommon/attribute/config.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/value_codec.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp b/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp index 947e4aa30c2..6390e30adff 100644 --- a/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp +++ b/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp @@ -16,9 +16,11 @@ #include <vespa/searchlib/queryeval/hitcollector.h> #include <vespa/searchlib/queryeval/simpleresult.h> #include <vespa/searchlib/test/searchiteratorverifier.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/compress.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/stllike/asciistream.h> #include <set> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/attribute/searchcontextelementiterator/searchcontextelementiterator_test.cpp b/searchlib/src/tests/attribute/searchcontextelementiterator/searchcontextelementiterator_test.cpp index 14861819bb5..06ca299b9ba 100644 --- a/searchlib/src/tests/attribute/searchcontextelementiterator/searchcontextelementiterator_test.cpp +++ b/searchlib/src/tests/attribute/searchcontextelementiterator/searchcontextelementiterator_test.cpp @@ -7,7 +7,7 @@ #include <vespa/searchlib/queryeval/fake_search.h> #include <vespa/searchlib/query/query_term_simple.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> -# +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/gtest/gtest.h> diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index eb73edbde46..e8450104963 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -1,11 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/document/base/exceptions.h> -#include <vespa/eval/eval/simple_value.h> -#include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/eval/value.h> -#include <vespa/eval/eval/test/value_compare.h> -#include <vespa/fastos/file.h> #include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/queryeval/nearest_neighbor_blueprint.h> @@ -23,6 +17,7 @@ #include <vespa/searchlib/tensor/tensor_attribute.h> #include <vespa/searchlib/test/directory_handler.h> #include <vespa/searchlib/util/fileutil.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/test/insertion_operators.h> @@ -30,6 +25,12 @@ #include <vespa/vespalib/util/mmap_file_allocator_factory.h> #include <vespa/searchlib/util/bufferwriter.h> #include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/document/base/exceptions.h> +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/test/value_compare.h> +#include <vespa/fastos/file.h> #include <vespa/log/log.h> LOG_SETUP("tensorattribute_test"); diff --git a/searchlib/src/tests/common/location_iterator/location_iterator_test.cpp b/searchlib/src/tests/common/location_iterator/location_iterator_test.cpp index fff296f35ed..44a2f1697cd 100644 --- a/searchlib/src/tests/common/location_iterator/location_iterator_test.cpp +++ b/searchlib/src/tests/common/location_iterator/location_iterator_test.cpp @@ -3,11 +3,11 @@ #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/attribute/attributevector.h> -#include <vespa/searchlib/attribute/iattributemanager.h> #include <vespa/searchlib/attribute/integerbase.h> #include <vespa/searchlib/common/geo_location.h> #include <vespa/searchlib/common/location.h> #include <vespa/searchlib/common/locationiterators.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp index afd2c2a2fac..059f198a070 100644 --- a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp +++ b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp @@ -1,6 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/searchcommon/common/undefinedvalues.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attributecontext.h> #include <vespa/searchlib/attribute/attributemanager.h> @@ -11,6 +10,8 @@ #include <vespa/searchlib/attribute/singleboolattribute.h> #include <vespa/searchlib/expression/attributenode.h> #include <vespa/searchlib/expression/resultvector.h> +#include <vespa/searchcommon/attribute/config.h> +#include <vespa/searchcommon/common/undefinedvalues.h> #include <vespa/searchlib/test/make_attribute_map_lookup_node.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/features/euclidean_distance/euclidean_distance_test.cpp b/searchlib/src/tests/features/euclidean_distance/euclidean_distance_test.cpp index 51609ef7f67..eed59fdde5e 100644 --- a/searchlib/src/tests/features/euclidean_distance/euclidean_distance_test.cpp +++ b/searchlib/src/tests/features/euclidean_distance/euclidean_distance_test.cpp @@ -7,11 +7,11 @@ #include <vespa/searchlib/attribute/floatbase.h> #include <vespa/searchlib/features/setup.h> #include <vespa/searchlib/fef/test/indexenvironment.h> -#include <vespa/searchlib/fef/test/indexenvironmentbuilder.h> -#include <vespa/searchlib/fef/test/queryenvironment.h> #include <vespa/searchlib/fef/test/ftlib.h> #include <vespa/searchlib/features/euclidean_distance_feature.h> #include <vespa/searchlib/fef/fef.h> +#include <vespa/searchcommon/attribute/config.h> + using search::feature_t; using namespace search::fef; diff --git a/searchlib/src/tests/features/featurebenchmark.cpp b/searchlib/src/tests/features/featurebenchmark.cpp index 5a70f1752fc..a3188ce21fa 100644 --- a/searchlib/src/tests/features/featurebenchmark.cpp +++ b/searchlib/src/tests/features/featurebenchmark.cpp @@ -12,6 +12,7 @@ #include <vespa/searchlib/fef/test/plugin/setup.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/searchlib/fef/test/ftlib.h> +#include <vespa/searchcommon/attribute/config.h> #include <fstream> #include <iomanip> diff --git a/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/internal_max_reduce_prod_join_feature_test.cpp b/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/internal_max_reduce_prod_join_feature_test.cpp index fba88644dcb..532f646ae4d 100644 --- a/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/internal_max_reduce_prod_join_feature_test.cpp +++ b/searchlib/src/tests/features/internal_max_reduce_prod_join_feature/internal_max_reduce_prod_join_feature_test.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/fef/test/ftlib.h> #include <vespa/searchlib/features/internal_max_reduce_prod_join_feature.h> #include <vespa/searchlib/attribute/attribute.h> +#include <vespa/searchcommon/attribute/config.h> using search::feature_t; using namespace search::fef; diff --git a/searchlib/src/tests/features/prod_features.cpp b/searchlib/src/tests/features/prod_features.cpp index 002cfdc7f3b..2c04a326e3e 100644 --- a/searchlib/src/tests/features/prod_features.cpp +++ b/searchlib/src/tests/features/prod_features.cpp @@ -41,6 +41,7 @@ #include <vespa/searchlib/fef/queryproperties.h> #include <vespa/searchlib/fef/test/plugin/setup.h> #include <vespa/searchlib/fef/test/dummy_dependency_handler.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/geo/zcurve.h> diff --git a/searchlib/src/tests/features/prod_features_attributematch.cpp b/searchlib/src/tests/features/prod_features_attributematch.cpp index 1db6212abef..fd6fdca96fc 100644 --- a/searchlib/src/tests/features/prod_features_attributematch.cpp +++ b/searchlib/src/tests/features/prod_features_attributematch.cpp @@ -4,6 +4,7 @@ #include <vespa/searchlib/features/attributematchfeature.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/log/log.h> LOG_SETUP(".prod_features_attributematch"); diff --git a/searchlib/src/tests/features/tensor/tensor_test.cpp b/searchlib/src/tests/features/tensor/tensor_test.cpp index a8c4d6714d8..54807273aea 100644 --- a/searchlib/src/tests/features/tensor/tensor_test.cpp +++ b/searchlib/src/tests/features/tensor/tensor_test.cpp @@ -7,10 +7,9 @@ #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/fef/test/ftlib.h> #include <vespa/searchlib/fef/test/indexenvironment.h> -#include <vespa/searchlib/fef/test/indexenvironmentbuilder.h> -#include <vespa/searchlib/fef/test/queryenvironment.h> #include <vespa/searchlib/tensor/tensor_attribute.h> #include <vespa/searchlib/tensor/direct_tensor_attribute.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/eval/eval/function.h> #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> diff --git a/searchlib/src/tests/features/tensor_from_labels/tensor_from_labels_test.cpp b/searchlib/src/tests/features/tensor_from_labels/tensor_from_labels_test.cpp index 8ac73253625..b2143bacfd9 100644 --- a/searchlib/src/tests/features/tensor_from_labels/tensor_from_labels_test.cpp +++ b/searchlib/src/tests/features/tensor_from_labels/tensor_from_labels_test.cpp @@ -10,8 +10,7 @@ #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/fef/test/ftlib.h> #include <vespa/searchlib/fef/test/indexenvironment.h> -#include <vespa/searchlib/fef/test/indexenvironmentbuilder.h> -#include <vespa/searchlib/fef/test/queryenvironment.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/eval/eval/function.h> #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> diff --git a/searchlib/src/tests/features/tensor_from_weighted_set/tensor_from_weighted_set_test.cpp b/searchlib/src/tests/features/tensor_from_weighted_set/tensor_from_weighted_set_test.cpp index 4d213434073..e832812a2e1 100644 --- a/searchlib/src/tests/features/tensor_from_weighted_set/tensor_from_weighted_set_test.cpp +++ b/searchlib/src/tests/features/tensor_from_weighted_set/tensor_from_weighted_set_test.cpp @@ -10,8 +10,7 @@ #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/fef/test/ftlib.h> #include <vespa/searchlib/fef/test/indexenvironment.h> -#include <vespa/searchlib/fef/test/indexenvironmentbuilder.h> -#include <vespa/searchlib/fef/test/queryenvironment.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/eval/eval/function.h> #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> diff --git a/searchlib/src/tests/fef/attributecontent/attributecontent_test.cpp b/searchlib/src/tests/fef/attributecontent/attributecontent_test.cpp index f62bfa87d45..08ae2701eb9 100644 --- a/searchlib/src/tests/fef/attributecontent/attributecontent_test.cpp +++ b/searchlib/src/tests/fef/attributecontent/attributecontent_test.cpp @@ -1,10 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("attributecontent_test"); + #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/integerbase.h> +#include <vespa/log/log.h> +LOG_SETUP("attributecontent_test"); using namespace search::attribute; diff --git a/searchlib/src/tests/predicate/simple_index_test.cpp b/searchlib/src/tests/predicate/simple_index_test.cpp index c89987eea7d..dfa8c12deec 100644 --- a/searchlib/src/tests/predicate/simple_index_test.cpp +++ b/searchlib/src/tests/predicate/simple_index_test.cpp @@ -10,6 +10,7 @@ #include <vespa/vespalib/btree/btreestore.hpp> #include <vespa/vespalib/btree/btreenodeallocator.hpp> #include <vespa/vespalib/datastore/buffer_type.hpp> +#include <vespa/vespalib/util/rcuvector.hpp> #include <map> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/queryeval/matching_elements_search/matching_elements_search_test.cpp b/searchlib/src/tests/queryeval/matching_elements_search/matching_elements_search_test.cpp index 3bb59480ed9..561691cc5c8 100644 --- a/searchlib/src/tests/queryeval/matching_elements_search/matching_elements_search_test.cpp +++ b/searchlib/src/tests/queryeval/matching_elements_search/matching_elements_search_test.cpp @@ -8,6 +8,7 @@ #include <vespa/searchlib/attribute/i_document_weight_attribute.h> #include <vespa/searchlib/common/matching_elements.h> #include <vespa/searchlib/queryeval/matching_elements_search.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/gtest/gtest.h> using search::attribute::BasicType; diff --git a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp index d34ae0dfda6..029b74ff914 100644 --- a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp +++ b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp @@ -14,6 +14,7 @@ #include <vespa/searchlib/tensor/distance_function_factory.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/searchlib/queryeval/nns_index_iterator.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/log/log.h> LOG_SETUP("nearest_neighbor_test"); diff --git a/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp b/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp index 44731bac7f6..56e745afa50 100644 --- a/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp +++ b/searchlib/src/tests/queryeval/predicate/predicate_blueprint_test.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for predicate_blueprint. -#include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/attribute/predicate_attribute.h> #include <vespa/searchlib/predicate/predicate_tree_annotator.h> #include <vespa/searchlib/predicate/predicate_index.h> @@ -33,14 +32,13 @@ namespace { struct Fixture { FieldSpecBase field; AttributeVector::SP attribute; - vespalib::GenerationHandler generation_handler; SimplePredicateQuery query; using IntervalRange = PredicateAttribute::IntervalRange; Fixture() : field(42, 0), - attribute(std::make_shared<PredicateAttribute>("f", attribute::Config(attribute::BasicType::PREDICATE))), + attribute(std::make_shared<PredicateAttribute>("f")), query(std::make_unique<PredicateQueryTerm>(),"view", 0, Weight(1)) { query.getTerm()->addFeature("key", "value"); diff --git a/searchlib/src/tests/queryeval/queryeval.cpp b/searchlib/src/tests/queryeval/queryeval.cpp index dbf8ddba52c..2f03525644a 100644 --- a/searchlib/src/tests/queryeval/queryeval.cpp +++ b/searchlib/src/tests/queryeval/queryeval.cpp @@ -18,6 +18,7 @@ #include <vespa/searchlib/queryeval/isourceselector.h> #include <vespa/searchlib/query/query_term_simple.h> #include <vespa/searchlib/attribute/singleboolattribute.h> +#include <vespa/searchcommon/common/growstrategy.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/searchlib/fef/fef.h> #include <vespa/vespalib/data/slime/slime.h> diff --git a/searchlib/src/tests/ranksetup/ranksetup_test.cpp b/searchlib/src/tests/ranksetup/ranksetup_test.cpp index ddd3f946cc5..63209446746 100644 --- a/searchlib/src/tests/ranksetup/ranksetup_test.cpp +++ b/searchlib/src/tests/ranksetup/ranksetup_test.cpp @@ -8,6 +8,7 @@ #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/attribute/integerbase.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/fef/blueprint.h> #include <vespa/searchlib/fef/blueprintfactory.h> diff --git a/searchlib/src/tests/searchcommon/attribute/config/attribute_config_test.cpp b/searchlib/src/tests/searchcommon/attribute/config/attribute_config_test.cpp index 918e14546e6..f9e77376b25 100644 --- a/searchlib/src/tests/searchcommon/attribute/config/attribute_config_test.cpp +++ b/searchlib/src/tests/searchcommon/attribute/config/attribute_config_test.cpp @@ -101,11 +101,11 @@ TEST("test operator== on attribute config for tensor type") } TEST("Test GrowStrategy consistency") { - GrowStrategy g(1024, 0.5, 17, 0.4f); - EXPECT_EQUAL(1024u, g.getDocsInitialCapacity()); - EXPECT_EQUAL(50u, g.getDocsGrowPercent()); - EXPECT_EQUAL(0.5, g.getDocsGrowFactor()); - EXPECT_EQUAL(17u, g.getDocsGrowDelta()); + GrowStrategy g(1024, 0.5, 17, 3, 0.4f); + EXPECT_EQUAL(1024u, g.getInitialCapacity()); + EXPECT_EQUAL(0.5, g.getGrowFactor()); + EXPECT_EQUAL(17u, g.getGrowDelta()); + EXPECT_EQUAL(3u, g.getMinimumCapacity()); EXPECT_EQUAL(0.4f, g.getMultiValueAllocGrowFactor()); } diff --git a/searchlib/src/tests/sortspec/multilevelsort.cpp b/searchlib/src/tests/sortspec/multilevelsort.cpp index 9c1caaff662..18c1ef9a615 100644 --- a/searchlib/src/tests/sortspec/multilevelsort.cpp +++ b/searchlib/src/tests/sortspec/multilevelsort.cpp @@ -7,6 +7,7 @@ #include <vespa/searchlib/attribute/attributecontext.h> #include <vespa/searchlib/attribute/attributemanager.h> #include <vespa/searchlib/uca/ucaconverter.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/util/testclock.h> #include <vespa/vespalib/testkit/testapp.h> #include <type_traits> diff --git a/searchlib/src/vespa/searchcommon/attribute/iattributevector.h b/searchlib/src/vespa/searchcommon/attribute/iattributevector.h index fa91f301b92..bac5181871a 100644 --- a/searchlib/src/vespa/searchcommon/attribute/iattributevector.h +++ b/searchlib/src/vespa/searchcommon/attribute/iattributevector.h @@ -363,6 +363,11 @@ public: return getBasicType() == BasicType::STRING; } + + virtual bool isPredicateType() const { return getBasicType() == BasicType::PREDICATE; } + virtual bool isTensorType() const { return getBasicType() == BasicType::TENSOR; } + virtual bool isReferenceType() const { return getBasicType() == BasicType::REFERENCE; } + /** * Returns whether this is a multi value attribute. **/ diff --git a/searchlib/src/vespa/searchcommon/common/growstrategy.cpp b/searchlib/src/vespa/searchcommon/common/growstrategy.cpp index f35cdbaa640..27bd5ea904e 100644 --- a/searchlib/src/vespa/searchcommon/common/growstrategy.cpp +++ b/searchlib/src/vespa/searchcommon/common/growstrategy.cpp @@ -5,11 +5,13 @@ namespace search { -std::ostream& operator<<(std::ostream& os, const GrowStrategy& grow_strategy) +std::ostream& +operator<<(std::ostream& os, const GrowStrategy& grow_strategy) { - os << "{docsInitialCapacity=" << grow_strategy.getDocsInitialCapacity() << - ", docsGrowFactor=" << grow_strategy.getDocsGrowFactor() << - ", docsGrowDelta=" << grow_strategy.getDocsGrowDelta() << + os << "{docsInitialCapacity=" << grow_strategy.getInitialCapacity() << + ", docsMinimumCapacity=" << grow_strategy.getMinimumCapacity() << + ", docsGrowFactor=" << grow_strategy.getGrowFactor() << + ", docsGrowDelta=" << grow_strategy.getGrowDelta() << ", multiValueAllocGrowFactor=" << grow_strategy.getMultiValueAllocGrowFactor() << "}"; return os; diff --git a/searchlib/src/vespa/searchcommon/common/growstrategy.h b/searchlib/src/vespa/searchcommon/common/growstrategy.h index b9b4a42cf72..bc04047aa3c 100644 --- a/searchlib/src/vespa/searchcommon/common/growstrategy.h +++ b/searchlib/src/vespa/searchcommon/common/growstrategy.h @@ -8,47 +8,30 @@ namespace search { -class GrowStrategy +class GrowStrategy : public vespalib::GrowStrategy { private: - uint32_t _docsInitialCapacity; - float _docsGrowFactor; - uint32_t _docsGrowDelta; float _multiValueAllocGrowFactor; public: GrowStrategy() noexcept - : GrowStrategy(1024, 0.5, 0, 0.2) + : GrowStrategy(1024, 0.5, 0, 0, 0.2) {} GrowStrategy(uint32_t docsInitialCapacity, float docsGrowFactor, - uint32_t docsGrowDelta, float multiValueAllocGrowFactor) noexcept - : _docsInitialCapacity(docsInitialCapacity), - _docsGrowFactor(docsGrowFactor), - _docsGrowDelta(docsGrowDelta), + uint32_t docsGrowDelta, uint32_t docsMinimumCapacity, float multiValueAllocGrowFactor) noexcept + : vespalib::GrowStrategy(docsInitialCapacity, docsGrowFactor, docsGrowDelta, docsMinimumCapacity), _multiValueAllocGrowFactor(multiValueAllocGrowFactor) { } static GrowStrategy make(uint32_t docsInitialCapacity, float docsGrowFactor, uint32_t docsGrowDelta) { - return GrowStrategy(docsInitialCapacity, docsGrowFactor, docsGrowDelta, 0.2); + return GrowStrategy(docsInitialCapacity, docsGrowFactor, docsGrowDelta, 0, 0.2); } - uint32_t getDocsInitialCapacity() const { return _docsInitialCapacity; } - uint32_t getDocsGrowPercent() const { return _docsGrowFactor*100; } - float getDocsGrowFactor() const { return _docsGrowFactor; } - uint32_t getDocsGrowDelta() const { return _docsGrowDelta; } float getMultiValueAllocGrowFactor() const { return _multiValueAllocGrowFactor; } - void setDocsInitialCapacity(uint32_t v) { _docsInitialCapacity = v; } - void setDocsGrowDelta(uint32_t v) { _docsGrowDelta = v; } - - vespalib::GrowStrategy to_generic_strategy() const { - return vespalib::GrowStrategy(_docsInitialCapacity, _docsGrowFactor, _docsGrowDelta); - } bool operator==(const GrowStrategy & rhs) const { - return _docsInitialCapacity == rhs._docsInitialCapacity && - _docsGrowFactor == rhs._docsGrowFactor && - _docsGrowDelta == rhs._docsGrowDelta && - _multiValueAllocGrowFactor == rhs._multiValueAllocGrowFactor; + return vespalib::GrowStrategy::operator==(rhs) && + (_multiValueAllocGrowFactor == rhs._multiValueAllocGrowFactor); } bool operator!=(const GrowStrategy & rhs) const { return !(operator==(rhs)); diff --git a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt index 5fa691394fb..5e86c350b55 100644 --- a/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/attribute/CMakeLists.txt @@ -23,6 +23,7 @@ vespa_add_library(searchlib_attribute OBJECT attributesaver.cpp attributevector.cpp attrvector.cpp + basename.cpp bitvector_search_cache.cpp changevector.cpp configconverter.cpp @@ -42,6 +43,7 @@ vespa_add_library(searchlib_attribute OBJECT enumattributesaver.cpp enumcomparator.cpp enumhintsearchcontext.cpp + enummodifier.cpp enum_store_compaction_spec.cpp enum_store_dictionary.cpp enum_store_loaders.cpp @@ -128,5 +130,6 @@ vespa_add_library(searchlib_attribute OBJECT string_matcher.cpp string_search_context.cpp string_search_helper.cpp + valuemodifier.cpp DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_operation.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_operation.cpp index 347793be4c6..57980237f21 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_operation.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_operation.cpp @@ -1,10 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attribute_operation.h" -#include <vespa/searchlib/attribute/singlenumericattribute.h> +#include "singlenumericattribute.h" #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchcommon/attribute/basictype.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/array.hpp> #include <vespa/log/log.h> @@ -96,7 +97,7 @@ struct UpdateFast { void operator()(uint32_t docid) { attr->set(docid, op(attr->getFast(docid))); } bool valid() const { return (attr != nullptr) && - (attr->getConfig().isMutable()); } + (attr->isMutable()); } }; template <typename OP> diff --git a/searchlib/src/vespa/searchlib/attribute/attributefactory.cpp b/searchlib/src/vespa/searchlib/attribute/attributefactory.cpp index 4f846e8d68a..0f9c01b357f 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefactory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributefactory.cpp @@ -2,6 +2,7 @@ #include "attributefactory.h" #include "attributevector.h" +#include <vespa/searchcommon/attribute/config.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.attributefactory"); diff --git a/searchlib/src/vespa/searchlib/attribute/attributefactory.h b/searchlib/src/vespa/searchlib/attribute/attributefactory.h index fcdef25a963..73b8e2b0eb6 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefactory.h +++ b/searchlib/src/vespa/searchlib/attribute/attributefactory.h @@ -2,9 +2,10 @@ #pragma once -#include <vespa/searchcommon/attribute/config.h> +#include <vespa/vespalib/stllike/string.h> #include <memory> +namespace search::attribute { class Config; } namespace search { class AttributeVector; diff --git a/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp b/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp index 4b87435dd2e..38416b009eb 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp @@ -6,6 +6,7 @@ #include "attributemanager.h" #include "attrvector.h" #include "interlock.h" +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/exceptions.h> #include <condition_variable> @@ -205,7 +206,7 @@ AttributeManager::createBaseFileName(const string & name) const dir += "/"; dir += getSnapshot().dirName; } - return AttributeVector::BaseName(dir, name); + return attribute::BaseName(dir, name); } bool diff --git a/searchlib/src/vespa/searchlib/attribute/attributemanager.h b/searchlib/src/vespa/searchlib/attribute/attributemanager.h index daa6c725908..28e50490a17 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributemanager.h +++ b/searchlib/src/vespa/searchlib/attribute/attributemanager.h @@ -5,10 +5,11 @@ #include "iattributemanager.h" #include "interlock.h" #include <vespa/searchlib/common/indexmetainfo.h> -#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/stllike/hash_map.h> #include <mutex> +namespace search::attribute { class Config; } + namespace search { /** @@ -18,7 +19,7 @@ namespace search { class AttributeManager : public IAttributeManager { private: - typedef attribute::Config Config; + using Config = attribute::Config; public: typedef std::vector<string> StringVector; typedef search::IndexMetaInfo::Snapshot Snapshot; diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index d7c9bb8d224..c0df44d26e9 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -1,24 +1,27 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "attributevector.h" #include "address_space_components.h" #include "attribute_read_guard.h" #include "attributefilesavetarget.h" #include "attributesaver.h" -#include "attributevector.h" #include "attributevector.hpp" #include "floatbase.h" #include "interlock.h" #include "ipostinglistattributebase.h" #include "stringbase.h" +#include "enummodifier.h" +#include "valuemodifier.h" #include <vespa/document/update/assignvalueupdate.h> #include <vespa/document/update/mapvalueupdate.h> #include <vespa/fastlib/io/bufferedfile.h> #include <vespa/searchcommon/attribute/attribute_utils.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/query/query_term_decoder.h> #include <vespa/searchlib/util/file_settings.h> -#include <vespa/searchlib/util/logutil.h> +#include <vespa/vespalib/util/jsonwriter.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/mmap_file_allocator_factory.h> #include <vespa/vespalib/util/size_literals.h> @@ -31,13 +34,12 @@ using vespalib::getLastErrorString; using document::ValueUpdate; using document::AssignValueUpdate; -using vespalib::make_string; -using vespalib::Array; using vespalib::IllegalStateException; using search::attribute::SearchContextParams; using search::common::FileHeaderContext; using search::index::DummyFileHeaderContext; using search::queryeval::SearchIterator; +using namespace vespalib::make_string_short; namespace { @@ -50,61 +52,6 @@ const vespalib::string docIdLimitTag = "docIdLimit"; namespace search { -IMPLEMENT_IDENTIFIABLE_ABSTRACT(AttributeVector, vespalib::Identifiable); - -AttributeVector::BaseName::BaseName(vespalib::stringref base, vespalib::stringref name) - : string(base), - _name(name) -{ - if (!empty()) { - push_back('/'); - } - append(name); -} - -AttributeVector::BaseName::~BaseName() = default; - - -AttributeVector::BaseName::string -AttributeVector::BaseName::createAttributeName(vespalib::stringref s) -{ - size_t p(s.rfind('/')); - if (p == string::npos) { - return s; - } else { - return s.substr(p+1); - } -} - - -AttributeVector::BaseName::string -AttributeVector::BaseName::getDirName() const -{ - size_t p = rfind('/'); - if (p == string::npos) { - return ""; - } else { - return substr(0, p); - } -} - - -AttributeVector::ValueModifier::ValueModifier(AttributeVector &attr) - : _attr(&attr) -{ } - - -AttributeVector::ValueModifier::ValueModifier(const ValueModifier &rhs) - : _attr(rhs.stealAttr()) -{ } - - -AttributeVector::ValueModifier::~ValueModifier() { - if (_attr) { - _attr->incGeneration(); - } -} - namespace { bool @@ -136,7 +83,7 @@ make_memory_allocator(const vespalib::string& name, const search::attribute::Con AttributeVector::AttributeVector(vespalib::stringref baseFileName, const Config &c) : _baseFileName(baseFileName), - _config(c), + _config(std::make_unique<Config>(c)), _interlock(std::make_shared<attribute::Interlock>()), _enumLock(), _genHandler(), @@ -170,6 +117,16 @@ AttributeVector::updateStat(bool force) { bool AttributeVector::hasEnum() const { return _hasEnum; } uint32_t AttributeVector::getMaxValueCount() const { return _highestValueCount.load(std::memory_order_relaxed); } +bool AttributeVector::hasMultiValue() const { return _config->collectionType().isMultiValue(); } +bool AttributeVector::hasWeightedSetType() const { return _config->collectionType().isWeightedSet(); } +size_t AttributeVector::getFixedWidth() const { return _config->basicType().fixedSize(); } +attribute::BasicType AttributeVector::getInternalBasicType() const { return _config->basicType(); } +attribute::CollectionType AttributeVector::getInternalCollectionType() const { return _config->collectionType(); } +bool AttributeVector::hasArrayType() const { return _config->collectionType().isArray(); } +bool AttributeVector::getIsFilter() const { return _config->getIsFilter(); } +bool AttributeVector::getIsFastSearch() const { return _config->fastSearch(); } +bool AttributeVector::isMutable() const { return _config->isMutable(); } +bool AttributeVector::getEnableOnlyBitVector() const { return _config->getEnableOnlyBitVector(); } bool AttributeVector::isEnumerated(const vespalib::GenericHeader &header) @@ -217,7 +174,6 @@ AttributeVector::addDocs(DocId &startDoc, DocId &lastDoc, uint32_t numDocs) return true; } - bool AttributeVector::addDocs(uint32_t numDocs) { @@ -225,7 +181,6 @@ AttributeVector::addDocs(uint32_t numDocs) return addDocs(doc, doc, numDocs); } - void AttributeVector::incGeneration() { @@ -236,7 +191,6 @@ AttributeVector::incGeneration() removeAllOldGenerations(); } - void AttributeVector::updateStatistics(uint64_t numValues, uint64_t numUniqueValue, uint64_t allocated, uint64_t used, uint64_t dead, uint64_t onHold) @@ -351,21 +305,21 @@ void AttributeVector::onSave(IAttributeSaveTarget &) bool AttributeVector::hasLoadData() const { FastOS_StatInfo statInfo; - if (!FastOS_File::Stat(make_string("%s.dat", getBaseFileName().c_str()).c_str(), &statInfo)) { + if (!FastOS_File::Stat(fmt("%s.dat", getBaseFileName().c_str()).c_str(), &statInfo)) { return false; } if (hasMultiValue() && - !FastOS_File::Stat(make_string("%s.idx", getBaseFileName().c_str()).c_str(), &statInfo)) + !FastOS_File::Stat(fmt("%s.idx", getBaseFileName().c_str()).c_str(), &statInfo)) { return false; } if (hasWeightedSetType() && - !FastOS_File::Stat(make_string("%s.weight", getBaseFileName().c_str()).c_str(), &statInfo)) + !FastOS_File::Stat(fmt("%s.weight", getBaseFileName().c_str()).c_str(), &statInfo)) { return false; } if (isEnumeratedSaveFormat() && - !FastOS_File::Stat(make_string("%s.udat", getBaseFileName().c_str()).c_str(), &statInfo)) + !FastOS_File::Stat(fmt("%s.udat", getBaseFileName().c_str()).c_str(), &statInfo)) { return false; } @@ -376,12 +330,12 @@ AttributeVector::hasLoadData() const { bool AttributeVector::isEnumeratedSaveFormat() const { - vespalib::string datName(vespalib::make_string("%s.dat", getBaseFileName().c_str())); + vespalib::string datName(fmt("%s.dat", getBaseFileName().c_str())); Fast_BufferedFile datFile; vespalib::FileHeader datHeader(FileSettings::DIRECTIO_ALIGNMENT); if ( ! datFile.OpenReadOnly(datName.c_str()) ) { LOG(error, "could not open %s: %s", datFile.GetFileName(), getLastErrorString().c_str()); - throw IllegalStateException(make_string("Failed opening attribute data file '%s' for reading", + throw IllegalStateException(fmt("Failed opening attribute data file '%s' for reading", datFile.GetFileName())); } datHeader.readFile(datFile); @@ -489,16 +443,14 @@ AttributeVector::addReservedDoc() assert(docId < getNumDocs()); clearDoc(docId); commit(); - const vespalib::Identifiable::RuntimeClass &info = getClass(); - if (info.inherits(search::FloatingPointAttribute::classId)) { - FloatingPointAttribute &vec = - static_cast<FloatingPointAttribute &>(*this); + FloatingPointAttribute * vec = dynamic_cast<FloatingPointAttribute *>(this); + if (vec) { if (hasMultiValue()) { - bool appendedUndefined = vec.append(0, attribute::getUndefined<double>(), 1); + bool appendedUndefined = vec->append(0, attribute::getUndefined<double>(), 1); assert(appendedUndefined); (void) appendedUndefined; } else { - bool updatedUndefined = vec.update(0, attribute::getUndefined<double>()); + bool updatedUndefined = vec->update(0, attribute::getUndefined<double>()); assert(updatedUndefined); (void) updatedUndefined; } @@ -582,11 +534,16 @@ AttributeVector::clearDocs(DocId lidLow, DocId lidLimit, bool in_shrink_lid_spac } } -AttributeVector::EnumModifier +attribute::EnumModifier AttributeVector::getEnumModifier() { attribute::InterlockGuard interlockGuard(*_interlock); - return EnumModifier(_enumLock, interlockGuard); + return attribute::EnumModifier(_enumLock, interlockGuard); +} + +attribute::ValueModifier +AttributeVector::getValueModifier() { + return ValueModifier(*this); } @@ -752,7 +709,7 @@ AttributeVector::logEnumStoreEvent(const char *reason, const char *stage) jstr.beginObject(); jstr.appendKey("path").appendString(getBaseFileName()); jstr.endObject(); - vespalib::string eventName(make_string("%s.attribute.enumstore.%s", reason, stage)); + vespalib::string eventName(fmt("%s.attribute.enumstore.%s", reason, stage)); EV_STATE(eventName.c_str(), jstr.toString().data()); } @@ -774,12 +731,12 @@ void AttributeVector::update_config(const Config& cfg) { commit(true); - _config.setGrowStrategy(cfg.getGrowStrategy()); - if (cfg.getCompactionStrategy() == _config.getCompactionStrategy()) { + _config->setGrowStrategy(cfg.getGrowStrategy()); + if (cfg.getCompactionStrategy() == _config->getCompactionStrategy()) { return; } drain_hold(1_Mi); // Wait until 1MiB or less on hold - _config.setCompactionStrategy(cfg.getCompactionStrategy()); + _config->setCompactionStrategy(cfg.getCompactionStrategy()); updateStat(true); commit(); // might trigger compaction drain_hold(1_Mi); // Wait until 1MiB or less on hold diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index 57c73067671..f245a216aeb 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -5,8 +5,7 @@ #include "address_space_usage.h" #include "changevector.h" #include "readable_attribute_vector.h" -#include <vespa/fastlib/text/normwordfolder.h> -#include <vespa/searchcommon/attribute/config.h> +#include "basename.h" #include <vespa/searchcommon/attribute/i_search_context.h> #include <vespa/searchcommon/attribute/iattributevector.h> #include <vespa/searchcommon/attribute/search_context_params.h> @@ -14,13 +13,9 @@ #include <vespa/searchcommon/common/range.h> #include <vespa/searchcommon/common/undefinedvalues.h> #include <vespa/searchlib/common/i_compactable_lid_space.h> -#include <vespa/searchlib/common/identifiable.h> #include <vespa/searchlib/common/commit_param.h> #include <vespa/searchlib/queryeval/searchiterator.h> -#include <vespa/vespalib/objects/identifiable.h> -#include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/util/address_space.h> -#include <vespa/vespalib/util/rcuvector.h> +#include <vespa/vespalib/util/generationholder.h> #include <vespa/vespalib/util/time.h> #include <cmath> #include <mutex> @@ -38,6 +33,11 @@ namespace vespalib { class Executor; } +namespace vespalib::alloc { + class MemoryAllocator; + class Alloc; +} + namespace search { template <typename T> class ComponentGuard; @@ -61,6 +61,9 @@ namespace search { class InterlockGuard; class SearchContext; class MultiValueMappingBase; + class Config; + class ValueModifier; + class EnumModifier; } namespace fileutil { @@ -76,28 +79,6 @@ using document::ArithmeticValueUpdate; using document::MapValueUpdate; using document::FieldValue; -template <typename T> -class UnWeightedType -{ -public: - UnWeightedType() : _value(T()) { } - UnWeightedType(T v) : _value(v) { } - const T & getValue() const { return _value; } - void setValue(const T & v) { _value = v; } - int32_t getWeight() const { return 1; } - void setWeight(int32_t w) { (void) w; } - - bool operator==(const UnWeightedType<T> & rhs) const { - return _value == rhs._value; - } - -private: - T _value; -}; - -template <typename T> -vespalib::asciistream & operator << (vespalib::asciistream & os, const UnWeightedType<T> & v); - class IExtendAttribute { public: @@ -105,11 +86,10 @@ public: virtual bool add(double, int32_t = 1) { return false; } virtual bool add(const char *, int32_t = 1) { return false; } - virtual ~IExtendAttribute() {} + virtual ~IExtendAttribute() = default; }; -class AttributeVector : public vespalib::Identifiable, - public attribute::IAttributeVector, +class AttributeVector : public attribute::IAttributeVector, public common::ICompactableLidSpace, public attribute::ReadableAttributeVector { @@ -120,31 +100,10 @@ protected: using QueryTermSimpleUP = std::unique_ptr<QueryTermSimple>; using QueryPacketT = vespalib::stringref; using stringref = vespalib::stringref; + using ValueModifier = attribute::ValueModifier; + using EnumModifier = attribute::EnumModifier; public: - typedef std::shared_ptr<AttributeVector> SP; - class BaseName : public vespalib::string - { - public: - typedef vespalib::string string; - BaseName(vespalib::stringref s) - : string(s), - _name(createAttributeName(s)) - { } - BaseName & operator = (vespalib::stringref s) { - BaseName n(s); - std::swap(*this, n); - return *this; - } - - BaseName(vespalib::stringref base, vespalib::stringref name); - ~BaseName(); - - const string & getAttributeName() const { return _name; } - string getDirName() const; - private: - static string createAttributeName(vespalib::stringref s); - string _name; - }; + using SP = std::shared_ptr<AttributeVector>; using GenerationHandler = vespalib::GenerationHandler; using GenerationHolder = vespalib::GenerationHolder; @@ -180,46 +139,11 @@ protected: void setNumDocs(uint32_t n) { _status.setNumDocs(n); } void incNumDocs() { _status.incNumDocs(); } - class ValueModifier - { - public: - ValueModifier(AttributeVector &attr); - ValueModifier(const ValueModifier &rhs); - ~ValueModifier(); - private: - AttributeVector * stealAttr() const { - AttributeVector * ret(_attr); - _attr = nullptr; - return ret; - } - - mutable AttributeVector * _attr; - }; - public: - class EnumModifier - { - std::unique_lock<std::shared_mutex> _enumLock; - public: - EnumModifier(std::shared_mutex &lock, attribute::InterlockGuard &interlockGuard) - : _enumLock(lock) - { - (void) interlockGuard; - } - EnumModifier(EnumModifier &&rhs) - : _enumLock(std::move(rhs._enumLock)) - { } - EnumModifier &operator=(EnumModifier &&rhs) - { - _enumLock = std::move(rhs._enumLock); - return *this; - } - virtual ~EnumModifier() { } - }; EnumModifier getEnumModifier(); protected: - ValueModifier getValueModifier() { return ValueModifier(*this); } + ValueModifier getValueModifier(); void updateCommittedDocIdLimit() { if (_uncommittedDocIdLimit != 0) { @@ -349,23 +273,27 @@ protected: const std::shared_ptr<vespalib::alloc::MemoryAllocator>& get_memory_allocator() const noexcept { return _memory_allocator; } vespalib::alloc::Alloc get_initial_alloc(); public: - DECLARE_IDENTIFIABLE_ABSTRACT(AttributeVector); bool isLoaded() const { return _loaded; } void logEnumStoreEvent(const char *reason, const char *stage); /** Return the fixed length of the attribute. If 0 then you must inquire each document. */ - size_t getFixedWidth() const override { return _config.basicType().fixedSize(); } - const Config &getConfig() const noexcept { return _config; } + size_t getFixedWidth() const override; + BasicType getInternalBasicType() const; + CollectionType getInternalCollectionType() const; + bool hasArrayType() const; + bool getIsFilter() const override final; + bool getIsFastSearch() const override final; + bool isMutable() const; + bool getEnableOnlyBitVector() const; + + const Config &getConfig() const noexcept { return *_config; } void update_config(const Config& cfg); - BasicType getInternalBasicType() const { return _config.basicType(); } - CollectionType getInternalCollectionType() const { return _config.collectionType(); } - const BaseName & getBaseFileName() const { return _baseFileName; } + const attribute::BaseName & getBaseFileName() const { return _baseFileName; } void setBaseFileName(vespalib::stringref name) { _baseFileName = name; } bool isUpdateableInMemoryOnly() const { return _isUpdateableInMemoryOnly; } const vespalib::string & getName() const override final { return _baseFileName.getAttributeName(); } - bool hasArrayType() const { return _config.collectionType().isArray(); } bool hasEnum() const override final; uint32_t getMaxValueCount() const override; uint32_t getEnumMax() const { return _enumMax; } @@ -390,8 +318,6 @@ public: BasicType::Type getBasicType() const override final { return getInternalBasicType().type(); } CollectionType::Type getCollectionType() const override final { return getInternalCollectionType().type(); } - bool getIsFilter() const override final { return _config.getIsFilter(); } - bool getIsFastSearch() const override final { return _config.fastSearch(); } uint32_t getCommittedDocIdLimit() const override final { return _committedDocIdLimit.load(std::memory_order_acquire); } bool isImported() const override; @@ -491,8 +417,8 @@ private: virtual bool onLoad(vespalib::Executor * executor); - BaseName _baseFileName; - Config _config; + attribute::BaseName _baseFileName; + std::unique_ptr<Config> _config; std::shared_ptr<attribute::Interlock> _interlock; mutable std::shared_mutex _enumLock; GenerationHandler _genHandler; @@ -535,8 +461,8 @@ private: friend class AttributeManagerTest; public: bool headerTypeOK(const vespalib::GenericHeader &header) const; - bool hasMultiValue() const override final { return _config.collectionType().isMultiValue(); } - bool hasWeightedSetType() const override final { return _config.collectionType().isWeightedSet(); } + bool hasMultiValue() const override final; + bool hasWeightedSetType() const override final; /** * Should be called by the writer thread. */ diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.hpp b/searchlib/src/vespa/searchlib/attribute/attributevector.hpp index c75f7788467..959d72bd5ec 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.hpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.hpp @@ -87,7 +87,7 @@ AttributeVector::applyArithmetic(ChangeVectorT< ChangeTemplate<T> > & changes, D } else if (op == ArithmeticValueUpdate::Mul) { changes.push_back(ChangeTemplate<T>(ChangeBase::MUL, doc, 0, 0)); } else if (op == ArithmeticValueUpdate::Div) { - if (this->getClass().inherits(IntegerAttribute::classId) && aop == 0) { + if ((aop == 0) && isIntegerType()) { divideByZeroWarning(); } else { changes.push_back(ChangeTemplate<T>(ChangeBase::DIV, doc, 0, 0)); @@ -173,10 +173,5 @@ bool AttributeVector::remove(ChangeVectorT< ChangeTemplate<T> > & changes, DocId return retval; } -template<typename T> -vespalib::asciistream & operator << (vespalib::asciistream & os, const UnWeightedType<T> & v) { - return os << "(" << v._value << ", 1)"; -} - } diff --git a/searchlib/src/vespa/searchlib/attribute/basename.cpp b/searchlib/src/vespa/searchlib/attribute/basename.cpp new file mode 100644 index 00000000000..34a98c19630 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/basename.cpp @@ -0,0 +1,52 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "basename.h" + +namespace search::attribute { + +BaseName::BaseName(vespalib::stringref base, vespalib::stringref name) + : string(base), + _name(name) +{ + if (!empty()) { + push_back('/'); + } + append(name); +} + +BaseName::BaseName(vespalib::stringref s) + : string(s), + _name(createAttributeName(s)) +{ } +BaseName & +BaseName::operator = (vespalib::stringref s) { + BaseName n(s); + std::swap(*this, n); + return *this; + } + +BaseName::~BaseName() = default; + +vespalib::string +BaseName::createAttributeName(vespalib::stringref s) +{ + size_t p(s.rfind('/')); + if (p == string::npos) { + return s; + } else { + return s.substr(p+1); + } +} + +vespalib::string +BaseName::getDirName() const +{ + size_t p = rfind('/'); + if (p == string::npos) { + return ""; + } else { + return substr(0, p); + } +} + +} diff --git a/searchlib/src/vespa/searchlib/attribute/basename.h b/searchlib/src/vespa/searchlib/attribute/basename.h new file mode 100644 index 00000000000..23c8d28b2d8 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/basename.h @@ -0,0 +1,25 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace search::attribute { + +class BaseName : public vespalib::string +{ +public: + using string = vespalib::string; + BaseName(vespalib::stringref s); + BaseName(vespalib::stringref base, vespalib::stringref name); + BaseName & operator = (vespalib::stringref s); + ~BaseName(); + + const string & getAttributeName() const { return _name; } + string getDirName() const; +private: + static string createAttributeName(vespalib::stringref s); + string _name; +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/changevector.h b/searchlib/src/vespa/searchlib/attribute/changevector.h index d929e8615a9..5f858a3e012 100644 --- a/searchlib/src/vespa/searchlib/attribute/changevector.h +++ b/searchlib/src/vespa/searchlib/attribute/changevector.h @@ -3,6 +3,7 @@ #pragma once #include <vespa/searchcommon/common/undefinedvalues.h> +#include <vespa/vespalib/util/memoryusage.h> #include <vector> namespace vespalib { class MemoryUsage; } diff --git a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp index f6e8266f858..9aac4f301d5 100644 --- a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "configconverter.h" +#include <vespa/searchcommon/attribute/config.h> using namespace vespa::config::search; diff --git a/searchlib/src/vespa/searchlib/attribute/createarrayfastsearch.cpp b/searchlib/src/vespa/searchlib/attribute/createarrayfastsearch.cpp index 9de2a1750e1..5e22352c1d4 100644 --- a/searchlib/src/vespa/searchlib/attribute/createarrayfastsearch.cpp +++ b/searchlib/src/vespa/searchlib/attribute/createarrayfastsearch.cpp @@ -5,6 +5,7 @@ #include "floatbase.h" #include "flagattribute.h" #include "defines.h" +#include <vespa/searchcommon/attribute/config.h> #include <vespa/log/log.h> LOG_SETUP(".createarrayfastsearch"); @@ -39,7 +40,7 @@ AttributeFactory::createArrayFastSearch(stringref name, const Config & info) case BasicType::UINT4: break; case BasicType::INT8: - ret.reset(static_cast<AttributeVector *>(new FlagAttribute(name, info))); + ret = std::make_shared<FlagAttribute>(name, info); break; case BasicType::INT16: ret.reset(CREATEINTARRAY(int16_t, name, info)); @@ -57,7 +58,7 @@ AttributeFactory::createArrayFastSearch(stringref name, const Config & info) ret.reset(CREATEFLOATARRAY(double, name, info)); break; case BasicType::STRING: - ret.reset(static_cast<AttributeVector *>(new ArrayStringPostingAttribute(name, info))); + ret = std::make_shared<ArrayStringPostingAttribute>(name, info); break; default: break; diff --git a/searchlib/src/vespa/searchlib/attribute/createsinglefastsearch.cpp b/searchlib/src/vespa/searchlib/attribute/createsinglefastsearch.cpp index 66e6e7ccd4d..05b6bb07a19 100644 --- a/searchlib/src/vespa/searchlib/attribute/createsinglefastsearch.cpp +++ b/searchlib/src/vespa/searchlib/attribute/createsinglefastsearch.cpp @@ -6,6 +6,7 @@ #include "defines.h" #include "singlestringattribute.h" #include "singleboolattribute.h" +#include <vespa/searchcommon/attribute/config.h> #include "singlestringpostattribute.hpp" #include "singlenumericpostattribute.hpp" #include <vespa/searchlib/tensor/direct_tensor_attribute.h> diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp index 24dfe742120..86f0803aa3e 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/util/hdr_abort.h> #include <vespa/searchlib/attribute/enumattribute.h> #include <vespa/searchlib/attribute/enumstore.hpp> +#include <vespa/searchcommon/attribute/config.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/enummodifier.cpp b/searchlib/src/vespa/searchlib/attribute/enummodifier.cpp new file mode 100644 index 00000000000..72fd624d18a --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/enummodifier.cpp @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "enummodifier.h" + +namespace search::attribute { + +EnumModifier::EnumModifier(std::shared_mutex &lock, attribute::InterlockGuard &interlockGuard) + : _enumLock(lock) +{ + (void) interlockGuard; +} + +EnumModifier::~EnumModifier() = default; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/enummodifier.h b/searchlib/src/vespa/searchlib/attribute/enummodifier.h new file mode 100644 index 00000000000..c147ac5cd75 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/enummodifier.h @@ -0,0 +1,20 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <shared_mutex> +#include <mutex> + +namespace search::attribute { + +class InterlockGuard; + +class EnumModifier +{ + std::unique_lock<std::shared_mutex> _enumLock; +public: + EnumModifier(std::shared_mutex &lock, InterlockGuard &interlockGuard); + ~EnumModifier(); +}; + +} diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.cpp b/searchlib/src/vespa/searchlib/attribute/enumstore.cpp index 31523012c1a..d7d627ff2bc 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.cpp @@ -2,7 +2,6 @@ #include "enumstore.hpp" #include <vespa/vespalib/datastore/sharded_hash_map.h> -#include <vespa/vespalib/util/rcuvector.hpp> #include <iomanip> #include <vespa/log/log.h> @@ -89,7 +88,3 @@ template class EnumStoreT<float>; template class EnumStoreT<double>; } // namespace search - -namespace vespalib { - template class RcuVectorBase<search::IEnumStore::Index>; -} diff --git a/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.cpp b/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.cpp index fef05a8ce08..ad3b9c403a0 100644 --- a/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.cpp +++ b/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.cpp @@ -5,7 +5,7 @@ namespace search::attribute { template <class MultiValueType> -ExtendableStringArrayMultiValueReadView<MultiValueType>::ExtendableStringArrayMultiValueReadView(const std::vector<char>& buffer, const vespalib::Array<uint32_t>& offsets, const std::vector<uint32_t>& idx) +ExtendableStringArrayMultiValueReadView<MultiValueType>::ExtendableStringArrayMultiValueReadView(const std::vector<char>& buffer, const Offsets & offsets, const std::vector<uint32_t>& idx) : attribute::IMultiValueReadView<MultiValueType>(), _buffer(buffer), _offsets(offsets), diff --git a/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.h b/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.h index 1104d749886..d83398b5568 100644 --- a/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.h +++ b/searchlib/src/vespa/searchlib/attribute/extendable_string_array_multi_value_read_view.h @@ -3,7 +3,7 @@ #pragma once #include <vespa/searchcommon/attribute/i_multi_value_read_view.h> -#include <vespa/vespalib/util/array.h> +#include <vespa/vespalib/stllike/allocator.h> namespace search::attribute { @@ -16,12 +16,13 @@ namespace search::attribute { template <typename MultiValueType> class ExtendableStringArrayMultiValueReadView : public attribute::IMultiValueReadView<MultiValueType> { + using Offsets = std::vector<uint32_t, vespalib::allocator_large<uint32_t>>; const std::vector<char>& _buffer; - const vespalib::Array<uint32_t>& _offsets; + const Offsets & _offsets; const std::vector<uint32_t>& _idx; mutable std::vector<MultiValueType> _copy; public: - ExtendableStringArrayMultiValueReadView(const std::vector<char>& buffer, const vespalib::Array<uint32_t>& offsets, const std::vector<uint32_t>& idx); + ExtendableStringArrayMultiValueReadView(const std::vector<char>& buffer, const Offsets & offsets, const std::vector<uint32_t>& idx); ~ExtendableStringArrayMultiValueReadView() override; vespalib::ConstArrayRef<MultiValueType> get_values(uint32_t doc_id) const override; }; diff --git a/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.cpp b/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.cpp index eda50dd7172..f69938f061d 100644 --- a/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.cpp +++ b/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.cpp @@ -5,7 +5,7 @@ namespace search::attribute { template <class MultiValueType> -ExtendableStringWeightedSetMultiValueReadView<MultiValueType>::ExtendableStringWeightedSetMultiValueReadView(const std::vector<char>& buffer, const vespalib::Array<uint32_t>& offsets, const std::vector<uint32_t>& idx, const std::vector<int32_t>& weights) +ExtendableStringWeightedSetMultiValueReadView<MultiValueType>::ExtendableStringWeightedSetMultiValueReadView(const std::vector<char>& buffer, const Offsets & offsets, const std::vector<uint32_t>& idx, const std::vector<int32_t>& weights) : attribute::IMultiValueReadView<MultiValueType>(), _buffer(buffer), _offsets(offsets), diff --git a/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.h b/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.h index 2c55452faaf..011669ffd47 100644 --- a/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.h +++ b/searchlib/src/vespa/searchlib/attribute/extendable_string_weighted_set_multi_value_read_view.h @@ -3,7 +3,7 @@ #pragma once #include <vespa/searchcommon/attribute/i_multi_value_read_view.h> -#include <vespa/vespalib/util/array.h> +#include <vespa/vespalib/stllike/allocator.h> namespace search::attribute { @@ -16,13 +16,14 @@ namespace search::attribute { template <typename MultiValueType> class ExtendableStringWeightedSetMultiValueReadView : public attribute::IMultiValueReadView<MultiValueType> { + using Offsets = std::vector<uint32_t, vespalib::allocator_large<uint32_t>>; const std::vector<char>& _buffer; - const vespalib::Array<uint32_t>& _offsets; + const Offsets & _offsets; const std::vector<uint32_t>& _idx; const std::vector<int32_t>& _weights; mutable std::vector<MultiValueType> _copy; public: - ExtendableStringWeightedSetMultiValueReadView(const std::vector<char>& buffer, const vespalib::Array<uint32_t>& offsets, const std::vector<uint32_t>& idx, const std::vector<int32_t>& weights); + ExtendableStringWeightedSetMultiValueReadView(const std::vector<char>& buffer, const Offsets & offsets, const std::vector<uint32_t>& idx, const std::vector<int32_t>& weights); ~ExtendableStringWeightedSetMultiValueReadView() override; vespalib::ConstArrayRef<MultiValueType> get_values(uint32_t doc_id) const override; }; diff --git a/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp b/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp index 941450fccb5..30b509562e2 100644 --- a/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp +++ b/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "extendableattributes.h" -#include "attrvector.hpp" +#include "extendableattributes.hpp" #include "extendable_numeric_array_multi_value_read_view.h" #include "extendable_numeric_weighted_set_multi_value_read_view.h" #include "extendable_string_array_multi_value_read_view.h" #include "extendable_string_weighted_set_multi_value_read_view.h" #include <vespa/vespalib/util/stash.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.extendable_attributes"); @@ -129,7 +129,15 @@ WeightedSetIntegerExtAttribute::WeightedSetIntegerExtAttribute(const vespalib::s { } -WeightedSetIntegerExtAttribute::~WeightedSetIntegerExtAttribute() {} +WeightedSetIntegerExtAttribute::~WeightedSetIntegerExtAttribute() = default; + +std::unique_ptr<attribute::SearchContext> +WeightedSetIntegerExtAttribute::getSearch(QueryTermSimpleUP term, const attribute::SearchContextParams & params) const +{ + (void) term; + (void) params; + return {}; +} bool WeightedSetIntegerExtAttribute::add(int64_t v, int32_t w) @@ -161,7 +169,15 @@ WeightedSetFloatExtAttribute::WeightedSetFloatExtAttribute(const vespalib::strin { } -WeightedSetFloatExtAttribute::~WeightedSetFloatExtAttribute() {} +WeightedSetFloatExtAttribute::~WeightedSetFloatExtAttribute() = default; + +std::unique_ptr<attribute::SearchContext> +WeightedSetFloatExtAttribute::getSearch(QueryTermSimpleUP term, const attribute::SearchContextParams & params) const +{ + (void) term; + (void) params; + return {}; +} bool WeightedSetFloatExtAttribute::add(double v, int32_t w) @@ -228,4 +244,10 @@ template class MultiExtAttribute<int32_t>; template class MultiExtAttribute<int64_t>; template class MultiExtAttribute<double>; +template class SingleExtAttribute<int8_t>; +template class SingleExtAttribute<int16_t>; +template class SingleExtAttribute<int32_t>; +template class SingleExtAttribute<int64_t>; +template class SingleExtAttribute<double>; + } diff --git a/searchlib/src/vespa/searchlib/attribute/extendableattributes.h b/searchlib/src/vespa/searchlib/attribute/extendableattributes.h index 86e6be8eff7..a3a0de5bfc5 100644 --- a/searchlib/src/vespa/searchlib/attribute/extendableattributes.h +++ b/searchlib/src/vespa/searchlib/attribute/extendableattributes.h @@ -7,7 +7,6 @@ #pragma once #include "attrvector.h" -#include "attrvector.hpp" #include <vespa/searchcommon/attribute/i_multi_value_attribute.h> namespace search { @@ -41,44 +40,25 @@ class SingleExtAttribute using QueryTermSimpleUP = typename Super::QueryTermSimpleUP; std::unique_ptr<attribute::SearchContext> - getSearch(QueryTermSimpleUP term, const attribute::SearchContextParams & params) const override - { - (void) term; - (void) params; - return {}; - } + getSearch(QueryTermSimpleUP term, const attribute::SearchContextParams & params) const override; IExtendAttribute * getExtendInterface() override { return this; } public: - SingleExtAttribute(const vespalib::string &name) - : Super(name, Config(BasicType::fromType(T()), - attribute::CollectionType::SINGLE)) {} + SingleExtAttribute(const vespalib::string &name); - bool addDoc(typename Super::DocId &docId) override { - docId = this->_data.size(); - this->_data.push_back(attribute::getUndefined<T>()); - this->incNumDocs(); - this->setCommittedDocIdLimit(this->getNumDocs()); - return true; - } - bool add(typename AddValueType<T>::Type v, int32_t = 1) override { - this->_data.back() = v; - return true; - } + bool addDoc(typename Super::DocId &docId) override; + bool add(typename AddValueType<T>::Type v, int32_t = 1) override; bool onLoad(vespalib::Executor *) override { return false; // Emulate that this attribute is never loaded } - void onAddDocs(typename Super::DocId lidLimit) override { - this->_data.reserve(lidLimit); - } + void onAddDocs(typename Super::DocId lidLimit) override; }; -typedef SingleExtAttribute<int8_t> SingleInt8ExtAttribute; -typedef SingleExtAttribute<int16_t> SingleInt16ExtAttribute; -typedef SingleExtAttribute<int32_t> SingleInt32ExtAttribute; -typedef SingleExtAttribute<int64_t> SingleInt64ExtAttribute; -typedef SingleExtAttribute<double> SingleFloatExtAttribute; - -typedef SingleInt64ExtAttribute SingleIntegerExtAttribute; +using SingleInt8ExtAttribute = SingleExtAttribute<int8_t>; +using SingleInt16ExtAttribute = SingleExtAttribute<int16_t>; +using SingleInt32ExtAttribute = SingleExtAttribute<int32_t>; +using SingleInt64ExtAttribute = SingleExtAttribute<int64_t>; +using SingleFloatExtAttribute = SingleExtAttribute<double>; +using SingleIntegerExtAttribute = SingleInt64ExtAttribute; class SingleStringExtAttribute : public StringDirectAttrVector< AttrVector::Features<false> >, @@ -109,45 +89,22 @@ protected: typedef typename Super::BasicType BasicType; using QueryTermSimpleUP = AttributeVector::QueryTermSimpleUP; - MultiExtAttribute(const vespalib::string &name, const attribute::CollectionType &ctype) - : Super(name, Config(BasicType::fromType(T()), ctype)) - { } + MultiExtAttribute(const vespalib::string &name, const attribute::CollectionType &ctype); private: std::unique_ptr<attribute::SearchContext> - getSearch(QueryTermSimpleUP term, const attribute::SearchContextParams & params) const override - { - (void) term; - (void) params; - return {}; - } + getSearch(QueryTermSimpleUP term, const attribute::SearchContextParams & params) const override; IExtendAttribute * getExtendInterface() override { return this; } public: - MultiExtAttribute(const vespalib::string &name) - : Super(name, Config(BasicType::fromType(static_cast<T>(0)), - attribute::CollectionType::ARRAY)) {} + MultiExtAttribute(const vespalib::string &name); ~MultiExtAttribute() override; - bool addDoc(typename Super::DocId &docId) override { - docId = this->_idx.size() - 1; - this->_idx.push_back(this->_idx.back()); - this->incNumDocs(); - this->setCommittedDocIdLimit(this->getNumDocs()); - return true; - } - bool add(typename AddValueType<T>::Type v, int32_t = 1) override { - this->_data.push_back(v); - std::vector<uint32_t> &idx = this->_idx; - idx.back()++; - this->checkSetMaxValueCount(idx.back() - idx[idx.size() - 2]); - return true; - } + bool addDoc(typename Super::DocId &docId) override; + bool add(typename AddValueType<T>::Type v, int32_t = 1) override; bool onLoad(vespalib::Executor *) override { return false; // Emulate that this attribute is never loaded } - void onAddDocs(uint32_t lidLimit) override { - this->_data.reserve(lidLimit); - } + void onAddDocs(uint32_t lidLimit) override; const attribute::IMultiValueAttribute* as_multi_value_attribute() const override; // Implements attribute::IMultiValueAttribute @@ -155,16 +112,12 @@ public: const attribute::IWeightedSetReadView<T>* make_read_view(attribute::IMultiValueAttribute::WeightedSetTag<T>, vespalib::Stash& stash) const override; }; -template <typename T> -MultiExtAttribute<T>::~MultiExtAttribute() = default; - -typedef MultiExtAttribute<int8_t> MultiInt8ExtAttribute; -typedef MultiExtAttribute<int16_t> MultiInt16ExtAttribute; -typedef MultiExtAttribute<int32_t> MultiInt32ExtAttribute; -typedef MultiExtAttribute<int64_t> MultiInt64ExtAttribute; -typedef MultiExtAttribute<double> MultiFloatExtAttribute; - -typedef MultiInt64ExtAttribute MultiIntegerExtAttribute; +using MultiInt8ExtAttribute = MultiExtAttribute<int8_t>; +using MultiInt16ExtAttribute = MultiExtAttribute<int16_t>; +using MultiInt32ExtAttribute = MultiExtAttribute<int32_t>; +using MultiInt64ExtAttribute = MultiExtAttribute<int64_t>; +using MultiFloatExtAttribute = MultiExtAttribute<double>; +using MultiIntegerExtAttribute = MultiInt64ExtAttribute; class MultiStringExtAttribute : public StringDirectAttrVector< AttrVector::Features<true> >, @@ -199,17 +152,12 @@ private: std::vector<int32_t> _weights; protected: - void addWeight(int32_t w) { - _weights.push_back(w); - } + void addWeight(int32_t w); int32_t getWeightHelper(AttributeVector::DocId docId, uint32_t idx) const { return _weights[this->_idx[docId] + idx]; } - WeightedSetExtAttributeBase(const vespalib::string & name) : - B(name, attribute::CollectionType::WSET), - _weights() - {} - ~WeightedSetExtAttributeBase() {} + WeightedSetExtAttributeBase(const vespalib::string & name); + ~WeightedSetExtAttributeBase(); const std::vector<int32_t>& get_weights() const noexcept { return _weights; } }; @@ -217,12 +165,7 @@ class WeightedSetIntegerExtAttribute : public WeightedSetExtAttributeBase<MultiIntegerExtAttribute> { std::unique_ptr<attribute::SearchContext> - getSearch(QueryTermSimpleUP term, const attribute::SearchContextParams & params) const override - { - (void) term; - (void) params; - return {}; - } + getSearch(QueryTermSimpleUP term, const attribute::SearchContextParams & params) const override; public: WeightedSetIntegerExtAttribute(const vespalib::string & name); ~WeightedSetIntegerExtAttribute(); @@ -236,12 +179,7 @@ class WeightedSetFloatExtAttribute : public WeightedSetExtAttributeBase<MultiFloatExtAttribute> { std::unique_ptr<attribute::SearchContext> - getSearch(QueryTermSimpleUP term, const attribute::SearchContextParams & params) const override - { - (void) term; - (void) params; - return {}; - } + getSearch(QueryTermSimpleUP term, const attribute::SearchContextParams & params) const override; public: WeightedSetFloatExtAttribute(const vespalib::string & name); ~WeightedSetFloatExtAttribute(); diff --git a/searchlib/src/vespa/searchlib/attribute/extendableattributes.hpp b/searchlib/src/vespa/searchlib/attribute/extendableattributes.hpp new file mode 100644 index 00000000000..c515a1bec7f --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/extendableattributes.hpp @@ -0,0 +1,110 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @class search::SearchVisitor + * + * @brief Visitor that applies a search query to visitor data and converts them to a SearchResultCommand + */ +#pragma once + +#include "extendableattributes.h" +#include "attrvector.hpp" + +namespace search { + +template <typename T> +std::unique_ptr<attribute::SearchContext> +SingleExtAttribute<T>::getSearch(QueryTermSimpleUP term, const attribute::SearchContextParams & params) const +{ + (void) term; + (void) params; + return {}; +} + +template <typename T> +SingleExtAttribute<T>::SingleExtAttribute(const vespalib::string &name) + : Super(name, Config(BasicType::fromType(T()), attribute::CollectionType::SINGLE)) +{} + +template <typename T> +bool +SingleExtAttribute<T>::addDoc(typename Super::DocId &docId) { + docId = this->_data.size(); + this->_data.push_back(attribute::getUndefined<T>()); + this->incNumDocs(); + this->setCommittedDocIdLimit(this->getNumDocs()); + return true; +} +template <typename T> +bool +SingleExtAttribute<T>::add(typename AddValueType<T>::Type v, int32_t ) { + this->_data.back() = v; + return true; +} +template <typename T> +void +SingleExtAttribute<T>::onAddDocs(typename Super::DocId lidLimit) { + this->_data.reserve(lidLimit); +} + +template <typename T> +MultiExtAttribute<T>::MultiExtAttribute(const vespalib::string &name, const attribute::CollectionType &ctype) + : Super(name, Config(BasicType::fromType(T()), ctype)) +{ } +template <typename T> +std::unique_ptr<attribute::SearchContext> +MultiExtAttribute<T>::getSearch(QueryTermSimpleUP term, const attribute::SearchContextParams & params) const +{ + (void) term; + (void) params; + return {}; +} + +template <typename T> +MultiExtAttribute<T>::MultiExtAttribute(const vespalib::string &name) + : Super(name, Config(BasicType::fromType(static_cast<T>(0)), attribute::CollectionType::ARRAY)) +{} + +template <typename T> +bool +MultiExtAttribute<T>::addDoc(typename Super::DocId &docId) { + docId = this->_idx.size() - 1; + this->_idx.push_back(this->_idx.back()); + this->incNumDocs(); + this->setCommittedDocIdLimit(this->getNumDocs()); + return true; +} +template <typename T> +bool +MultiExtAttribute<T>::add(typename AddValueType<T>::Type v, int32_t) { + this->_data.push_back(v); + std::vector<uint32_t> &idx = this->_idx; + idx.back()++; + this->checkSetMaxValueCount(idx.back() - idx[idx.size() - 2]); + return true; +} +template <typename T> +void +MultiExtAttribute<T>::onAddDocs(uint32_t lidLimit) { + this->_data.reserve(lidLimit); +} + +template <typename T> +MultiExtAttribute<T>::~MultiExtAttribute() = default; + +template <typename B> +WeightedSetExtAttributeBase<B>::WeightedSetExtAttributeBase(const vespalib::string & name) + : B(name, attribute::CollectionType::WSET), + _weights() +{} + +template <typename B> +WeightedSetExtAttributeBase<B>::~WeightedSetExtAttributeBase() = default; + +template <typename B> +void +WeightedSetExtAttributeBase<B>::addWeight(int32_t w) { + _weights.push_back(w); +} + +} // namespace search + diff --git a/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.cpp b/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.cpp index 22200633229..a2e14ea429d 100644 --- a/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "fixedsourceselector.h" +#include <vespa/searchcommon/attribute/config.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.fixed_source_selector"); @@ -8,7 +9,8 @@ LOG_SETUP(".searchlib.attribute.fixed_source_selector"); namespace search { namespace { - attribute::Config getConfig() { return attribute::Config(attribute::BasicType::INT8); } + +attribute::Config getConfig() { return attribute::Config(attribute::BasicType::INT8); } uint32_t capSelector(queryeval::sourceselector::Iterator::SourceStore &store, queryeval::Source defaultSource) diff --git a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp index 6e19d35a966..ef796d3f3d2 100644 --- a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp @@ -46,7 +46,7 @@ FlagAttributeT<B>::FlagAttributeT(const vespalib::string & baseFileName, const A _bitVectorHolder(), _bitVectorStore(256), _bitVectors(256), - _bitVectorSize(cfg.getGrowStrategy().getDocsInitialCapacity()) + _bitVectorSize(cfg.getGrowStrategy().getInitialCapacity()) { } @@ -211,7 +211,7 @@ void FlagAttributeT<B>::resizeBitVectors(uint32_t neededSize) { const GrowStrategy & gs = this->getConfig().getGrowStrategy(); - uint32_t newSize = neededSize + (neededSize * gs.getDocsGrowFactor()) + gs.getDocsGrowDelta(); + uint32_t newSize = neededSize + (neededSize * gs.getGrowFactor()) + gs.getGrowDelta(); for (size_t i(0), m(_bitVectors.size()); i < m; i++) { BitVector *bv = _bitVectors[i].load_relaxed(); if (bv != nullptr) { diff --git a/searchlib/src/vespa/searchlib/attribute/floatbase.cpp b/searchlib/src/vespa/searchlib/attribute/floatbase.cpp index 3ba79ded478..b9d7fb7c81b 100644 --- a/searchlib/src/vespa/searchlib/attribute/floatbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/floatbase.cpp @@ -6,8 +6,6 @@ namespace search { -IMPLEMENT_IDENTIFIABLE_ABSTRACT(FloatingPointAttribute, NumericAttribute); - FloatingPointAttribute::FloatingPointAttribute(const vespalib::string & name, const Config & c) : NumericAttribute(name, c), _changes() diff --git a/searchlib/src/vespa/searchlib/attribute/floatbase.h b/searchlib/src/vespa/searchlib/attribute/floatbase.h index 8ca6eda6421..40b0b9c72e2 100644 --- a/searchlib/src/vespa/searchlib/attribute/floatbase.h +++ b/searchlib/src/vespa/searchlib/attribute/floatbase.h @@ -11,7 +11,6 @@ namespace search { class FloatingPointAttribute : public NumericAttribute { public: - DECLARE_IDENTIFIABLE_ABSTRACT(FloatingPointAttribute); ~FloatingPointAttribute() override; template<typename Accessor> bool append(DocId doc, Accessor & ac) { @@ -60,7 +59,7 @@ public: using LoadedVector = SequentialReadModifyWriteInterface<LoadedNumericValueT>; virtual T get(DocId doc) const = 0; virtual T getFromEnum(EnumHandle e) const = 0; - T defaultValue() const { return getConfig().isMutable() ? 0.0 : attribute::getUndefined<T>(); } + T defaultValue() const { return isMutable() ? 0.0 : attribute::getUndefined<T>(); } protected: FloatingPointAttributeTemplate(const vespalib::string & name); FloatingPointAttributeTemplate(const vespalib::string & name, const Config & c); diff --git a/searchlib/src/vespa/searchlib/attribute/floatbase.hpp b/searchlib/src/vespa/searchlib/attribute/floatbase.hpp index 7b235b43124..b406f5e85b1 100644 --- a/searchlib/src/vespa/searchlib/attribute/floatbase.hpp +++ b/searchlib/src/vespa/searchlib/attribute/floatbase.hpp @@ -3,6 +3,8 @@ #include "floatbase.h" #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/searchcommon/attribute/config.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h index 0b36e5657f8..15e0c3228bb 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h @@ -2,8 +2,8 @@ #pragma once -#include "attributevector.h" #include "bitvector_search_cache.h" +#include <vespa/searchcommon/attribute/search_context_params.h> #include <vespa/searchcommon/attribute/i_search_context.h> #include <vespa/searchlib/attribute/posting_list_merger.h> #include <vespa/searchlib/common/i_document_meta_store_context.h> @@ -11,9 +11,10 @@ #include <vespa/vespalib/util/arrayref.h> namespace search::fef { class TermFieldMatchData; } - +namespace search { class QueryTermSimple; } namespace search::attribute { +class IAttributeVector; class ReferenceAttribute; class ImportedAttributeVector; class SearchContextParams; @@ -65,7 +66,7 @@ public: const QueryTermUCS4 * queryTerm() const override; const vespalib::string& attributeName() const override; - using DocId = IAttributeVector::DocId; + using DocId = uint32_t; int32_t find(DocId docId, int32_t elemId, int32_t& weight) const { return _target_search_context->find(getTargetLid(docId), elemId, weight); diff --git a/searchlib/src/vespa/searchlib/attribute/integerbase.cpp b/searchlib/src/vespa/searchlib/attribute/integerbase.cpp index cd38b119e7c..b9d33f0aa9e 100644 --- a/searchlib/src/vespa/searchlib/attribute/integerbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/integerbase.cpp @@ -7,8 +7,6 @@ namespace search { -IMPLEMENT_IDENTIFIABLE_ABSTRACT(IntegerAttribute, NumericAttribute); - IntegerAttribute::IntegerAttribute(const vespalib::string & name, const Config & c) : NumericAttribute(name, c), _changes() diff --git a/searchlib/src/vespa/searchlib/attribute/integerbase.h b/searchlib/src/vespa/searchlib/attribute/integerbase.h index 65d16ce934a..170d6fcf941 100644 --- a/searchlib/src/vespa/searchlib/attribute/integerbase.h +++ b/searchlib/src/vespa/searchlib/attribute/integerbase.h @@ -12,7 +12,6 @@ class IntegerAttribute : public NumericAttribute { public: ~IntegerAttribute(); - DECLARE_IDENTIFIABLE_ABSTRACT(IntegerAttribute); bool update(DocId doc, largeint_t v) { return AttributeVector::update(_changes, doc, NumericChangeData<largeint_t>(v)); } @@ -58,7 +57,7 @@ public: using LoadedVector = SequentialReadModifyWriteInterface<LoadedNumericValueT>; virtual T get(DocId doc) const = 0; virtual T getFromEnum(EnumHandle e) const = 0; - T defaultValue() const { return getConfig().isMutable() ? 0 : attribute::getUndefined<T>(); } + T defaultValue() const { return isMutable() ? 0 : attribute::getUndefined<T>(); } protected: IntegerAttributeTemplate(const vespalib::string & name); IntegerAttributeTemplate(const vespalib::string & name, const Config & c); diff --git a/searchlib/src/vespa/searchlib/attribute/integerbase.hpp b/searchlib/src/vespa/searchlib/attribute/integerbase.hpp index 94ffe17af4c..93e9a68b074 100644 --- a/searchlib/src/vespa/searchlib/attribute/integerbase.hpp +++ b/searchlib/src/vespa/searchlib/attribute/integerbase.hpp @@ -3,6 +3,8 @@ #include "integerbase.h" #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/searchcommon/attribute/config.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.h b/searchlib/src/vespa/searchlib/attribute/load_utils.h index 8801b0fe5a4..98b32031d1f 100644 --- a/searchlib/src/vespa/searchlib/attribute/load_utils.h +++ b/searchlib/src/vespa/searchlib/attribute/load_utils.h @@ -3,17 +3,18 @@ #pragma once #include "atomic_utils.h" -#include "attributevector.h" #include "readerbase.h" #include <vespa/vespalib/util/arrayref.h> #include <vespa/searchcommon/attribute/multi_value_traits.h> namespace vespalib::datastore { + class AtomicEntryRef; + class EntryRef; +} -class AtomicEntryRef; -class EntryRef; +namespace vespalib { class GenerationHolder; } -} +namespace search { class AttributeVector; } namespace search::attribute { diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.hpp b/searchlib/src/vespa/searchlib/attribute/load_utils.hpp index 47839f4af98..62d645326ce 100644 --- a/searchlib/src/vespa/searchlib/attribute/load_utils.hpp +++ b/searchlib/src/vespa/searchlib/attribute/load_utils.hpp @@ -3,10 +3,10 @@ #pragma once #include "load_utils.h" +#include "attributevector.h" #include <vespa/searchcommon/attribute/multivalue.h> -namespace search { -namespace attribute { +namespace search::attribute { template <class MvMapping, class Saver> uint32_t @@ -86,5 +86,4 @@ loadFromEnumeratedSingleValue(Vector &vector, } } -} // namespace search::attribute -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp index 2a0fb24d70f..d5b50b591df 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp @@ -4,7 +4,6 @@ #include "multi_value_mapping.h" #include <vespa/vespalib/datastore/array_store.hpp> -#include <vespa/vespalib/util/rcuvector.hpp> namespace search::attribute { diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattribute.cpp b/searchlib/src/vespa/searchlib/attribute/multienumattribute.cpp index c3eefaaaf93..80ba87dde0e 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multienumattribute.cpp @@ -2,6 +2,7 @@ #include "multienumattribute.h" #include "multienumattribute.hpp" +#include "enummodifier.h" #include <stdexcept> namespace search::multienumattribute { @@ -18,7 +19,7 @@ remap_enum_store_refs(const EnumIndexRemapper& remapper, AttributeVector& v, att // update multi_value_mapping with new EnumIndex values after enum store has been compacted. v.logEnumStoreEvent("compactfixup", "drain"); { - AttributeVector::EnumModifier enum_guard(v.getEnumModifier()); + attribute::EnumModifier enum_guard(v.getEnumModifier()); auto& filter = remapper.get_entry_ref_filter(); v.logEnumStoreEvent("compactfixup", "start"); for (uint32_t doc = 0; doc < v.getNumDocs(); ++doc) { diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp index c1cb6b1f75b..ec948882312 100644 --- a/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp @@ -8,6 +8,7 @@ #include "load_utils.h" #include "enum_store_loaders.h" #include "ipostinglistattributebase.h" +#include "valuemodifier.h" #include <vespa/vespalib/stllike/hashtable.hpp> #include <vespa/vespalib/datastore/unique_store_remapper.h> diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp index 0946994f8ec..8cabd8483bf 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp @@ -8,6 +8,7 @@ #include "multi_numeric_search_context.h" #include "load_utils.h" #include "primitivereader.h" +#include "valuemodifier.h" #include <vespa/searchlib/query/query_term_simple.h> #include <vespa/searchlib/util/fileutil.h> diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp index 622058550a2..cbe46a8cb2e 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp @@ -6,6 +6,7 @@ #include "raw_multi_value_read_view.h" #include "copy_multi_value_read_view.h" #include <vespa/searchlib/attribute/multivalueattribute.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/memory_allocator.h> @@ -31,7 +32,7 @@ MultiValueAttribute(const vespalib::string &baseFileName, 8 * 1024, cfg.getGrowStrategy().getMultiValueAllocGrowFactor(), multivalueattribute::enable_free_lists), - cfg.getGrowStrategy().to_generic_strategy(), this->get_memory_allocator()) + cfg.getGrowStrategy(), this->get_memory_allocator()) { } diff --git a/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.cpp index 0602d983513..d8b01afe094 100644 --- a/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.cpp @@ -2,150 +2,140 @@ #include "not_implemented_attribute.h" #include "search_context.h" +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/classname.h> using vespalib::make_string_short::fmt; +using vespalib::getClassName; + namespace search { using largeint_t = attribute::IAttributeVector::largeint_t; using attribute::SearchContext; +NotImplementedAttribute::NotImplementedAttribute(const vespalib::string &name) + : NotImplementedAttribute(name, Config()) +{} + +NotImplementedAttribute::NotImplementedAttribute(const vespalib::string &name, const Config & cfg) + : AttributeVector(name, cfg) +{} + void NotImplementedAttribute::notImplemented() const { throw vespalib::UnsupportedOperationException(fmt("The function is not implemented for attribute '%s' of type '%s'.", - getName().c_str(), getNativeClassName().c_str())); + getName().c_str(), getClassName(*this).c_str())); } uint32_t NotImplementedAttribute::getValueCount(DocId) const { notImplemented(); - return 0; } largeint_t NotImplementedAttribute::getInt(DocId) const { notImplemented(); - return 0; } double NotImplementedAttribute::getFloat(DocId) const { notImplemented(); - return 0; } const char * NotImplementedAttribute::getString(DocId, char *, size_t) const { notImplemented(); - return NULL; } uint32_t NotImplementedAttribute::get(DocId, largeint_t *, uint32_t) const { notImplemented(); - return 0; } uint32_t NotImplementedAttribute::get(DocId, double *, uint32_t) const { notImplemented(); - return 0; } uint32_t NotImplementedAttribute::get(DocId, vespalib::string *, uint32_t) const { notImplemented(); - return 0; } uint32_t NotImplementedAttribute::get(DocId, const char **, uint32_t) const { notImplemented(); - return 0; } uint32_t NotImplementedAttribute::get(DocId, EnumHandle *, uint32_t) const { notImplemented(); - return 0; } uint32_t NotImplementedAttribute::get(DocId, WeightedInt *, uint32_t) const { notImplemented(); - return 0; } uint32_t NotImplementedAttribute::get(DocId, WeightedFloat *, uint32_t) const { notImplemented(); - return 0; } uint32_t NotImplementedAttribute::get(DocId, WeightedString *, uint32_t) const { notImplemented(); - return 0; } uint32_t NotImplementedAttribute::get(DocId, WeightedConstChar *, uint32_t) const { notImplemented(); - return 0; } uint32_t NotImplementedAttribute::get(DocId, WeightedEnum *, uint32_t) const { notImplemented(); - return 0; } bool NotImplementedAttribute::findEnum(const char *, EnumHandle &) const { notImplemented(); - return false; } std::vector<NotImplementedAttribute::EnumHandle> NotImplementedAttribute::findFoldedEnums(const char *) const { notImplemented(); - return std::vector<EnumHandle>(); } long NotImplementedAttribute::onSerializeForAscendingSort(DocId, void *, long, const common::BlobConverter *) const { notImplemented(); - return 0; } long NotImplementedAttribute::onSerializeForDescendingSort(DocId, void *, long, const common::BlobConverter *) const { notImplemented(); - return 0; } uint32_t NotImplementedAttribute::clearDoc(DocId) { notImplemented(); - return 0; } uint32_t NotImplementedAttribute::getEnum(DocId) const { notImplemented(); - return 0; } bool NotImplementedAttribute::addDoc(DocId &) { notImplemented(); - return false; } std::unique_ptr<SearchContext> NotImplementedAttribute::getSearch(QueryTermSimpleUP, const attribute::SearchContextParams &) const { notImplemented(); - return {}; } void NotImplementedAttribute::onAddDocs(DocId ) { } diff --git a/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h b/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h index 8987681d640..9f951f6a41d 100644 --- a/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h +++ b/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h @@ -7,8 +7,8 @@ namespace search { struct NotImplementedAttribute : AttributeVector { - using AttributeVector::AttributeVector; - + NotImplementedAttribute(const vespalib::string & name); + NotImplementedAttribute(const vespalib::string & name, const Config & config); void notImplemented [[noreturn]] () const; uint32_t getValueCount(DocId) const override; diff --git a/searchlib/src/vespa/searchlib/attribute/numericbase.cpp b/searchlib/src/vespa/searchlib/attribute/numericbase.cpp index 84e7cd841f4..d59616c5965 100644 --- a/searchlib/src/vespa/searchlib/attribute/numericbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/numericbase.cpp @@ -8,8 +8,6 @@ LOG_SETUP(".searchlib.attribute.numericbase"); namespace search { -IMPLEMENT_IDENTIFIABLE_ABSTRACT(NumericAttribute, AttributeVector); - void NumericAttribute::load_enumerated_data(ReaderBase&, enumstore::EnumeratedPostingsLoader&, diff --git a/searchlib/src/vespa/searchlib/attribute/numericbase.h b/searchlib/src/vespa/searchlib/attribute/numericbase.h index 91f50fc2961..f19ddcca706 100644 --- a/searchlib/src/vespa/searchlib/attribute/numericbase.h +++ b/searchlib/src/vespa/searchlib/attribute/numericbase.h @@ -24,9 +24,6 @@ protected: virtual void load_enumerated_data(ReaderBase& attrReader, enumstore::EnumeratedLoader& loader); virtual void load_posting_lists_and_update_enum_store(enumstore::EnumeratedPostingsLoader& loader); bool onAddDoc(DocId) override { return true; } - -public: - DECLARE_IDENTIFIABLE_ABSTRACT(NumericAttribute); }; } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp index 6268a6da701..29f04ebab27 100644 --- a/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp @@ -8,6 +8,7 @@ #include <vespa/document/predicate/predicate.h> #include <vespa/searchlib/predicate/predicate_index.h> #include <vespa/searchlib/util/fileutil.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/util/size_literals.h> @@ -64,11 +65,15 @@ adjustUpperBound(int32_t arity, int64_t upper_bound) { SimpleIndexConfig createSimpleIndexConfig(const search::attribute::Config &config) { return SimpleIndexConfig(config.predicateParams().dense_posting_list_threshold(), - config.getGrowStrategy().to_generic_strategy()); + config.getGrowStrategy()); } } // namespace +PredicateAttribute::PredicateAttribute(const vespalib::string &base_file_name) + : PredicateAttribute(base_file_name, Config(BasicType::PREDICATE)) +{} + PredicateAttribute::PredicateAttribute(const vespalib::string &base_file_name, const Config &config) : NotImplementedAttribute(base_file_name, config), _limit_provider(*this), @@ -76,8 +81,8 @@ PredicateAttribute::PredicateAttribute(const vespalib::string &base_file_name, c createSimpleIndexConfig(config), config.predicateParams().arity())), _lower_bound(adjustLowerBound(config.predicateParams().arity(), config.predicateParams().lower_bound())), _upper_bound(adjustUpperBound(config.predicateParams().arity(), config.predicateParams().upper_bound())), - _min_feature(config.getGrowStrategy().to_generic_strategy(), getGenerationHolder()), - _interval_range_vector(config.getGrowStrategy().to_generic_strategy(), getGenerationHolder()), + _min_feature(config.getGrowStrategy(), getGenerationHolder()), + _interval_range_vector(config.getGrowStrategy(), getGenerationHolder()), _max_interval_range(1) { } @@ -283,6 +288,4 @@ PredicateAttribute::updateValue(uint32_t doc_id, const PredicateFieldValue &valu assert(result.interval_range > 0); } -IMPLEMENT_IDENTIFIABLE_ABSTRACT(PredicateAttribute, AttributeVector); - } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/predicate_attribute.h b/searchlib/src/vespa/searchlib/attribute/predicate_attribute.h index 40823faff8a..f5d789298a0 100644 --- a/searchlib/src/vespa/searchlib/attribute/predicate_attribute.h +++ b/searchlib/src/vespa/searchlib/attribute/predicate_attribute.h @@ -38,8 +38,7 @@ public: using IntervalRange = uint16_t; using IntervalRangeVector = vespalib::RcuVectorBase<IntervalRange>; - DECLARE_IDENTIFIABLE_ABSTRACT(PredicateAttribute); - + PredicateAttribute(const vespalib::string &base_file_name); PredicateAttribute(const vespalib::string &base_file_name, const Config &config); ~PredicateAttribute() override; diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp index 473174de284..7ff04f789b2 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp @@ -11,6 +11,7 @@ #include <vespa/searchlib/common/i_gid_to_lid_mapper.h> #include <vespa/searchlib/common/i_gid_to_lid_mapper_factory.h> #include <vespa/searchlib/query/query_term_simple.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/datastore/unique_store_builder.h> #include <vespa/vespalib/datastore/datastore.hpp> @@ -39,11 +40,14 @@ extractUniqueValueCount(const vespalib::GenericHeader &header) } -ReferenceAttribute::ReferenceAttribute(const vespalib::stringref baseFileName, - const Config & cfg) +ReferenceAttribute::ReferenceAttribute(const vespalib::stringref baseFileName) + : ReferenceAttribute(baseFileName, Config(BasicType::REFERENCE)) +{} + +ReferenceAttribute::ReferenceAttribute(const vespalib::stringref baseFileName, const Config & cfg) : NotImplementedAttribute(baseFileName, cfg), _store({}), - _indices(getGenerationHolder()), + _indices(cfg.getGrowStrategy(), getGenerationHolder()), _compaction_spec(), _gidToLidMapperFactory(), _referenceMappings(getGenerationHolder(), getCommittedDocIdLimitRef()) @@ -507,8 +511,6 @@ ReferenceAttribute::getSearch(QueryTermSimpleUP term, const attribute::SearchCon return std::make_unique<ReferenceSearchContext>(*this, gid); } -IMPLEMENT_IDENTIFIABLE_ABSTRACT(ReferenceAttribute, AttributeVector); - } namespace vespalib::datastore { diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.h b/searchlib/src/vespa/searchlib/attribute/reference_attribute.h index c86793036cd..dc3e2ad729a 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.h +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.h @@ -69,9 +69,8 @@ private: public: using SP = std::shared_ptr<ReferenceAttribute>; - DECLARE_IDENTIFIABLE_ABSTRACT(ReferenceAttribute); - ReferenceAttribute(const vespalib::stringref baseFileName, - const Config & cfg); + ReferenceAttribute(const vespalib::stringref baseFileName); + ReferenceAttribute(const vespalib::stringref baseFileName, const Config & cfg); ~ReferenceAttribute() override; bool addDoc(DocId &doc) override; uint32_t clearDoc(DocId doc) override; diff --git a/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp b/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp index 65bbd9514c7..37b7a82ab55 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_mappings.cpp @@ -9,17 +9,15 @@ namespace search::attribute { ReferenceMappings::ReferenceMappings(GenerationHolder &genHolder, const std::atomic<uint32_t>& committedDocIdLimit) - : _reverseMappingIndices(genHolder), + : _reverseMappingIndices(vespalib::GrowStrategy(16, 1.0, 0, 0), genHolder), _targetLidLimit(0), _reverseMapping(), - _targetLids(genHolder), + _targetLids(vespalib::GrowStrategy(16, 1.0, 0, 0), genHolder), _committedDocIdLimit(committedDocIdLimit) { } -ReferenceMappings::~ReferenceMappings() -{ -} +ReferenceMappings::~ReferenceMappings() = default; void ReferenceMappings::clearMapping(const Reference &entry) diff --git a/searchlib/src/vespa/searchlib/attribute/search_context.cpp b/searchlib/src/vespa/searchlib/attribute/search_context.cpp index 8e8cca13671..3deacca5764 100644 --- a/searchlib/src/vespa/searchlib/attribute/search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/search_context.cpp @@ -74,7 +74,7 @@ SearchContext::attributeName() const bool SearchContext::getIsFilter() const { - return _attr.getConfig().getIsFilter(); + return _attr.getIsFilter(); } diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp index b0765680f78..7574508517d 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp @@ -6,10 +6,12 @@ #include "ipostinglistsearchcontext.h" #include "primitivereader.h" #include "search_context.h" +#include "valuemodifier.h" #include <vespa/searchlib/common/bitvectoriterator.h> #include <vespa/searchlib/query/query_term_simple.h> #include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/util/file_settings.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/util/size_literals.h> @@ -34,7 +36,7 @@ void SingleBoolAttribute::ensureRoom(DocId docIdLimit) { if (_bv.writer().capacity() < docIdLimit) { const GrowStrategy & gs = this->getConfig().getGrowStrategy(); - uint32_t newSize = docIdLimit + (docIdLimit * gs.getDocsGrowFactor()) + gs.getDocsGrowDelta(); + uint32_t newSize = docIdLimit + (docIdLimit * gs.getGrowFactor()) + gs.getGrowDelta(); bool incGen = _bv.reserve(newSize); if (incGen) { incGeneration(); diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h index 3465269de8a..7868c228e77 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h @@ -8,6 +8,8 @@ namespace search { +class GrowStrategy; + /** * Attributevector for boolean field values occupying a bit per document * and backed by a growable rcu bit vector. @@ -15,7 +17,7 @@ namespace search { class SingleBoolAttribute final : public IntegerAttributeTemplate<int8_t> { public: - SingleBoolAttribute(const vespalib::string & baseFileName, const search::GrowStrategy & grow, bool paged); + SingleBoolAttribute(const vespalib::string & baseFileName, const GrowStrategy & grow, bool paged); ~SingleBoolAttribute() override; void onCommit() override; diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp index 38ef186b717..a0a93946bb0 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.cpp @@ -5,6 +5,8 @@ #include "stringbase.h" #include "integerbase.h" #include "floatbase.h" +#include "enumattribute.h" +#include "enummodifier.h" #include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.single_enum_attribute"); @@ -15,18 +17,11 @@ using attribute::Config; SingleValueEnumAttributeBase:: SingleValueEnumAttributeBase(const Config & c, GenerationHolder &genHolder, const vespalib::alloc::Alloc& initial_alloc) - : _enumIndices(c.getGrowStrategy().getDocsInitialCapacity(), - c.getGrowStrategy().getDocsGrowPercent(), - c.getGrowStrategy().getDocsGrowDelta(), - genHolder, initial_alloc) -{ -} - - -SingleValueEnumAttributeBase::~SingleValueEnumAttributeBase() + : _enumIndices(c.getGrowStrategy(), genHolder, initial_alloc) { } +SingleValueEnumAttributeBase::~SingleValueEnumAttributeBase() = default; AttributeVector::DocId SingleValueEnumAttributeBase::addDoc(bool &incGeneration) @@ -36,7 +31,6 @@ SingleValueEnumAttributeBase::addDoc(bool &incGeneration) return _enumIndices.size() - 1; } - SingleValueEnumAttributeBase::EnumIndexCopyVector SingleValueEnumAttributeBase::getIndicesCopy(uint32_t size) const { @@ -68,7 +62,7 @@ SingleValueEnumAttributeBase::remap_enum_store_refs(const EnumIndexRemapper& rem } v.logEnumStoreEvent("compactfixup", "drain"); { - AttributeVector::EnumModifier enum_guard(v.getEnumModifier()); + attribute::EnumModifier enum_guard(v.getEnumModifier()); v.logEnumStoreEvent("compactfixup", "start"); _enumIndices.replaceVector(std::move(new_indexes)); } diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp index 9f11b444c2c..c4abcfbc25a 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp @@ -8,6 +8,7 @@ #include "singleenumattributesaver.h" #include "load_utils.h" #include "enum_store_loaders.h" +#include "valuemodifier.h" #include <vespa/vespalib/datastore/unique_store_remapper.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.h b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.h index cf3fa85a060..2e9303bef62 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.h @@ -41,12 +41,8 @@ protected: } public: - SingleValueNumericAttribute(const vespalib::string & baseFileName, - const AttributeVector::Config & c = - AttributeVector::Config(AttributeVector:: - BasicType::fromType(T()), - attribute::CollectionType::SINGLE)); - + SingleValueNumericAttribute(const vespalib::string & baseFileName); // Only for testing + SingleValueNumericAttribute(const vespalib::string & baseFileName, const AttributeVector::Config & c); ~SingleValueNumericAttribute(); diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp index 917f0f55894..b9c1c3686de 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp @@ -9,19 +9,24 @@ #include "singlenumericattribute.h" #include "singlenumericattributesaver.h" #include "single_numeric_search_context.h" +#include "valuemodifier.h" #include <vespa/searchlib/query/query_term_simple.h> +#include <vespa/searchcommon/attribute/config.h> namespace search { template <typename B> SingleValueNumericAttribute<B>:: -SingleValueNumericAttribute(const vespalib::string & baseFileName, const AttributeVector::Config & c) : - B(baseFileName, c), - _data(c.getGrowStrategy().getDocsInitialCapacity(), - c.getGrowStrategy().getDocsGrowPercent(), - c.getGrowStrategy().getDocsGrowDelta(), - getGenerationHolder(), - this->get_initial_alloc()) +SingleValueNumericAttribute(const vespalib::string & baseFileName) + : SingleValueNumericAttribute(baseFileName, attribute::Config(attribute::BasicType::fromType(T()), + attribute::CollectionType::SINGLE)) +{ } + +template <typename B> +SingleValueNumericAttribute<B>:: +SingleValueNumericAttribute(const vespalib::string & baseFileName, const AttributeVector::Config & c) + : B(baseFileName, c), + _data(c.getGrowStrategy(), getGenerationHolder(), this->get_initial_alloc()) { } template <typename B> diff --git a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp index eca74255026..b69ed017b52 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.cpp @@ -5,8 +5,10 @@ #include "iattributesavetarget.h" #include "primitivereader.h" #include "single_small_numeric_search_context.h" +#include "valuemodifier.h" #include <vespa/searchlib/query/query_term_simple.h> #include <vespa/searchlib/util/file_settings.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/util/size_literals.h> @@ -24,10 +26,7 @@ SingleValueSmallNumericAttribute(const vespalib::string & baseFileName, _valueShiftShift(valueShiftShift), _valueShiftMask(valueShiftMask), _wordShift(wordShift), - _wordData(c.getGrowStrategy().getDocsInitialCapacity(), - c.getGrowStrategy().getDocsGrowPercent(), - c.getGrowStrategy().getDocsGrowDelta(), - getGenerationHolder()) + _wordData(c.getGrowStrategy(), getGenerationHolder()) { assert(_valueMask + 1 == (1u << (1u << valueShiftShift))); assert((_valueShiftMask + 1) * (1u << valueShiftShift) == 8 * sizeof(Word)); diff --git a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.h b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.h index 646edc786a3..4bf120d7952 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.h @@ -10,6 +10,8 @@ namespace search { +class GrowStrategy; + class SingleValueSmallNumericAttribute : public IntegerAttributeTemplate<int8_t> { private: @@ -32,7 +34,7 @@ private: uint32_t _valueShiftMask; // 0x1f, 0x0f or 0x07 uint32_t _wordShift; // 0x05, 0x04 or 0x03 - typedef vespalib::RcuVectorBase<Word> DataVector; + using DataVector = vespalib::RcuVectorBase<Word>; DataVector _wordData; T getFromEnum(EnumHandle) const override { @@ -143,13 +145,13 @@ public: class SingleValueSemiNibbleNumericAttribute : public SingleValueSmallNumericAttribute { public: - SingleValueSemiNibbleNumericAttribute(const vespalib::string & baseFileName, const search::GrowStrategy & grow); + SingleValueSemiNibbleNumericAttribute(const vespalib::string & baseFileName, const GrowStrategy & grow); }; class SingleValueNibbleNumericAttribute : public SingleValueSmallNumericAttribute { public: - SingleValueNibbleNumericAttribute(const vespalib::string & baseFileName, const search::GrowStrategy & grow); + SingleValueNibbleNumericAttribute(const vespalib::string & baseFileName, const GrowStrategy & grow); }; } diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp index b60ec269383..48321747fc3 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp @@ -8,6 +8,7 @@ #include <vespa/searchlib/common/sort.h> #include <vespa/document/fieldvalue/fieldvalue.h> #include <vespa/searchlib/query/query_term_ucs4.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/locale/c.h> #include <vespa/vespalib/util/array.hpp> @@ -16,8 +17,6 @@ LOG_SETUP(".searchlib.attribute.stringbase"); namespace search { -IMPLEMENT_IDENTIFIABLE_ABSTRACT(StringAttribute, AttributeVector); - size_t StringAttribute::countZero(const char * bt, size_t sz) { @@ -285,6 +284,11 @@ StringAttribute::getChangeVectorMemoryUsage() const return _changes.getMemoryUsage(); } +bool +StringAttribute::get_match_is_cased() const noexcept { + return getConfig().get_match() == attribute::Config::Match::CASED; +} + template bool AttributeVector::clearDoc(StringAttribute::ChangeVector& changes, DocId doc); template bool AttributeVector::update(StringAttribute::ChangeVector& changes, DocId doc, const StringChangeData& v); template bool AttributeVector::append(StringAttribute::ChangeVector& changes, DocId doc, const StringChangeData& v, int32_t w, bool doCount); diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.h b/searchlib/src/vespa/searchlib/attribute/stringbase.h index e5e14829118..b1e55c3d640 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.h +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.h @@ -25,9 +25,8 @@ public: using EnumVector = IEnumStore::EnumVector; using LoadedValueType = const char*; using LoadedVector = NoLoadedVector; - using OffsetVector = vespalib::Array<uint32_t>; + using OffsetVector = std::vector<uint32_t, vespalib::allocator_large<uint32_t>>; public: - DECLARE_IDENTIFIABLE_ABSTRACT(StringAttribute); bool append(DocId doc, const vespalib::string & v, int32_t weight) { return AttributeVector::append(_changes, doc, StringChangeData(v), weight); } @@ -73,7 +72,7 @@ protected: vespalib::MemoryUsage getChangeVectorMemoryUsage() const override; - bool get_match_is_cased() const noexcept { return getConfig().get_match() == attribute::Config::Match::CASED; } + bool get_match_is_cased() const noexcept; private: virtual void load_posting_lists(LoadedVector& loaded); virtual void load_enum_store(LoadedVector& loaded); diff --git a/searchlib/src/vespa/searchlib/attribute/valuemodifier.cpp b/searchlib/src/vespa/searchlib/attribute/valuemodifier.cpp new file mode 100644 index 00000000000..56bffa2a551 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/valuemodifier.cpp @@ -0,0 +1,18 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "valuemodifier.h" +#include "attributevector.h" + +namespace search::attribute { + +ValueModifier::ValueModifier(AttributeVector &attr) + : _attr(&attr) +{ } + +ValueModifier::~ValueModifier() { + if (_attr) { + _attr->incGeneration(); + } +} + +} diff --git a/searchlib/src/vespa/searchlib/attribute/valuemodifier.h b/searchlib/src/vespa/searchlib/attribute/valuemodifier.h new file mode 100644 index 00000000000..47416eedc51 --- /dev/null +++ b/searchlib/src/vespa/searchlib/attribute/valuemodifier.h @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "interlock.h" + +namespace search { class AttributeVector; } + +namespace search::attribute { + +class ValueModifier +{ +public: + ValueModifier(AttributeVector &attr); + ValueModifier(const ValueModifier &) = delete; + ValueModifier & operator = (const ValueModifier &) = delete; + ~ValueModifier(); +private: + AttributeVector * _attr; +}; + +} diff --git a/searchlib/src/vespa/searchlib/common/condensedbitvectors.cpp b/searchlib/src/vespa/searchlib/common/condensedbitvectors.cpp index 569e1275080..15e09172718 100644 --- a/searchlib/src/vespa/searchlib/common/condensedbitvectors.cpp +++ b/searchlib/src/vespa/searchlib/common/condensedbitvectors.cpp @@ -17,7 +17,7 @@ class CondensedBitVectorT : public CondensedBitVector { public: CondensedBitVectorT(size_t sz, GenerationHolder &genHolder) : - _v(sz, 30, 1000, genHolder) + _v(vespalib::GrowStrategy(sz, 30, 1000, 0), genHolder) { for (size_t i = 0; i < sz; ++i) { _v.push_back(0); diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp index 51a4d392839..8531ea66d38 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp @@ -9,7 +9,6 @@ #include <vespa/vespalib/util/benchmark_timer.h> #include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/exceptions.h> -#include <vespa/vespalib/util/rcuvector.hpp> #include <vespa/vespalib/util/size_literals.h> #include <thread> @@ -67,9 +66,7 @@ LogDataStore::LogDataStore(vespalib::Executor &executor, const vespalib::string _tune(tune), _fileHeaderContext(fileHeaderContext), _genHandler(), - _lidInfo(growStrategy.getDocsInitialCapacity(), - growStrategy.getDocsGrowPercent(), - growStrategy.getDocsGrowDelta()), + _lidInfo(growStrategy), _fileChunks(), _holdFileChunks(), _active(0), diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_hash.h b/searchlib/src/vespa/searchlib/predicate/predicate_hash.h index 9000d8d509b..39ffe57745e 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_hash.h +++ b/searchlib/src/vespa/searchlib/predicate/predicate_hash.h @@ -14,7 +14,7 @@ struct PredicateHash { } static uint64_t hash64(const void *data, uint32_t origLen) { - int64_t a, b, c; + uint64_t a, b, c; int offset; // Current offset into the entire key. const uint8_t *aKey = static_cast<const uint8_t *>(data); @@ -54,18 +54,18 @@ struct PredicateHash { ((0xffLL & aKey[offset+23])<<56)); // Mix. This arithmetic must match the mix below. - a -= b; a -= c; a ^= (((uint64_t) c)>>43); + a -= b; a -= c; a ^= (c>>43); b -= c; b -= a; b ^= (a<<9); - c -= a; c -= b; c ^= (((uint64_t) b)>>8); - a -= b; a -= c; a ^= (((uint64_t) c)>>38); + c -= a; c -= b; c ^= (b>>8); + a -= b; a -= c; a ^= (c>>38); b -= c; b -= a; b ^= (a<<23); - c -= a; c -= b; c ^= (((uint64_t) b)>>5); - a -= b; a -= c; a ^= (((uint64_t) c)>>35); + c -= a; c -= b; c ^= (b>>5); + a -= b; a -= c; a ^= (c>>35); b -= c; b -= a; b ^= (a<<49); - c -= a; c -= b; c ^= (((uint64_t) b)>>11); - a -= b; a -= c; a ^= (((uint64_t) c)>>12); + c -= a; c -= b; c ^= (b>>11); + a -= b; a -= c; a ^= (c>>12); b -= c; b -= a; b ^= (a<<18); - c -= a; c -= b; c ^= (((uint64_t) b)>>22); + c -= a; c -= b; c ^= (b>>22); // End mix. offset += 24; len -= 24; @@ -102,21 +102,21 @@ struct PredicateHash { } // Mix. This arithmetic must match the mix above. - a -= b; a -= c; a ^= (((uint64_t) c)>>43); + a -= b; a -= c; a ^= (c>>43); b -= c; b -= a; b ^= (a<<9); - c -= a; c -= b; c ^= (((uint64_t) b)>>8); - a -= b; a -= c; a ^= (((uint64_t) c)>>38); + c -= a; c -= b; c ^= (b>>8); + a -= b; a -= c; a ^= (c>>38); b -= c; b -= a; b ^= (a<<23); - c -= a; c -= b; c ^= (((uint64_t) b)>>5); - a -= b; a -= c; a ^= (((uint64_t) c)>>35); + c -= a; c -= b; c ^= (b>>5); + a -= b; a -= c; a ^= (c>>35); b -= c; b -= a; b ^= (a<<49); - c -= a; c -= b; c ^= (((uint64_t) b)>>11); - a -= b; a -= c; a ^= (((uint64_t) c)>>12); + c -= a; c -= b; c ^= (b>>11); + a -= b; a -= c; a ^= (c>>12); b -= c; b -= a; b ^= (a<<18); - c -= a; c -= b; c ^= (((uint64_t) b)>>22); + c -= a; c -= b; c ^= (b>>22); // End mix. - return static_cast<uint64_t>(c); + return c; } }; diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_range_expander.h b/searchlib/src/vespa/searchlib/predicate/predicate_range_expander.h index 99d699b5bf3..79417f51f82 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_range_expander.h +++ b/searchlib/src/vespa/searchlib/predicate/predicate_range_expander.h @@ -103,10 +103,10 @@ public: if (to < 0) { // Special case for to==-1. -X-0 means the same as -X-1, // but is more efficient. - partitionRange(label, (to == -1 ? 0 : -to), -from, arity, + partitionRange(label, (to == -1 ? 0 : -to), uint64_t(0)-from, arity, true, out); } else { - partitionRange(label, 0, -from, arity, true, out); + partitionRange(label, 0, uint64_t(0)-from, arity, true, out); partitionRange(label, 0, to, arity, false, out); } } else { diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h b/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h index e0386dc4b44..90792912e64 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h +++ b/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h @@ -30,7 +30,7 @@ public: _upper_bound(upper_bound) { uint64_t t = _upper_bound; while ((t /= _arity) > 0) ++_max_positive_levels; - t = -_lower_bound; + t = uint64_t(0)-_lower_bound; while ((t /= _arity) > 0) ++_max_negative_levels; } @@ -55,7 +55,7 @@ void PredicateRangeTermExpander::expand(const vespalib::string &key, int64_t sig uint64_t value; int max_levels; if (negative) { - value = -signed_value; + value = uint64_t(0)-signed_value; buffer[prefix_size++] = '-'; max_levels = _max_negative_levels; } else { @@ -72,7 +72,7 @@ void PredicateRangeTermExpander::expand(const vespalib::string &key, int64_t sig for (int i = 0; i < max_levels; ++i) { uint64_t start = (value / level_size) * level_size; if (negative) { - if (start + level_size - 1 > uint64_t(-LLONG_MIN)) { + if (start + level_size - 1 > (uint64_t(0)-LLONG_MIN)) { break; } size = sprintf(buffer + prefix_size, "%" PRIu64 "-%" PRIu64, diff --git a/searchlib/src/vespa/searchlib/predicate/simple_index.hpp b/searchlib/src/vespa/searchlib/predicate/simple_index.hpp index 3373b6f5ca7..cb37fec26ea 100644 --- a/searchlib/src/vespa/searchlib/predicate/simple_index.hpp +++ b/searchlib/src/vespa/searchlib/predicate/simple_index.hpp @@ -3,7 +3,6 @@ #include "simple_index.h" #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/util/rcuvector.hpp> namespace search::predicate { diff --git a/searchlib/src/vespa/searchlib/predicate/tree_crumbs.h b/searchlib/src/vespa/searchlib/predicate/tree_crumbs.h index 41c41d317bb..a0a8ac6704f 100644 --- a/searchlib/src/vespa/searchlib/predicate/tree_crumbs.h +++ b/searchlib/src/vespa/searchlib/predicate/tree_crumbs.h @@ -34,6 +34,9 @@ public: size_t size() const { return _buffer.size(); } std::string getCrumb() const { + if (_buffer.empty()) { + return std::string(); + } return std::string(&_buffer[0], _buffer.size()); } }; diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp index 179a3fc29dd..8c03800b92a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp @@ -8,6 +8,7 @@ #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/distance_function_factory.h> +#include <vespa/vespalib/objects/objectvisitor.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.queryeval.nearest_neighbor_blueprint"); @@ -171,8 +172,12 @@ NearestNeighborBlueprint::createLeafSearch(const search::fef::TermFieldMatchData { assert(tfmda.size() == 1); fef::TermFieldMatchData &tfmd = *tfmda[0]; // always search in only one field - if (! _found_hits.empty()) { + switch (_algorithm) { + case Algorithm::INDEX_TOP_K_WITH_FILTER: + case Algorithm::INDEX_TOP_K: return NnsIndexIterator::create(tfmd, _found_hits, _dist_fun); + default: + ; } const Value &qT = *_query_tensor; return NearestNeighborIterator::create(strict, tfmd, qT, _attr_tensor, diff --git a/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp b/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp index 8357d403000..3d6f9669456 100644 --- a/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/sourceblendersearch.cpp @@ -2,6 +2,7 @@ #include "sourceblendersearch.h" #include "isourceselector.h" +#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/objects/visit.hpp> #include <vespa/vespalib/util/array.hpp> diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index a3b197ac499..2fdb73fcf96 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -10,6 +10,7 @@ #include <vespa/fastlib/io/bufferedfile.h> #include <vespa/searchlib/attribute/load_utils.h> #include <vespa/searchlib/attribute/readerbase.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/data/slime/inserter.h> #include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/lambdatask.h> diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp index 427e6700e8c..56af2ed3b35 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp @@ -3,7 +3,6 @@ #include "hnsw_graph.h" #include "hnsw_index.h" #include <vespa/vespalib/datastore/array_store.hpp> -#include <vespa/vespalib/util/rcuvector.hpp> namespace search::tensor { diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h index 06482c075cb..9e3850bd470 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h @@ -31,7 +31,6 @@ struct HnswGraph { // This stores the level arrays for all nodes. // Each node consists of an array of levels (from level 0 to n) where each entry is a reference to the link array at that level. using NodeStore = vespalib::datastore::ArrayStore<AtomicEntryRef, LevelArrayEntryRefType>; - using StoreConfig = vespalib::datastore::ArrayStoreConfig; using LevelArrayRef = NodeStore::ConstArrayRef; // This stores all link arrays. diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp index 0e09118a3f3..24bfacb8c5a 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp @@ -7,7 +7,6 @@ #include "hnsw_index_loader.hpp" #include "hnsw_index_saver.h" #include "random_level_generator.h" -#include "reusable_set_visited_tracker.h" #include <vespa/searchlib/attribute/address_space_components.h> #include <vespa/searchlib/attribute/address_space_usage.h> #include <vespa/searchlib/util/fileutil.h> @@ -17,7 +16,6 @@ #include <vespa/vespalib/datastore/array_store.hpp> #include <vespa/vespalib/datastore/compaction_strategy.h> #include <vespa/vespalib/util/memory_allocator.h> -#include <vespa/vespalib/util/rcuvector.hpp> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/time.h> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h index 48f4cb9d494..72a10724ff1 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h @@ -15,7 +15,6 @@ #include <vespa/vespalib/datastore/atomic_entry_ref.h> #include <vespa/vespalib/datastore/compaction_spec.h> #include <vespa/vespalib/datastore/entryref.h> -#include <vespa/vespalib/util/rcuvector.h> #include <vespa/vespalib/util/reusable_set_pool.h> #include <vespa/vespalib/stllike/allocator.h> diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp index 87537298d1f..2233eb77e89 100644 --- a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp @@ -4,7 +4,8 @@ #include "streamed_value_saver.h" #include <vespa/eval/eval/value.h> #include <vespa/fastlib/io/bufferedfile.h> -#include <vespa/vespalib/util/rcuvector.hpp> +#include <vespa/searchcommon/attribute/config.h> + #include <vespa/log/log.h> LOG_SETUP(".searchlib.tensor.serialized_fast_value_attribute"); diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp index c1cafd547aa..add5184c4eb 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp @@ -5,9 +5,9 @@ #include <vespa/document/datatype/tensor_data_type.h> #include <vespa/searchlib/attribute/address_space_components.h> #include <vespa/searchlib/util/state_explorer_utils.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/data/slime/cursor.h> #include <vespa/vespalib/data/slime/inserter.h> -#include <vespa/vespalib/util/rcuvector.hpp> #include <vespa/vespalib/util/shared_string_repo.h> #include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/value_codec.h> @@ -49,10 +49,7 @@ vespalib::string makeWrongTensorTypeMsg(const ValueType &fieldTensorType, const TensorAttribute::TensorAttribute(vespalib::stringref name, const Config &cfg, TensorStore &tensorStore) : NotImplementedAttribute(name, cfg), - _refVector(cfg.getGrowStrategy().getDocsInitialCapacity(), - cfg.getGrowStrategy().getDocsGrowPercent(), - cfg.getGrowStrategy().getDocsGrowDelta(), - getGenerationHolder()), + _refVector(cfg.getGrowStrategy(), getGenerationHolder()), _tensorStore(tensorStore), _is_dense(cfg.tensorType().is_dense()), _emptyTensor(createEmptyTensor(cfg.tensorType())), @@ -315,6 +312,9 @@ TensorAttribute::complete_set_tensor(DocId docid, const vespalib::eval::Value& t (void) prepare_result; } -IMPLEMENT_IDENTIFIABLE_ABSTRACT(TensorAttribute, AttributeVector); +attribute::DistanceMetric +TensorAttribute::distance_metric() const { + return getConfig().distance_metric(); +} } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h index 518caef9dc9..ae6a4a302ea 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h @@ -41,7 +41,6 @@ protected: EntryRef acquire_entry_ref(DocId doc_id) const noexcept { return _refVector.acquire_elem_ref(doc_id).load_acquire(); } public: - DECLARE_IDENTIFIABLE_ABSTRACT(TensorAttribute); using RefCopyVector = vespalib::Array<EntryRef>; TensorAttribute(vespalib::stringref name, const Config &cfg, TensorStore &tensorStore); ~TensorAttribute() override; @@ -68,9 +67,7 @@ public: virtual void update_tensor(DocId docId, const document::TensorUpdate &update, bool create_empty_if_non_existing); - DistanceMetric distance_metric() const override { - return getConfig().distance_metric(); - } + DistanceMetric distance_metric() const override; uint32_t get_num_docs() const override { return getNumDocs(); } /** diff --git a/searchlib/src/vespa/searchlib/test/document_weight_attribute_helper.cpp b/searchlib/src/vespa/searchlib/test/document_weight_attribute_helper.cpp index 68cb3140eab..16296fe5a99 100644 --- a/searchlib/src/vespa/searchlib/test/document_weight_attribute_helper.cpp +++ b/searchlib/src/vespa/searchlib/test/document_weight_attribute_helper.cpp @@ -1,8 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #include "document_weight_attribute_helper.h" +#include <vespa/searchcommon/attribute/config.h> + +namespace search::test { + +AttributeVector::SP +DocumentWeightAttributeHelper::make_attr() { + attribute::Config cfg(attribute::BasicType::INT64, attribute::CollectionType::WSET); + cfg.setFastSearch(true); + return AttributeFactory::createAttribute("my_attribute", cfg); +} -namespace search { -namespace test { +DocumentWeightAttributeHelper::~DocumentWeightAttributeHelper() = default; -} // namespace search::test -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/test/document_weight_attribute_helper.h b/searchlib/src/vespa/searchlib/test/document_weight_attribute_helper.h index 3331725d578..02cc05a9ae6 100644 --- a/searchlib/src/vespa/searchlib/test/document_weight_attribute_helper.h +++ b/searchlib/src/vespa/searchlib/test/document_weight_attribute_helper.h @@ -17,20 +17,18 @@ private: IntegerAttribute *_int_attr; const IDocumentWeightAttribute *_dwa; - AttributeVector::SP make_attr() { - attribute::Config cfg(attribute::BasicType::INT64, attribute::CollectionType::WSET); - cfg.setFastSearch(true); - return AttributeFactory::createAttribute("my_attribute", cfg); - } + AttributeVector::SP make_attr(); public: - DocumentWeightAttributeHelper() : _attr(make_attr()), - _int_attr(dynamic_cast<IntegerAttribute *>(_attr.get())), - _dwa(_attr->asDocumentWeightAttribute()) + DocumentWeightAttributeHelper() + : _attr(make_attr()), + _int_attr(dynamic_cast<IntegerAttribute *>(_attr.get())), + _dwa(_attr->asDocumentWeightAttribute()) { ASSERT_TRUE(_int_attr != nullptr); ASSERT_TRUE(_dwa != nullptr); } + ~DocumentWeightAttributeHelper(); void add_docs(size_t limit) { AttributeVector::DocId docid; diff --git a/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h b/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h index 150b5bf8746..97fc49ce251 100644 --- a/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h +++ b/searchlib/src/vespa/searchlib/test/imported_attribute_fixture.h @@ -16,6 +16,7 @@ #include <vespa/searchlib/common/i_document_meta_store_context.h> #include <vespa/searchlib/query/query_term_simple.h> #include <vespa/searchcommon/attribute/attributecontent.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/testkit/test_kit.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/util/logutil.cpp b/searchlib/src/vespa/searchlib/util/logutil.cpp index 0f8e28adeb0..7f10c1598df 100644 --- a/searchlib/src/vespa/searchlib/util/logutil.cpp +++ b/searchlib/src/vespa/searchlib/util/logutil.cpp @@ -6,8 +6,7 @@ using vespalib::JSONStringer; -namespace search { -namespace util { +namespace search::util { vespalib::string LogUtil::extractLastElements(const vespalib::string & path, size_t numElems) @@ -47,5 +46,4 @@ LogUtil::logDir(JSONStringer & jstr, const vespalib::string & path, size_t numEl jstr.endObject(); } -} // namespace util -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/util/logutil.h b/searchlib/src/vespa/searchlib/util/logutil.h index e30fd15a33a..6afb654a960 100644 --- a/searchlib/src/vespa/searchlib/util/logutil.h +++ b/searchlib/src/vespa/searchlib/util/logutil.h @@ -3,8 +3,7 @@ #include <vespa/vespalib/util/jsonwriter.h> -namespace search { -namespace util { +namespace search::util { class LogUtil { public: @@ -24,6 +23,4 @@ public: static void logDir(vespalib::JSONStringer & jstr, const vespalib::string & path, size_t numElems); }; -} // namespace util -} // namespace search - +} diff --git a/searchsummary/src/vespa/searchsummary/test/mock_attribute_manager.cpp b/searchsummary/src/vespa/searchsummary/test/mock_attribute_manager.cpp index bf64231e2fd..76e6934b0c9 100644 --- a/searchsummary/src/vespa/searchsummary/test/mock_attribute_manager.cpp +++ b/searchsummary/src/vespa/searchsummary/test/mock_attribute_manager.cpp @@ -2,10 +2,10 @@ #include "mock_attribute_manager.h" #include <vespa/searchlib/attribute/attributefactory.h> -#include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/attribute/floatbase.h> #include <vespa/searchlib/attribute/integerbase.h> #include <vespa/searchlib/attribute/stringbase.h> +#include <vespa/searchcommon/attribute/config.h> #include <cassert> using search::attribute::BasicType; diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceId.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceId.java index 877fb5f213c..a4bfeebef4f 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceId.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceId.java @@ -6,15 +6,13 @@ import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceType; -import javax.annotation.concurrent.Immutable; import java.util.Objects; /** - * Identifies a service. + * Identifies a service. Immutable. * * @author hakon */ -@Immutable public class ServiceId { private final ApplicationId applicationId; private final ClusterId clusterId; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index cfcc32a76e5..df1c0de36cc 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -232,7 +232,7 @@ FileStorHandlerImpl::getQueueSize() const { size_t sum(0); for (const auto & stripe : _stripes) { - sum += stripe.getQueueSize(); + sum += stripe.get_cached_queue_size(); } return sum; } @@ -755,7 +755,10 @@ FileStorHandlerImpl::remapQueueNoLock(const RemapInfo& source, std::vector<Remap stripe(bucket).exposeQueue().emplace_back(std::move(entry)); } } - + stripe(source.bucket).unsafe_update_cached_queue_size(); + for (const auto *target: targets) { + stripe(target->bucket).unsafe_update_cached_queue_size(); + } } void @@ -840,6 +843,7 @@ FileStorHandlerImpl::Stripe::failOperations(const document::Bucket &bucket, cons ++iter; } } + update_cached_queue_size(guard); } void @@ -909,6 +913,7 @@ FileStorHandlerImpl::Stripe::Stripe(const FileStorHandlerImpl & owner, MessageSe _lock(std::make_unique<std::mutex>()), _cond(std::make_unique<std::condition_variable>()), _queue(std::make_unique<PriorityQueue>()), + _cached_queue_size(_queue->size()), _lockedBuckets(), _active_merges(0), _active_operations_stats() @@ -1029,6 +1034,7 @@ FileStorHandlerImpl::Stripe::getMessage(monitor_guard & guard, PriorityIdx & idx std::shared_ptr<api::StorageMessage> msg = std::move(iter->_command); document::Bucket bucket(iter->_bucket); idx.erase(iter); // iter not used after this point. + update_cached_queue_size(guard); if (!messageTimedOutInQueue(*msg, waitTime)) { auto locker = std::make_unique<BucketLock>(guard, *this, bucket, msg->getPriority(), @@ -1088,6 +1094,7 @@ FileStorHandlerImpl::Stripe::abort(std::vector<std::shared_ptr<api::StorageReply ++it; } } + update_cached_queue_size(lockGuard); } bool @@ -1096,6 +1103,7 @@ FileStorHandlerImpl::Stripe::schedule(MessageEntry messageEntry) { std::lock_guard guard(*_lock); _queue->emplace_back(std::move(messageEntry)); + update_cached_queue_size(guard); } _cond->notify_all(); return true; @@ -1106,6 +1114,7 @@ FileStorHandlerImpl::Stripe::schedule_and_get_next_async_message(MessageEntry en { std::unique_lock guard(*_lock); _queue->emplace_back(std::move(entry)); + update_cached_queue_size(guard); auto lockedMessage = get_next_async_message(guard); if ( ! lockedMessage.msg) { if (guard.owns_lock()) { @@ -1174,7 +1183,7 @@ FileStorHandlerImpl::Stripe::release(const document::Bucket & bucket, } Clock::time_point now_ts = Clock::now(); double latency = std::chrono::duration<double, std::milli>(now_ts - start_time).count(); - _active_operations_stats.operation_done(latency); + _active_operations_stats.guard().stats().operation_done(latency); if (!entry._exclusiveLock && entry._sharedLocks.empty()) { _lockedBuckets.erase(iter); // No more locks held } @@ -1212,7 +1221,7 @@ FileStorHandlerImpl::Stripe::lock(const monitor_guard &, const document::Bucket (void) inserted; assert(inserted.second); } - _active_operations_stats.operation_started(); + _active_operations_stats.guard().stats().operation_started(); } bool @@ -1250,10 +1259,10 @@ FileStorHandlerImpl::Stripe::operationIsInhibited(const monitor_guard & guard, c ActiveOperationsStats FileStorHandlerImpl::Stripe::get_active_operations_stats(bool reset_min_max) const { - std::lock_guard guard(*_lock); - auto result = _active_operations_stats; + auto guard = _active_operations_stats.guard(); + auto result = guard.stats(); if (reset_min_max) { - _active_operations_stats.reset_min_max(); + guard.stats().reset_min_max(); } return result; } diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h index dbef1d06dad..a3667e93fff 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h @@ -30,6 +30,7 @@ #include <boost/multi_index/sequenced_index.hpp> #include <vespa/storage/common/messagesender.h> #include <vespa/vespalib/stllike/hash_map.h> +#include <vespa/vespalib/datastore/atomic_value_wrapper.h> #include <atomic> #include <optional> @@ -74,6 +75,7 @@ public: using BucketIdx = bmi::nth_index<PriorityQueue, 2>::type; using Clock = std::chrono::steady_clock; using monitor_guard = std::unique_lock<std::mutex>; + using atomic_size_t = vespalib::datastore::AtomicValueWrapper<size_t>; class Stripe { public: @@ -96,6 +98,25 @@ public: SharedLocks _sharedLocks; }; + class SafeActiveOperationsStats { + private: + std::unique_ptr<std::mutex> _lock; + ActiveOperationsStats _stats; + struct ctor_tag {}; + public: + class Guard { + private: + std::lock_guard<std::mutex> _guard; + ActiveOperationsStats &_stats; + public: + Guard(Guard &&) = delete; + Guard(std::mutex &lock, ActiveOperationsStats &stats_in, ctor_tag) : _guard(lock), _stats(stats_in) {} + ActiveOperationsStats &stats() { return _stats; } + }; + SafeActiveOperationsStats() : _lock(std::make_unique<std::mutex>()), _stats() {} + Guard guard() { return Guard(*_lock, _stats, ctor_tag()); } + }; + Stripe(const FileStorHandlerImpl & owner, MessageSender & messageSender); Stripe(Stripe &&) noexcept; Stripe(const Stripe &) = delete; @@ -111,10 +132,11 @@ public: void broadcast() { _cond->notify_all(); } - size_t getQueueSize() const { - std::lock_guard guard(*_lock); - return _queue->size(); + size_t get_cached_queue_size() const { return _cached_queue_size.load_relaxed(); } + void unsafe_update_cached_queue_size() { + _cached_queue_size.store_relaxed(_queue->size()); } + void release(const document::Bucket & bucket, api::LockingRequirements reqOfReleasedLock, api::StorageMessage::Id lockMsgId, bool was_active_merge); void decrease_active_sync_merges_counter() noexcept; @@ -142,6 +164,12 @@ public: void setMetrics(FileStorStripeMetrics * metrics) { _metrics = metrics; } ActiveOperationsStats get_active_operations_stats(bool reset_min_max) const; private: + void update_cached_queue_size(const std::lock_guard<std::mutex> &) { + _cached_queue_size.store_relaxed(_queue->size()); + } + void update_cached_queue_size(const std::unique_lock<std::mutex> &) { + _cached_queue_size.store_relaxed(_queue->size()); + } bool hasActive(monitor_guard & monitor, const AbortBucketOperationsCommand& cmd) const; FileStorHandler::LockedMessage get_next_async_message(monitor_guard& guard); [[nodiscard]] bool operation_type_should_be_throttled(api::MessageType::Id type_id) const noexcept; @@ -158,9 +186,10 @@ public: std::unique_ptr<std::mutex> _lock; std::unique_ptr<std::condition_variable> _cond; std::unique_ptr<PriorityQueue> _queue; + atomic_size_t _cached_queue_size; LockedBuckets _lockedBuckets; uint32_t _active_merges; - mutable ActiveOperationsStats _active_operations_stats; + mutable SafeActiveOperationsStats _active_operations_stats; }; class BucketLock : public FileStorHandler::BucketLockInterface { diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index 1e091d9a6d8..ddd2457ec49 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -12,6 +12,7 @@ #include <vespa/searchlib/common/packets.h> #include <vespa/searchlib/uca/ucaconverter.h> #include <vespa/searchlib/features/setup.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/geo/zcurve.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> @@ -373,12 +374,11 @@ SearchVisitor::AttributeInserter::onPrimitive(uint32_t, const Content & c) LOG(debug, "AttributeInserter: Adding value '%s'(%d) to attribute '%s' for docid '%d'", value.toString().c_str(), c.getWeight(), _attribute.getName().c_str(), _docId); search::IExtendAttribute & attr = *_attribute.getExtendInterface(); - const vespalib::Identifiable::RuntimeClass & aInfo = _attribute.getClass(); - if (aInfo.inherits(search::IntegerAttribute::classId)) { + if (_attribute.isIntegerType()) { attr.add(value.getAsLong(), c.getWeight()); - } else if (aInfo.inherits(search::FloatingPointAttribute::classId)) { + } else if (_attribute.isFloatingPointType()) { attr.add(value.getAsDouble(), c.getWeight()); - } else if (aInfo.inherits(search::StringAttribute::classId)) { + } else if (_attribute.isStringType()) { attr.add(value.getAsString().c_str(), c.getWeight()); } else { assert(false && "We got an attribute vector that is of an unknown type"); @@ -988,7 +988,7 @@ SearchVisitor::fillAttributeVectors(const vespalib::string & documentId, const S { for (const AttrInfo & finfo : _attributeFields) { const AttributeGuard &finfoGuard(*finfo._attr); - bool isPosition = finfoGuard->getClass().inherits(search::IntegerAttribute::classId) && PositionDataType::isZCurveFieldName(finfoGuard->getName()); + bool isPosition = finfoGuard->isIntegerType() && PositionDataType::isZCurveFieldName(finfoGuard->getName()); LOG(debug, "Filling attribute '%s', isPosition='%s'", finfoGuard->getName().c_str(), isPosition ? "true" : "false"); uint32_t fieldId = finfo._field; if (isPosition) { diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java index 1edf8e4edbe..b038106843e 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java @@ -46,7 +46,7 @@ public abstract class Maintainer implements Runnable { Objects.requireNonNull(startedAt); Objects.requireNonNull(clusterHostnames); Duration initialDelay = staggeredDelay(interval, startedAt, HostName.getLocalhost(), clusterHostnames) - .plus(Duration.ofSeconds(30)); // Let the system stabilize before maintenance + .plus(Duration.ofSeconds(30)); // Let the system stabilize before maintenance service = new ScheduledThreadPoolExecutor(1, r -> new Thread(r, name() + "-worker")); service.scheduleAtFixedRate(this, initialDelay.toMillis(), interval.toMillis(), TimeUnit.MILLISECONDS); jobControl.started(name(), this); diff --git a/vespalib/src/tests/btree/btree_store/btree_store_test.cpp b/vespalib/src/tests/btree/btree_store/btree_store_test.cpp index 974aafb392a..5e2aa89b59e 100644 --- a/vespalib/src/tests/btree/btree_store/btree_store_test.cpp +++ b/vespalib/src/tests/btree/btree_store/btree_store_test.cpp @@ -43,8 +43,8 @@ protected: additions.emplace_back(i, 0); } _store.apply(root, - &additions[0], &additions[0] + additions.size(), - &removals[0], &removals[0] + removals.size()); + additions.data(), additions.data() + additions.size(), + removals.data(), removals.data() + removals.size()); return root; } static std::vector<int> make_exp_sequence(int start_key, int end_key) diff --git a/vespalib/src/tests/btree/btreeaggregation_test.cpp b/vespalib/src/tests/btree/btreeaggregation_test.cpp index b1198e3828d..f4300499fcd 100644 --- a/vespalib/src/tests/btree/btreeaggregation_test.cpp +++ b/vespalib/src/tests/btree/btreeaggregation_test.cpp @@ -258,8 +258,8 @@ MyTreeForceApplyStore::remove(EntryRef &ref, const KeyType &key, std::vector<KeyType> removals; removals.push_back(key); apply(ref, - &additions[0], &additions[additions.size()], - &removals[0], &removals[removals.size()], + additions.data(), additions.data() + additions.size(), + removals.data(), removals.data() + removals.size(), comp); return retVal; } diff --git a/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp b/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp index 7161eed24c9..eb2b00f9e20 100644 --- a/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp +++ b/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp @@ -31,10 +31,15 @@ assertUsage(const MemoryUsage & exp, const MemoryUsage & act) return retval; } +GrowStrategy +growStrategy(size_t initial, float factor, size_t delta, size_t minimal = 0) { + return GrowStrategy(initial, factor, delta, minimal); +} + TEST(RcuVectorTest, basic) { { // insert - RcuVector<int32_t> v(4, 0, 4); + RcuVector<int32_t> v(growStrategy(4, 0, 4)); for (int32_t i = 0; i < 100; ++i) { v.push_back(i); EXPECT_EQ(i, v[i]); @@ -53,7 +58,7 @@ TEST(RcuVectorTest, basic) TEST(RcuVectorTest, resize) { { // resize percent - RcuVector<int32_t> v(2, 50, 0); + RcuVector<int32_t> v(growStrategy(2, 0.50, 0)); EXPECT_EQ(2u, v.capacity()); v.push_back(0); EXPECT_EQ(2u, v.capacity()); @@ -65,7 +70,7 @@ TEST(RcuVectorTest, resize) EXPECT_TRUE(v.isFull()); } { // resize delta - RcuVector<int32_t> v(1, 0, 3); + RcuVector<int32_t> v(growStrategy(1, 0, 3)); EXPECT_EQ(1u, v.capacity()); v.push_back(0); EXPECT_EQ(1u, v.capacity()); @@ -75,7 +80,7 @@ TEST(RcuVectorTest, resize) EXPECT_TRUE(!v.isFull()); } { // resize both - RcuVector<int32_t> v(2, 200, 3); + RcuVector<int32_t> v(growStrategy(2, 2.0, 3)); EXPECT_EQ(2u, v.capacity()); v.push_back(0); EXPECT_EQ(2u, v.capacity()); @@ -87,14 +92,14 @@ TEST(RcuVectorTest, resize) EXPECT_TRUE(!v.isFull()); } { // reserve - RcuVector<int32_t> v(2, 0, 0); + RcuVector<int32_t> v(growStrategy(2, 0, 0)); EXPECT_EQ(2u, v.capacity()); v.unsafe_reserve(8); EXPECT_EQ(8u, v.capacity()); } { // explicit resize GenerationHolder g; - RcuVectorBase<int8_t> v(g); + RcuVectorBase<int8_t> v(growStrategy(16, 1.0, 0), g); v.push_back(1); v.push_back(2); g.transferHoldLists(0); @@ -120,7 +125,7 @@ TEST(RcuVectorTest, resize) TEST(RcuVectorTest, generation_handling) { - RcuVector<int32_t> v(2, 0, 2); + RcuVector<int32_t> v(growStrategy(2, 0, 2)); v.push_back(0); v.push_back(10); EXPECT_EQ(0u, v.getMemoryUsage().allocatedBytesOnHold()); @@ -143,7 +148,7 @@ TEST(RcuVectorTest, generation_handling) TEST(RcuVectorTest, reserve) { - RcuVector<int32_t> v(2, 0, 2); + RcuVector<int32_t> v(growStrategy(2, 0, 2)); EXPECT_EQ(2u, v.capacity()); EXPECT_EQ(0u, v.size()); v.push_back(0); @@ -167,7 +172,7 @@ TEST(RcuVectorTest, reserve) TEST(RcuVectorTest, memory_usage) { - RcuVector<int8_t> v(2, 0, 2); + RcuVector<int8_t> v(growStrategy(2, 0, 2)); EXPECT_TRUE(assertUsage(MemoryUsage(2,0,0,0), v.getMemoryUsage())); v.push_back(0); EXPECT_TRUE(assertUsage(MemoryUsage(2,1,0,0), v.getMemoryUsage())); @@ -183,10 +188,11 @@ TEST(RcuVectorTest, memory_usage) EXPECT_TRUE(assertUsage(MemoryUsage(6,5,0,0), v.getMemoryUsage())); } -TEST(RcuVectorTest, shrink_with_buffer_copying) -{ +void verify_shrink_with_buffer_copying(size_t initial_size, size_t absolute_minimum) { + const size_t minimal_capacity = std::max(4ul, absolute_minimum); + const size_t initial_capacity = std::max(initial_size, minimal_capacity); GenerationHolder g; - RcuVectorBase<int8_t> v(16, 100, 0, g); + RcuVectorBase<int8_t> v(growStrategy(initial_size, 1.0, 0, absolute_minimum), g); v.push_back(1); v.push_back(2); v.push_back(3); @@ -196,9 +202,9 @@ TEST(RcuVectorTest, shrink_with_buffer_copying) MemoryUsage mu; mu = v.getMemoryUsage(); mu.incAllocatedBytesOnHold(g.getHeldBytes()); - EXPECT_TRUE(assertUsage(MemoryUsage(16, 4, 0, 0), mu)); + EXPECT_TRUE(assertUsage(MemoryUsage(initial_capacity, 4, 0, 0), mu)); EXPECT_EQ(4u, v.size()); - EXPECT_EQ(16u, v.capacity()); + EXPECT_EQ(initial_capacity, v.capacity()); EXPECT_EQ(1, v[0]); EXPECT_EQ(2, v[1]); EXPECT_EQ(3, v[2]); @@ -207,7 +213,7 @@ TEST(RcuVectorTest, shrink_with_buffer_copying) v.shrink(2); g.transferHoldLists(1); EXPECT_EQ(2u, v.size()); - EXPECT_EQ(4u, v.capacity()); + EXPECT_EQ(minimal_capacity, v.capacity()); EXPECT_EQ(1, v[0]); EXPECT_EQ(2, v[1]); EXPECT_EQ(1, old[0]); @@ -217,7 +223,14 @@ TEST(RcuVectorTest, shrink_with_buffer_copying) EXPECT_EQ(2, v[1]); mu = v.getMemoryUsage(); mu.incAllocatedBytesOnHold(g.getHeldBytes()); - EXPECT_TRUE(assertUsage(MemoryUsage(4, 2, 0, 0), mu)); + EXPECT_TRUE(assertUsage(MemoryUsage(minimal_capacity, 2, 0, 0), mu)); +} + +TEST(RcuVectorTest, shrink_with_buffer_copying) +{ + verify_shrink_with_buffer_copying(16, 8); + verify_shrink_with_buffer_copying(0, 8); + verify_shrink_with_buffer_copying(0, 0); } struct ShrinkFixture { @@ -229,7 +242,7 @@ struct ShrinkFixture { ShrinkFixture() : g(), initial_capacity(4 * page_ints()), initial_size(initial_capacity / 1024 * 1000), - vec(initial_capacity, 50, 0, g, alloc::Alloc::allocMMap()), oldPtr() + vec(growStrategy(initial_capacity, 0.50, 0), g, alloc::Alloc::allocMMap()), oldPtr() { for (size_t i = 0; i < initial_size; ++i) { vec.push_back(7); @@ -272,7 +285,7 @@ TEST(RcuVectorTest, shrink_can_shrink_mmap_allocation) TEST(RcuVectorTest, small_expand) { GenerationHolder g; - RcuVectorBase<int8_t> v(1, 50, 0, g); + RcuVectorBase<int8_t> v(growStrategy(1, 0.50, 0), g); EXPECT_EQ(1u, v.capacity()); EXPECT_EQ(0u, v.size()); v.push_back(1); @@ -321,7 +334,7 @@ struct Fixture : public FixtureBase { Fixture::Fixture() : FixtureBase(), - arr(g, initial_alloc) + arr(growStrategy(16, 1.0, 0), g, initial_alloc) { arr.reserve(100); } @@ -403,7 +416,7 @@ struct StressFixture : public FixtureBase { StressFixture::StressFixture() : FixtureBase(), - arr(g, initial_alloc), + arr(growStrategy(16, 1.0, 0), g, initial_alloc), stop_read(false), read_area(1000), generation_handler(), diff --git a/vespalib/src/vespa/vespalib/data/memory.h b/vespalib/src/vespa/vespalib/data/memory.h index 2765309f4c3..795bbf2ffe7 100644 --- a/vespalib/src/vespa/vespalib/data/memory.h +++ b/vespalib/src/vespa/vespalib/data/memory.h @@ -27,9 +27,13 @@ struct Memory vespalib::string make_string() const; vespalib::stringref make_stringref() const { return stringref(data, size); } bool operator == (const Memory &rhs) const noexcept { - return ((size == rhs.size) && - ((data == rhs.data) || - (memcmp(data, rhs.data, size) == 0))); + if (size != rhs.size) { + return false; + } + if ((size == 0) || (data == rhs.data)) { + return true; + } + return (memcmp(data, rhs.data, size) == 0); } }; diff --git a/vespalib/src/vespa/vespalib/data/simple_buffer.cpp b/vespalib/src/vespa/vespalib/data/simple_buffer.cpp index 38692552575..628b110d2fe 100644 --- a/vespalib/src/vespa/vespalib/data/simple_buffer.cpp +++ b/vespalib/src/vespa/vespalib/data/simple_buffer.cpp @@ -16,7 +16,7 @@ SimpleBuffer::~SimpleBuffer() = default; Memory SimpleBuffer::obtain() { - return Memory(&_data[0], _used); + return Memory(_data.data(), _used); } Input & diff --git a/vespalib/src/vespa/vespalib/data/simple_buffer.h b/vespalib/src/vespa/vespalib/data/simple_buffer.h index f91235894ae..e94a1ceb437 100644 --- a/vespalib/src/vespa/vespalib/data/simple_buffer.h +++ b/vespalib/src/vespa/vespalib/data/simple_buffer.h @@ -36,7 +36,7 @@ public: ++_used; return *this; } - Memory get() const { return Memory(&_data[0], _used); } + Memory get() const { return Memory(_data.data(), _used); } bool operator==(const SimpleBuffer &rhs) const { return (get() == rhs.get()); } }; diff --git a/vespalib/src/vespa/vespalib/data/slime/basic_value.cpp b/vespalib/src/vespa/vespalib/data/slime/basic_value.cpp index 46525c4e6bd..9c805ccf565 100644 --- a/vespalib/src/vespa/vespalib/data/slime/basic_value.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/basic_value.cpp @@ -11,7 +11,9 @@ Memory store(Memory m, Stash & stash) { char * buf = stash.alloc(m.size); - memcpy(buf, m.data, m.size); + if (m.data) { + memcpy(buf, m.data, m.size); + } return Memory(buf, m.size); } diff --git a/vespalib/src/vespa/vespalib/data/smart_buffer.cpp b/vespalib/src/vespa/vespalib/data/smart_buffer.cpp index c0ccc9177da..de079261d6d 100644 --- a/vespalib/src/vespa/vespalib/data/smart_buffer.cpp +++ b/vespalib/src/vespa/vespalib/data/smart_buffer.cpp @@ -14,10 +14,14 @@ SmartBuffer::ensure_free(size_t bytes) if ((unused() < bytes) || ((unused() * 3) < read_len())) { size_t new_size = std::max(_data.size() * 2, read_len() + bytes); alloc::Alloc new_buf(alloc::Alloc::alloc(new_size)); - memcpy(new_buf.get(), read_ptr(), read_len()); + if (read_ptr()) { + memcpy(new_buf.get(), read_ptr(), read_len()); + } _data.swap(new_buf); } else { - memmove(_data.get(), read_ptr(), read_len()); + if (read_ptr()) { + memmove(_data.get(), read_ptr(), read_len()); + } } _write_pos = read_len(); _read_pos = 0; diff --git a/vespalib/src/vespa/vespalib/datastore/datastore.cpp b/vespalib/src/vespa/vespalib/datastore/datastore.cpp index 1c68b79453e..c0f5c122283 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastore.cpp +++ b/vespalib/src/vespa/vespalib/datastore/datastore.cpp @@ -14,4 +14,5 @@ template class DataStoreT<EntryRefT<22> >; template void vespalib::Array<vespalib::datastore::DataStoreBase::ElemHold1ListElem>::increase(size_t); template class vespalib::RcuVector<vespalib::datastore::EntryRef>; template class vespalib::RcuVectorBase<vespalib::datastore::EntryRef>; -//template void vespalib::RcuVectorBase<vespalib::datastore::EntryRef>::expandAndInsert(const vespalib::datastore::EntryRef &); +template class vespalib::RcuVector<vespalib::datastore::AtomicEntryRef>; +template class vespalib::RcuVectorBase<vespalib::datastore::AtomicEntryRef>; diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.h b/vespalib/src/vespa/vespalib/objects/nbostream.h index cc4f91d91b4..e7bed8d59ff 100644 --- a/vespalib/src/vespa/vespalib/objects/nbostream.h +++ b/vespalib/src/vespa/vespalib/objects/nbostream.h @@ -145,7 +145,7 @@ public: size_t capacity() const { return _wbuf.size(); } bool empty() const { return size() == 0; } const char * data() const { return &_rbuf[0]; } - const char * peek() const { return &_rbuf[_rp]; } + const char * peek() const { return _rbuf.c_str() + _rp; } size_t rp() const { return _rp; } nbostream & rp(size_t pos) { if (pos > _wp) fail(eof); _rp = pos; return *this; } nbostream & wp(size_t pos) { if (pos > _wbuf.size()) fail(oob); _wp = pos; return *this; } diff --git a/vespalib/src/vespa/vespalib/stllike/string.h b/vespalib/src/vespa/vespalib/stllike/string.h index 2e61d7ef0e9..7bf03895a88 100644 --- a/vespalib/src/vespa/vespalib/stllike/string.h +++ b/vespalib/src/vespa/vespalib/stllike/string.h @@ -594,7 +594,9 @@ private: void init(const void *s) noexcept { if (__builtin_expect(_sz < StackSize, true)) { _bufferSize = StackSize; - memcpy(_stack, s, _sz); + if (s) { + memcpy(_stack, s, _sz); + } _stack[_sz] = '\0'; } else { init_slower(s); diff --git a/vespalib/src/vespa/vespalib/util/array.h b/vespalib/src/vespa/vespalib/util/array.h index ec91c51b483..99bee183485 100644 --- a/vespalib/src/vespa/vespalib/util/array.h +++ b/vespalib/src/vespa/vespalib/util/array.h @@ -123,10 +123,10 @@ public: const_iterator end() const { return array(_sz); } iterator begin() { return array(0); } iterator end() { return array(_sz); } - const_reverse_iterator rbegin() const { return array(_sz) - 1; } - const_reverse_iterator rend() const { return array(0) - 1; } - reverse_iterator rbegin() { return array(_sz) - 1; } - reverse_iterator rend() { return array(0) - 1; } + const_reverse_iterator rbegin() const { return empty() ? array(0) : array(_sz) - 1; } + const_reverse_iterator rend() const { return empty() ? array(0) : array(0) - 1; } + reverse_iterator rbegin() { return empty() ? array(0) : array(_sz) - 1; } + reverse_iterator rend() { return empty() ? array(0) : array(0) - 1; } size_t size() const { return _sz; } size_t byteSize() const { return _sz * sizeof(T); } size_t byteCapacity() const { return _array.size(); } diff --git a/vespalib/src/vespa/vespalib/util/generationholder.h b/vespalib/src/vespa/vespalib/util/generationholder.h index 971304a2a11..ed68a80a308 100644 --- a/vespalib/src/vespa/vespalib/util/generationholder.h +++ b/vespalib/src/vespa/vespalib/util/generationholder.h @@ -2,9 +2,9 @@ #pragma once +#include "generationhandler.h" #include <vector> #include <memory> -#include "generationhandler.h" namespace vespalib { diff --git a/vespalib/src/vespa/vespalib/util/growstrategy.h b/vespalib/src/vespa/vespalib/util/growstrategy.h index dad83a5f704..c46c0b45667 100644 --- a/vespalib/src/vespa/vespalib/util/growstrategy.h +++ b/vespalib/src/vespa/vespalib/util/growstrategy.h @@ -9,32 +9,32 @@ namespace vespalib { class GrowStrategy { private: size_t _initialCapacity; - float _growFactor; + size_t _minimumCapacity; size_t _growDelta; + float _growFactor; public: GrowStrategy() noexcept - : GrowStrategy(1024, 0.5, 0) + : GrowStrategy(1024, 0.5, 0, 0) {} - GrowStrategy(size_t initialCapacity, float growPercent, size_t growDelta) noexcept + GrowStrategy(size_t initialCapacity, float growPercent, size_t growDelta, size_t minimumCapacity) noexcept : _initialCapacity(initialCapacity), - _growFactor(growPercent), - _growDelta(growDelta) + _minimumCapacity(minimumCapacity), + _growDelta(growDelta), + _growFactor(growPercent) { } - static GrowStrategy make(size_t initialCapacity, float growFactor, size_t growDelta) noexcept { - return GrowStrategy(initialCapacity, growFactor, growDelta); - } - + size_t getMinimumCapacity() const noexcept { return _minimumCapacity; } size_t getInitialCapacity() const noexcept { return _initialCapacity; } - size_t getGrowPercent() const noexcept { return _growFactor*100; } float getGrowFactor() const noexcept { return _growFactor; } size_t getGrowDelta() const noexcept { return _growDelta; } - void setInitialCapacity(size_t v) noexcept { _initialCapacity = v; } - void setGrowDelta(size_t v) noexcept { _growDelta = v; } + + void setInitialCapacity(size_t v) noexcept { _initialCapacity = v; } + void setGrowDelta(size_t v) noexcept { _growDelta = v; } bool operator==(const GrowStrategy & rhs) const noexcept { return (_initialCapacity == rhs._initialCapacity && + _minimumCapacity == rhs._minimumCapacity && _growFactor == rhs._growFactor && _growDelta == rhs._growDelta); } diff --git a/vespalib/src/vespa/vespalib/util/rcuvector.cpp b/vespalib/src/vespa/vespalib/util/rcuvector.cpp index e75836218e6..ec8c2d4d244 100644 --- a/vespalib/src/vespa/vespalib/util/rcuvector.cpp +++ b/vespalib/src/vespa/vespalib/util/rcuvector.cpp @@ -7,6 +7,7 @@ namespace vespalib { template class RcuVectorBase<uint8_t>; template class RcuVectorBase<uint16_t>; template class RcuVectorBase<uint32_t>; +template class RcuVectorBase<uint64_t>; template class RcuVectorBase<int8_t>; template class RcuVectorBase<int16_t>; template class RcuVectorBase<int32_t>; @@ -17,6 +18,7 @@ template class RcuVectorBase<double>; template class RcuVector<uint8_t>; template class RcuVector<uint16_t>; template class RcuVector<uint32_t>; +template class RcuVector<uint64_t>; template class RcuVector<int8_t>; template class RcuVector<int16_t>; template class RcuVector<int32_t>; @@ -27,6 +29,7 @@ template class RcuVector<double>; template class RcuVectorHeld<uint8_t>; template class RcuVectorHeld<uint16_t>; template class RcuVectorHeld<uint32_t>; +template class RcuVectorHeld<uint64_t>; template class RcuVectorHeld<int8_t>; template class RcuVectorHeld<int16_t>; template class RcuVectorHeld<int32_t>; diff --git a/vespalib/src/vespa/vespalib/util/rcuvector.h b/vespalib/src/vespa/vespalib/util/rcuvector.h index ce48082099c..5d084fe3815 100644 --- a/vespalib/src/vespa/vespalib/util/rcuvector.h +++ b/vespalib/src/vespa/vespalib/util/rcuvector.h @@ -46,17 +46,11 @@ protected: private: ArrayType _data; std::atomic<const T*> _vector_start; - size_t _growPercent; - size_t _growDelta; + GrowStrategy _growStrategy; GenerationHolderType &_genHolder; - size_t calcNewSize(size_t baseSize) const { - size_t delta = (baseSize * _growPercent / 100) + _growDelta; - return baseSize + std::max(delta, static_cast<size_t>(1)); - } - size_t calcNewSize() const { - return calcNewSize(_data.capacity()); - } + size_t calcNewSize(size_t baseSize) const; + size_t calcNewSize() const; void expand(size_t newCapacity); void expandAndInsert(const T & v); void update_vector_start(); @@ -65,8 +59,6 @@ protected: public: using ValueType = T; - RcuVectorBase(GenerationHolderType &genHolder, - const Alloc &initialAlloc = Alloc::alloc()); /** * Construct a new vector with the given initial capacity and grow @@ -75,10 +67,6 @@ public: * New capacity is calculated based on old capacity and grow parameters: * nc = oc + (oc * growPercent / 100) + growDelta. **/ - RcuVectorBase(size_t initialCapacity, size_t growPercent, size_t growDelta, - GenerationHolderType &genHolder, - const Alloc &initialAlloc = Alloc::alloc()); - RcuVectorBase(GrowStrategy growStrategy, GenerationHolderType &genHolder, const Alloc &initialAlloc = Alloc::alloc()); @@ -185,7 +173,6 @@ public: * New capacity is calculated based on old capacity and grow parameters: * nc = oc + (oc * growPercent / 100) + growDelta. **/ - RcuVector(size_t initialCapacity, size_t growPercent, size_t growDelta); RcuVector(GrowStrategy growStrategy); ~RcuVector(); diff --git a/vespalib/src/vespa/vespalib/util/rcuvector.hpp b/vespalib/src/vespa/vespalib/util/rcuvector.hpp index 81928251b19..97a73a73cc9 100644 --- a/vespalib/src/vespa/vespalib/util/rcuvector.hpp +++ b/vespalib/src/vespa/vespalib/util/rcuvector.hpp @@ -18,6 +18,17 @@ template <typename T> RcuVectorHeld<T>::~RcuVectorHeld() = default; template <typename T> +size_t RcuVectorBase<T>::calcNewSize(size_t baseSize) const { + size_t delta = (baseSize * _growStrategy.getGrowFactor()) + _growStrategy.getGrowDelta(); + size_t newSize = baseSize + std::max(delta, static_cast<size_t>(1)); + return std::max(newSize, _growStrategy.getMinimumCapacity()); +} +template <typename T> +size_t RcuVectorBase<T>::calcNewSize() const { + return calcNewSize(_data.capacity()); +} + +template <typename T> void RcuVectorBase<T>::unsafe_resize(size_t n) { _data.resize(n); @@ -111,44 +122,19 @@ RcuVectorBase<T>::shrink(size_t newSize) } template <typename T> -RcuVectorBase<T>::RcuVectorBase(GenerationHolderType &genHolder, - const Alloc &initialAlloc) - : _data(initialAlloc), - _vector_start(nullptr), - _growPercent(100), - _growDelta(0), - _genHolder(genHolder) -{ - _data.reserve(16); - update_vector_start(); -} - -template <typename T> -RcuVectorBase<T>::RcuVectorBase(size_t initialCapacity, - size_t growPercent, - size_t growDelta, +RcuVectorBase<T>::RcuVectorBase(GrowStrategy growStrategy, GenerationHolderType &genHolder, const Alloc &initialAlloc) : _data(initialAlloc), _vector_start(nullptr), - _growPercent(growPercent), - _growDelta(growDelta), + _growStrategy(growStrategy), _genHolder(genHolder) { - _data.reserve(initialCapacity); + _data.reserve(_growStrategy.getInitialCapacity()); update_vector_start(); } template <typename T> -RcuVectorBase<T>::RcuVectorBase(GrowStrategy growStrategy, - GenerationHolderType &genHolder, - const Alloc &initialAlloc) - : RcuVectorBase(growStrategy.getInitialCapacity(), growStrategy.getGrowPercent(), - growStrategy.getGrowDelta(), genHolder, initialAlloc) -{ -} - -template <typename T> MemoryUsage RcuVectorBase<T>::getMemoryUsage() const { @@ -162,7 +148,7 @@ template <typename T> void RcuVectorBase<T>::update_vector_start() { - _vector_start.store(&_data[0], std::memory_order_release); + _vector_start.store(_data.data(), std::memory_order_release); } template <typename T> @@ -181,14 +167,7 @@ RcuVector<T>::onReallocation() { template <typename T> RcuVector<T>::RcuVector() - : RcuVectorBase<T>(_genHolderStore), - _generation(0), - _genHolderStore() -{ } - -template <typename T> -RcuVector<T>::RcuVector(size_t initialCapacity, size_t growPercent, size_t growDelta) - : RcuVectorBase<T>(initialCapacity, growPercent, growDelta, _genHolderStore), + : RcuVectorBase<T>(GrowStrategy(16, 1.0, 0, 0), _genHolderStore), _generation(0), _genHolderStore() { } diff --git a/vespamalloc/src/vespamalloc/malloc/load_as_huge.cpp b/vespamalloc/src/vespamalloc/malloc/load_as_huge.cpp index 00916ec9fac..a5d595a8486 100644 --- a/vespamalloc/src/vespamalloc/malloc/load_as_huge.cpp +++ b/vespamalloc/src/vespamalloc/malloc/load_as_huge.cpp @@ -21,7 +21,7 @@ mmap_huge(size_t sz) { void * mem = mmap(nullptr, sz, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0); assert(mem != MAP_FAILED); if (madvise(mem, sz, MADV_HUGEPAGE) != 0) { - perror("load_as_huge.cpp:mmap_huge => madvise( MADV_HUGEPAGE) failed"); + fprintf(stderr, "load_as_huge.cpp:mmap_huge = > madvise(%p, %ld, MADV_HUGEPAGE) FAILED, errno= %d = %s\n", mem, sz, errno, strerror(errno)); } return mem; } @@ -76,7 +76,7 @@ remap_segments(size_t base_vaddr, const Elf64_Phdr * segments, size_t count) { last_end = dest + sz; if (madvise(dest, sz, MADV_HUGEPAGE) != 0) { - perror("load_as_huge.cpp:mmap_huge => madvise( MADV_HUGEPAGE) failed"); + fprintf(stderr, "load_as_huge.cpp:remap_segments => madvise(%p, %ld, MADV_HUGEPAGE) FAILED, errno= %d = %s\n", dest, sz, errno, strerror(errno)); } non_optimized_non_inlined_memcpy(dest, reinterpret_cast<void*>(vaddr), sz); int prot = PROT_READ; @@ -89,7 +89,7 @@ remap_segments(size_t base_vaddr, const Elf64_Phdr * segments, size_t count) { void * remapped = mremap(dest, sz, sz, MREMAP_FIXED | MREMAP_MAYMOVE, vaddr); assert(remapped != MAP_FAILED); assert(remapped == reinterpret_cast<void *>(vaddr)); - fprintf(stderr, "remapped dest=%p, size=%lu to %p\n", dest, sz, remapped); + fprintf(stdout, "remapped dest=%p, size=%lu to %p\n", dest, sz, remapped); } assert(new_huge_end >= last_end); if (new_huge_end > last_end) { @@ -103,16 +103,16 @@ int remapElfHeader(struct dl_phdr_info *info, size_t info_size, void *data) { (void) info_size; (void) data; - fprintf(stderr, "processing elf header '%s' with %d entries, start=%lx\n", + fprintf(stdout, "processing elf header '%s' with %d entries, start=%lx\n", info->dlpi_name, info->dlpi_phnum, info->dlpi_addr); for (int i = 0; i < info->dlpi_phnum; i++) { const Elf64_Phdr &phdr = info->dlpi_phdr[i]; - //fprintf(stderr, "p_vaddr=%lx p_paddr=%lx, p_offset=%lx p_filesz=%lx, p_memsz=%lx, allign=%lu type=%d flags=%x\n", + //fprintf(stdout, "p_vaddr=%lx p_paddr=%lx, p_offset=%lx p_filesz=%lx, p_memsz=%lx, allign=%lu type=%d flags=%x\n", // phdr.p_vaddr, phdr.p_paddr, phdr.p_offset, phdr.p_filesz, phdr.p_memsz, phdr.p_align, phdr.p_type, phdr.p_flags); if ((phdr.p_type == PT_LOAD) && (phdr.p_flags == (PF_R | PF_X))) { //void *vaddr = reinterpret_cast<void *>(info->dlpi_addr + phdr.p_vaddr); //uint64_t size = phdr.p_filesz; - //fprintf(stderr, "LOAD_RX: vaddr=%lx p_filesz=%lu, p_memsz=%lu\n", phdr.p_vaddr, phdr.p_filesz, phdr.p_memsz); + //fprintf(stdout, "LOAD_RX: vaddr=%lx p_filesz=%lu, p_memsz=%lu\n", phdr.p_vaddr, phdr.p_filesz, phdr.p_memsz); remap_segments(info->dlpi_addr, &phdr, 1); } } @@ -126,7 +126,7 @@ extern "C" int remapTextWithHugePages(); int remapTextWithHugePages() { int retval = dl_iterate_phdr(remapElfHeader, nullptr); - fprintf(stderr, "dl_iterate_phdr() = %d\n", retval); + fprintf(stdout, "dl_iterate_phdr() = %d\n", retval); return retval; } diff --git a/yolean/src/main/java/com/yahoo/yolean/concurrent/ThreadRobustList.java b/yolean/src/main/java/com/yahoo/yolean/concurrent/ThreadRobustList.java index 978ee87d20c..f6d8b68416c 100644 --- a/yolean/src/main/java/com/yahoo/yolean/concurrent/ThreadRobustList.java +++ b/yolean/src/main/java/com/yahoo/yolean/concurrent/ThreadRobustList.java @@ -18,7 +18,7 @@ import java.util.NoSuchElementException; * <p>The <code>ThreadRobustList</code> does not permit adding <code>null</code> items.</p> * <p>The usage of <code>ThreadRobustList</code> has no memory consistency effects. </p> * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen * @author bratseth * @since 5.1.15 */ diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/api/VespaCurator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/api/VespaCurator.java index dc717f6d65d..32aef2da118 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/api/VespaCurator.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/api/VespaCurator.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.curator.api; +import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.path.Path; import java.time.Duration; @@ -14,6 +15,6 @@ import java.time.Duration; public interface VespaCurator { /** Create and acquire a re-entrant lock in given path. This blocks until the lock is acquired or timeout elapses. */ - AutoCloseable lock(Path path, Duration timeout); + AutoCloseable lock(Path path, Duration timeout) throws UncheckedTimeoutException; } |
